rollbacking_flags.patch J_ROLLBACKING should be 4, not 3, because we're using it as a flag. close_best_effort.patch Try to close as much as possible in jclose(). unlock_in_jtrans_add.patch Unlock ts->lock if malloc() fails inside jtrans_add(). fix_constants.patch Fix the constants numbering. use_ssize_t.patch Make commit and rollback return ssize_t instead of int. flags_are_unsigned.patch Make flags 32 bits and unsigned, as they should be. initialize_fds.patch Initialize all file descriptors properly so error paths work as expected. open_exit.patch Create a proper error exit path in jopen() to cleanup in case of errors. close_locking.patch Remove bogus lock inside jclose(), caller must take care of it, just like in real life. jsync_locking.patch Fix locking in jsync and use fs->lock to protect the linger list. doc_updates.patch Various documentation updates. jclose_check_fd.patch Check if fds exist before closing them. lseek_on_incomplete_io.patch Advance the file pointer correctly in partial reads. initialize_results.patch Initialize results in jfsck(). clear_commit_flag.patch Clear the commit flag before committing. committed_mispelling.patch I wrote "COMMITED" and it's "COMMITTED". Fix it now while we still can. upgrade_doc.patch Add a document on upgrading the library. python_bindings.patch Add python bindings. mailing_list.patch Add documentation on the mailing list. version-0.19.patch Version 0.19 INSTALL | 4 Make.conf | 2 Makefile | 7 README | 4 UPGRADING | 19 + bindings/python/libjio.c | 626 +++++++++++++++++++++++++++++++++++++++++++++++ bindings/python/setup.py | 16 + check.c | 25 + doc/guide.lyx | 37 ++ doc/guide.txt | 28 +- doc/libjio.3 | 20 - doc/libjio.lyx | 26 - doc/libjio.txt | 34 +- libjio.h | 20 - trans.c | 111 +++++--- unix.c | 2 16 files changed, 875 insertions(+), 106 deletions(-) unchanged: --- cur-root/libjio.h 2004-08-28 21:22:56.677731320 -0300 +++ cur-root/libjio.h 2004-08-30 11:58:56.286143744 -0300 @@ -27,7 +27,7 @@ int jdirfd; /* journal directory file descriptor */ int jfd; /* journal's lock file descriptor */ unsigned int *jmap; /* journal's lock file mmap area */ - int flags; /* journal flags */ + uint32_t flags; /* journal flags */ struct jlinger *ltrans; /* lingered transactions */ pthread_mutex_t lock; /* a soft lock used in some operations */ }; @@ -49,7 +49,7 @@ struct jfs *fs; /* journal file structure to operate on */ char *name; /* name of the transaction file */ int id; /* transaction id */ - int flags; /* misc flags */ + uint32_t flags; /* transaction flags */ unsigned int numops; /* quantity of operations in the list */ pthread_mutex_t lock; /* used to modify the operation list */ struct joper *op; /* list of operations */ @@ -95,8 +95,8 @@ int jopen(struct jfs *fs, const char *name, int flags, int mode, int jflags); void jtrans_init(struct jfs *fs, struct jtrans *ts); int jtrans_add(struct jtrans *ts, const void *buf, size_t count, off_t offset); -int jtrans_commit(struct jtrans *ts); -int jtrans_rollback(struct jtrans *ts); +ssize_t jtrans_commit(struct jtrans *ts); +ssize_t jtrans_rollback(struct jtrans *ts); void jtrans_free(struct jtrans *ts); int jsync(struct jfs *fs); int jclose(struct jfs *fs); @@ -132,15 +132,13 @@ FILE *jfsopen(struct jfs *stream, const char *mode); -/* jfs constants */ +/* jfs and jtrans constants */ #define J_NOLOCK 1 /* don't lock the file before operating on it */ #define J_NOROLLBACK 2 /* no need to read rollback information */ -#define J_LINGER 3 /* use lingering transactions */ - -/* jtrans constants */ -#define J_COMMITED 1 /* mark a transaction as commited */ -#define J_ROLLBACKED 2 /* mark a transaction as rollbacked */ -#define J_ROLLBACKING 3 /* mark a transaction as rollbacking */ +#define J_LINGER 4 /* use lingering transactions */ +#define J_COMMITTED 8 /* mark a transaction as committed */ +#define J_ROLLBACKED 16 /* mark a transaction as rollbacked */ +#define J_ROLLBACKING 32 /* mark a transaction as rollbacking */ /* disk constants */ #define J_DISKHEADSIZE 12 /* length of disk_header */ unchanged: --- cur-root/trans.c 2004-08-30 11:55:51.042305088 -0300 +++ cur-root/trans.c 2004-08-30 11:59:25.947634512 -0300 @@ -140,16 +140,20 @@ pthread_mutex_lock(&(ts->lock)); if (ts->op == NULL) { ts->op = malloc(sizeof(struct joper)); - if (ts->op == NULL) + if (ts->op == NULL) { + pthread_mutex_unlock(&(ts->lock)); return 0; + } jop = ts->op; jop->prev = NULL; } else { for (tmpop = ts->op; tmpop->next != NULL; tmpop = tmpop->next) ; tmpop->next = malloc(sizeof(struct joper)); - if (tmpop->next == NULL) + if (tmpop->next == NULL) { + pthread_mutex_unlock(&(ts->lock)); return 0; + } tmpop->next->prev = tmpop; jop = tmpop->next; } @@ -176,9 +180,10 @@ } /* commit a transaction */ -int jtrans_commit(struct jtrans *ts) +ssize_t jtrans_commit(struct jtrans *ts) { - int id, rv, fd = -1; + int id, fd = -1; + ssize_t rv; uint32_t csum; char *name; unsigned char *buf_init, *bufp; @@ -189,6 +194,10 @@ pthread_mutex_lock(&(ts->lock)); + /* clear the flags */ + ts->flags = ts->flags & ~J_COMMITTED; + ts->flags = ts->flags & ~J_ROLLBACKED; + name = (char *) malloc(PATH_MAX); if (name == NULL) goto exit; @@ -358,9 +367,11 @@ linger->id = id; linger->name = strdup(name); - linger->next = ts->fs->ltrans; + pthread_mutex_lock(&(ts->fs->lock)); + linger->next = ts->fs->ltrans; ts->fs->ltrans = linger; + pthread_mutex_unlock(&(ts->fs->lock)); } else { /* the transaction has been applied, so we cleanup and remove * it from the disk */ @@ -369,7 +380,7 @@ } /* mark the transaction as commited, _after_ it was removed */ - ts->flags = ts->flags | J_COMMITED; + ts->flags = ts->flags | J_COMMITTED; rollback_exit: @@ -384,7 +395,7 @@ * Transactions that were successfuly recovered by rollbacking them * will have J_ROLLBACKED in their flags, so the caller can verify if * the failure was recovered or not. */ - if (!(ts->flags & J_COMMITED) && !(ts->flags & J_ROLLBACKING)) { + if (!(ts->flags & J_COMMITTED) && !(ts->flags & J_ROLLBACKING)) { rv = ts->flags; ts->flags = ts->flags | J_NOLOCK | J_ROLLBACKING; if (jtrans_rollback(ts) >= 0) { @@ -395,7 +406,7 @@ } unlink_exit: - if (!(ts->flags & J_COMMITED)) { + if (!(ts->flags & J_COMMITTED)) { unlink(name); free_tid(ts->fs, ts->id); } @@ -410,7 +421,7 @@ pthread_mutex_unlock(&(ts->lock)); /* return the length only if it was properly commited */ - if (ts->flags & J_COMMITED) + if (ts->flags & J_COMMITTED) return written; else return -1; @@ -418,9 +429,9 @@ } /* rollback a transaction */ -int jtrans_rollback(struct jtrans *ts) +ssize_t jtrans_rollback(struct jtrans *ts) { - int rv; + ssize_t rv; struct jtrans newts; struct joper *op, *curop, *lop; @@ -496,11 +507,16 @@ /* open a file */ int jopen(struct jfs *fs, const char *name, int flags, int mode, int jflags) { - int fd, jfd, rv; + int jfd, rv; unsigned int t; char jdir[PATH_MAX], jlockfile[PATH_MAX]; struct stat sinfo; + fs->fd = -1; + fs->jfd = -1; + fs->jdirfd = -1; + fs->jmap = MAP_FAILED; + /* we always need read and write access, because when we commit a * transaction we read the current contents before applying, and write * access is needed for locking with fcntl */ @@ -508,19 +524,14 @@ flags = flags & ~O_RDONLY; flags = flags | O_RDWR; - fd = open(name, flags, mode); - if (fd < 0) - return -1; - - fs->fd = fd; fs->name = strdup(name); fs->flags = jflags; fs->ltrans = NULL; - /* Note on fs->lock usage: this lock is used only inside the wrappers, - * and exclusively to protect the file pointer. This means that it - * must only be held while performing operations that depend or alter - * the file pointer (jread, jreadv, jwrite, jwritev), but the others + /* Note on fs->lock usage: this lock is used only to protect the file + * pointer and the linger list. This means that it must only be held + * while performing operations that depend or alter the file pointer + * or the linger list (jread, jreadv, jwrite, jwritev), but the others * (jpread, jpwrite) are left unprotected because they can be * performed in paralell as long as they don't affect the same portion * of the file (this is protected by lockf). The lock doesn't slow @@ -531,23 +542,27 @@ * real life. */ pthread_mutex_init( &(fs->lock), NULL); + fs->fd = open(name, flags, mode); + if (fs->fd < 0) + goto error_exit; + if (!get_jdir(name, jdir)) - return -1; + goto error_exit; rv = mkdir(jdir, 0750); rv = lstat(jdir, &sinfo); if (rv < 0 || !S_ISDIR(sinfo.st_mode)) - return -1; + goto error_exit; /* open the directory, we will use it to flush transaction files' * metadata in jtrans_commit() */ fs->jdirfd = open(jdir, O_RDONLY); if (fs->jdirfd < 0) - return -1; + goto error_exit; snprintf(jlockfile, PATH_MAX, "%s/%s", jdir, "lock"); jfd = open(jlockfile, O_RDWR | O_CREAT, 0600); if (jfd < 0) - return -1; + goto error_exit; /* initialize the lock file by writing the first tid to it, but only * if its empty, otherwise there is a race if two processes call @@ -559,7 +574,7 @@ rv = spwrite(jfd, &t, sizeof(t), 0); if (rv != sizeof(t)) { plockf(jfd, F_UNLOCK, 0, 0); - return -1; + goto error_exit; } } plockf(jfd, F_UNLOCK, 0, 0); @@ -569,9 +584,16 @@ fs->jmap = (unsigned int *) mmap(NULL, sizeof(unsigned int), PROT_READ | PROT_WRITE, MAP_SHARED, jfd, 0); if (fs->jmap == MAP_FAILED) - return -1; + goto error_exit; - return fd; + return fs->fd; + +error_exit: + /* if there was an error, clean up as much as possible so we don't + * leak anything, and return failure; jclose just does this cleaning + * for us */ + jclose(fs); + return -1; } /* sync a file (makes sense only if using lingering transactions) */ @@ -580,12 +602,14 @@ int rv; struct jlinger *linger, *ltmp; - pthread_mutex_lock(&(fs->lock)); + if (fs->fd < 0) + return -1; rv = fsync(fs->fd); if (rv != 0) - goto exit; + return rv; + pthread_mutex_lock(&(fs->lock)); linger = fs->ltrans; while (linger != NULL) { free_tid(fs, linger->id); @@ -599,27 +623,32 @@ } -exit: pthread_mutex_unlock(&(fs->lock)); - return rv; + return 0; } /* close a file */ int jclose(struct jfs *fs) { + int ret; + + ret = 0; + if (jsync(fs)) - return -1; - if (close(fs->fd)) - return -1; - if (close(fs->jfd)) - return -1; - if (close(fs->jdirfd)) - return -1; + ret = -1; + if (fs->fd < 0 || close(fs->fd)) + ret = -1; + if (fs->jfd < 0 || close(fs->jfd)) + ret = -1; + if (fs->jdirfd < 0 || close(fs->jdirfd)) + ret = -1; if (fs->name) /* allocated by strdup() in jopen() */ free(fs->name); - munmap(fs->jmap, sizeof(unsigned int)); + if (fs->jmap != MAP_FAILED) + munmap(fs->jmap, sizeof(unsigned int)); + pthread_mutex_destroy(&(fs->lock)); - return 0; + return ret; } unchanged: --- cur-root/check.c 2004-08-28 21:22:57.397621880 -0300 +++ cur-root/check.c 2004-08-30 02:39:16.000000000 -0300 @@ -95,7 +95,7 @@ /* check the journal and rollback incomplete transactions */ int jfsck(const char *name, struct jfsck_result *res) { - int fd, tfd, rv, i, ret; + int tfd, rv, i, ret; unsigned int maxtid; uint32_t csum1, csum2; char jdir[PATH_MAX], jlockfile[PATH_MAX], tname[PATH_MAX]; @@ -108,20 +108,30 @@ unsigned char *map; off_t filelen; - fd = tfd = -1; + tfd = -1; filelen = 0; dir = NULL; - fs.jmap = NULL; + fs.fd = -1; + fs.jfd = -1; + fs.jdirfd = -1; + fs.jmap = MAP_FAILED; map = NULL; ret = 0; - fd = open(name, O_RDWR | O_SYNC | O_LARGEFILE); - if (fd < 0) { + res->total = 0; + res->invalid = 0; + res->in_progress = 0; + res->broken = 0; + res->corrupt = 0; + res->apply_error = 0; + res->reapplied = 0; + + fs.fd = open(name, O_RDWR | O_SYNC | O_LARGEFILE); + if (fs.fd < 0) { ret = J_ENOENT; goto exit; } - fs.fd = fd; fs.name = (char *) name; if (!get_jdir(name, jdir)) { @@ -154,7 +164,6 @@ PROT_READ | PROT_WRITE, MAP_SHARED, fs.jfd, 0); if (fs.jmap == MAP_FAILED) { ret = J_ENOJOURNAL; - fs.jmap = NULL; goto exit; } @@ -283,7 +292,7 @@ close(fs.jdirfd); if (dir != NULL) closedir(dir); - if (fs.jmap != NULL) + if (fs.jmap != MAP_FAILED) munmap(fs.jmap, sizeof(unsigned int)); return ret; unchanged: --- cur/doc/libjio.lyx~doc_updates 2004-08-30 00:51:59.220321208 -0300 +++ cur-root/doc/libjio.lyx 2004-08-30 00:51:59.229319840 -0300 @@ -106,23 +106,23 @@ Inside, there are two kind of files: the The first one is used as a general lock and holds the next transaction ID to assign, and there is only one; the second one holds one transaction, which is composed by a header of fixed size and a variable-size payload, - and can be as many as in-flight transactions. + and can be as many as the number of in-flight transactions. \layout Standard -This impose some restrictions to the kind of operations you can perform +This imposes some restrictions on the kind of operations you can perform over a file while it's currently being used: you can't move it (because the journal directory name depends on the filename) and you can't unlink it (for similar reasons). \layout Standard -This warnings are no different from a normal simultaneous use under classic - UNIX environments, but they are here to remind you that even tho the library - warantees a lot and eases many things from its user (specially from complex - cases, like multiple threads using the file at the same time), you should - still be careful when doing strange things with files while working on - them. +These warnings are no different from a normal simultaneous use under classic + UNIX environments, but they are here to remind you that even though the + library warantees a lot and eases many things for its user (specially from + complex cases, like multiple threads using the file at the same time), + you should still be careful when doing strange things with files while + working on them. \layout Subsection @@ -137,8 +137,8 @@ The header holds basic information about the ID, some flags, and the amount of operations it includes. Then the payload has all the operations one after the other, divided in two parts: the first one includes static information about the operation - (the lenght of the data, the offset of the file where it should be applied, - etc.) and the data itself, which is saved by the library prior applying + (the length of the data, the offset of the file where it should be applied, + etc.) and the data itself, which is saved by the library prior to applying the commit, so transactions can be reapplied if necesary. The last part is just a 32 bit integer with the checksum of all the previous data, used for integrity verification during the recovery process. @@ -192,15 +192,15 @@ jtrans_commit() : \layout Itemize -Lock the file offsets where the commit takes place -\layout Itemize - Open the transaction file \layout Itemize Write the header \layout Itemize +Lock the file offsets where the commit takes place +\layout Itemize + Read all the previous data from the file \layout Itemize unchanged: --- cur/doc/libjio.txt~doc_updates 2004-08-30 00:51:59.221321056 -0300 +++ cur-root/doc/libjio.txt 2004-08-30 00:51:59.229319840 -0300 @@ -71,22 +71,22 @@ lock and holds the next transaction ID t there is only one; the second one holds one transaction, which is composed by a header of fixed size and a variable-size payload, and can be as many as -in-flight transactions. +the number of in-flight transactions. -This impose some restrictions to the kind of operations -you can perform over a file while it's currently being -used: you can't move it (because the journal directory -name depends on the filename) and you can't unlink it -(for similar reasons). +This imposes some restrictions on the kind of +operations you can perform over a file while it's +currently being used: you can't move it (because the +journal directory name depends on the filename) and you +can't unlink it (for similar reasons). -This warnings are no different from a normal +These warnings are no different from a normal simultaneous use under classic UNIX environments, but -they are here to remind you that even tho the library -warantees a lot and eases many things from its user -(specially from complex cases, like multiple threads -using the file at the same time), you should still be -careful when doing strange things with files while -working on them. +they are here to remind you that even though the +library warantees a lot and eases many things for its +user (specially from complex cases, like multiple +threads using the file at the same time), you should +still be careful when doing strange things with files +while working on them. 2.1 The transaction file @@ -98,9 +98,9 @@ transaction itself, including the ID, so the amount of operations it includes. Then the payload has all the operations one after the other, divided in two parts: the first one includes static information -about the operation (the lenght of the data, the offset +about the operation (the length of the data, the offset of the file where it should be applied, etc.) and the -data itself, which is saved by the library prior +data itself, which is saved by the library prior to applying the commit, so transactions can be reapplied if necesary. The last part is just a 32 bit integer with the checksum of all the previous data, used for @@ -134,12 +134,12 @@ Well, so much for talking, now let's get applies commits in a very simple and straightforward way, inside jtrans_commit(): -* Lock the file offsets where the commit takes place - * Open the transaction file * Write the header +* Lock the file offsets where the commit takes place + * Read all the previous data from the file * Write the data in the transaction unchanged: --- cur/doc/guide.lyx~doc_updates 2004-08-30 00:51:59.223320752 -0300 +++ cur-root/doc/guide.lyx 2004-08-30 00:51:59.230319688 -0300 @@ -482,6 +482,43 @@ jiofsck , which can be used from the shell to perform the checking and cleanup. \layout Subsection +Threads and locking +\layout Standard + +The library is completely safe to use in multithreaded applications; however, + there are some very basic and intuitive locking rules you have to bear + in mind. +\layout Standard + +Most is fully threadsafe so you don't need to worry about concurrency; in + fact, a lot of effort has been put in making paralell operation safe and + fast. +\layout Standard + +You need to care only when opening, closing and checking for integrity. + In practise, that means that you shouldn't call +\family typewriter +jopen() +\family default +, +\family typewriter +jclose() +\family default + in paralell with the same jfs structure, or in the middle of an I/O operation, + just like you do when using the normal UNIX calls. + In the case of +\family typewriter +jfsck() +\family default +, you shouldn't invoke it for the same file more than once at the time; + while it will cope with that situation, it's not recommended. +\layout Standard + +All other operations (commiting a transaction, rollbacking it, adding operations +, etc.) and all the wrappers are safe and don't require any special consideration +s. +\layout Subsection + Lingering transactions \layout Standard unchanged: --- cur/doc/guide.txt~doc_updates 2004-08-30 00:51:59.224320600 -0300 +++ cur-root/doc/guide.txt 2004-08-30 00:51:59.230319688 -0300 @@ -12,7 +12,8 @@ Table of Contents 5.1 Interaction with reads 5.2 Rollback 5.3 Integrity checking and recovery - 5.4 Lingering transactions + 5.4 Threads and locking + 5.5 Lingering transactions 6 Disk layout 7 Other APIs 7.1 UNIX API @@ -234,7 +235,30 @@ You can also do this manually with an ut jiofsck, which can be used from the shell to perform the checking and cleanup. -5.4 Lingering transactions +5.4 Threads and locking + +The library is completely safe to use in multithreaded +applications; however, there are some very basic and +intuitive locking rules you have to bear in mind. + +Most is fully threadsafe so you don't need to worry +about concurrency; in fact, a lot of effort has been +put in making paralell operation safe and fast. + +You need to care only when opening, closing and +checking for integrity. In practise, that means that +you shouldn't call jopen(), jclose() in paralell with +the same jfs structure, or in the middle of an I/O +operation, just like you do when using the normal UNIX +calls. In the case of jfsck(), you shouldn't invoke it +for the same file more than once at the time; while it +will cope with that situation, it's not recommended. + +All other operations (commiting a transaction, +rollbacking it, adding operations, etc.) and all the +wrappers are safe and don't require any special considerations. + +5.5 Lingering transactions If you need to increase performance, you can use lingering transactions. In this mode, transactions take unchanged: --- cur/doc/libjio.3~doc_updates 2004-08-30 00:51:59.226320296 -0300 +++ cur-root/doc/libjio.3 2004-08-30 01:30:31.226824232 -0300 @@ -20,6 +20,8 @@ libjio - A library for Journaled I/O .BI "ssize_t jwritev(struct jfs *" fs ", const struct iovec *" vector ", int " count " ); +.BI "off_t jlseek(struct jfs *" fs ", off_t " offset ", int " whence " );" + .BI "int jtruncate(struct jfs *" fs ", off_t " lenght " ); .BI "int jsync(struct jfs *" fs " ); @@ -38,7 +40,7 @@ libjio - A library for Journaled I/O .BI "int jfsck(const char *" name ", struct jfsck_result *" res " ); -.BI "int jfsck_cleanup(const char *" name" );" +.BI "int jfsck_cleanup(const char *" name" ); .SH STRUCTURES .PP @@ -48,9 +50,7 @@ libjio - A library for Journaled I/O struct jfs { int fd; /* main file descriptor */ char *name; /* and its name */ - int jfd; /* journal's lock file descriptor */ int flags; /* journal mode options used in jopen() */ - pthread_mutex_t lock; /* a soft lock used in some operations */ ... }; .FI @@ -123,9 +123,12 @@ jfsck_cleanup(). The first one, jfsck(), is used to perform journal checking and recovery in case of a crash. It must be performed when nobody else is using the file (like in the case of a filesystem which can't be mounted), and it returns 0 if -success or -1 in case of a failure. If it succeed, a structure jfsck_result -that summarizes the outcome of the operation. There is also a program named -jiofsck which is just a simple human frontend to this function. +success or a positive number indicating the error in case of a failure. If it +succeed, a structure jfsck_result that summarizes the outcome of the +operation. There is also a program named jiofsck which is just a simple human +frontend to this function. The error values can be J_ENOENT (No such file), +J_ENOJOURNAL (No journal associated with the file), and J_ENOMEM (not enough +free memory). The second, jfsck_cleanup(), is intended to be used after jfsck() by programs wanting to remove all the stall transaction files and leave the journal @@ -137,8 +140,9 @@ if it succeeded. .SH UNIX API The UNIX API, as explained before, consists of the functions jread(), -jpread(), jreadv(), jwrite(), jpwrite(), jwritev(), jtruncate(). In most cases -you will only need to use this, because they're simple and familiar. +jpread(), jreadv(), jwrite(), jpwrite(), jwritev(), jtruncate() and jlseek(). +In most cases you will only need to use this, because they're simple and +familiar. They are all exactly like the UNIX equivalent (if you still don't get it, take the initial 'j' out), and behave the same way, with the only exception that unchanged: --- cur/unix.c~lseek_on_incomplete_io 2004-08-30 01:30:42.300140832 -0300 +++ cur-root/unix.c 2004-08-30 01:30:42.303140376 -0300 @@ -31,7 +31,7 @@ ssize_t jread(struct jfs *fs, void *buf, rv = spread(fs->fd, buf, count, pos); plockf(fs->fd, F_UNLOCK, pos, count); - if (rv == count) { + if (rv >= 0) { /* if success, advance the file pointer */ lseek(fs->fd, count, SEEK_CUR); } unchanged: --- /dev/null 2004-04-13 23:59:22.000000000 -0300 +++ cur-root/UPGRADING 2004-08-30 12:21:20.179711144 -0300 @@ -0,0 +1,19 @@ + +Here are the notes for upgrading libjio from one version to another. + +While normally nothing should be done, sometimes things change and some +actions need to be done. Here's the listing for the releases, you should +always check it before upgrading. + +You should always clean all your files before upgrading. While I don't expect +the transaction on-disk format to change, it's a good practise and it doesn't +take much. If it's mandatory, it will be noted. + +If you want to see what motivated the changes, see the changelog or just ask. + + +0.18 -> 0.19 +* Applications need to be recompiled due to a change in the flag numbering. +* A flag number has changed, from J_COMMITED to J_COMMITTED. If you used that + flag, you need to rename it. + unchanged: --- /dev/null 2004-04-13 23:59:22.000000000 -0300 +++ cur-root/bindings/python/libjio.c 2004-08-30 12:24:49.153942232 -0300 @@ -0,0 +1,626 @@ + +/* + * Python bindings for libjio + * Alberto Bertogli (albertogli@telpin.com.ar) + * Aug/2004 + */ + + +#include + +#include +#include +#include +#include +#include + +/* + * This module provides two classes and some functions. + * + * The classes are jfile (created with open()) and jtrans (created with + * jfile.new_trans()). + * + * The first one represents a journaled file where you operate using read(), + * write() and so on; to close it, just call del(). This is similar to the + * UNIX file. + * + * The second one represents a single transaction, which is composed of + * several operations that get added by its add() method. It gets commited + * with commit(), and rolled back with rollback(). + * + * There rest of the module's functions are related to file checking, called + * jfsck() and jfsck_cleanup(), which are just wrappers to the real C + * functions. + */ + +/* + * Type definitions + */ + +/* jfile */ +typedef struct { + PyObject_HEAD + struct jfs *fs; +} jfileobject; +static PyTypeObject JFileType; + +/* jtrans */ +typedef struct { + PyObject_HEAD + struct jtrans *ts; + jfileobject *jfile; +} jtransobject; +static PyTypeObject JTransType; + + +/* + * The jfile object + */ + +/* delete */ +static void jf_dealloc(jfileobject *fp) +{ + if (fp->fs) { + jclose(fp->fs); + free(fp->fs); + } + PyObject_Del(fp); +} + +/* read */ +PyDoc_STRVAR(jf_read__doc, +"read(size)\n\ +\n\ +Read at most size bytes from the file, returns the string with\n\ +the contents.\n\ +It's a wrapper to jread().\n"); + +static PyObject *jf_read(jfileobject *fp, PyObject *args) +{ + long rv; + long len; + unsigned char *buf; + PyObject *r; + + if (!PyArg_ParseTuple(args, "i:read", &len)) + return NULL; + + buf = malloc(len); + if (buf == NULL) + return PyErr_NoMemory(); + + Py_BEGIN_ALLOW_THREADS + rv = jread(fp->fs, buf, len); + Py_END_ALLOW_THREADS + + if (rv < 0) { + r = PyErr_SetFromErrno(PyExc_IOError); + } else { + r = PyString_FromStringAndSize(buf, rv); + } + + free(buf); + return r; +} + +/* pread */ +PyDoc_STRVAR(jf_pread__doc, +"pread(size, offset)\n\ +\n\ +Read size bytes from the file at the given offset, return a string with the\n\ +contents.\n\ +It's a wrapper to jpread().\n"); + +static PyObject *jf_pread(jfileobject *fp, PyObject *args) +{ + long rv; + long len; + long offset; + unsigned char *buf; + PyObject *r; + + if (!PyArg_ParseTuple(args, "il:pread", &len, &offset)) + return NULL; + + buf = malloc(len); + if (buf == NULL) + return PyErr_NoMemory(); + + Py_BEGIN_ALLOW_THREADS + rv = jpread(fp->fs, buf, len, offset); + Py_END_ALLOW_THREADS + + if (rv < 0) { + r = PyErr_SetFromErrno(PyExc_IOError); + } else { + r = PyString_FromStringAndSize(buf, rv); + } + + free(buf); + return r; +} + +/* write */ +PyDoc_STRVAR(jf_write__doc, +"write(buf)\n\ +\n\ +Write the contents of the given buffer (a string) to the file, returns the\n\ +number of bytes written.\n\ +It's a wrapper to jwrite().\n"); + +static PyObject *jf_write(jfileobject *fp, PyObject *args) +{ + long rv; + unsigned char *buf; + long len; + + if (!PyArg_ParseTuple(args, "s#:write", &buf, &len)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + rv = jwrite(fp->fs, buf, len); + Py_END_ALLOW_THREADS + + return PyLong_FromLong(rv); +} + +/* pwrite */ +PyDoc_STRVAR(jf_pwrite__doc, +"pwrite(buf, offset)\n\ +\n\ +Write the contents of the given buffer (a string) to the file at the given\n\ +offset, returns the number of bytes written.\n\ +It's a wrapper to jpwrite().\n"); + +static PyObject *jf_pwrite(jfileobject *fp, PyObject *args) +{ + long rv; + unsigned char *buf; + long offset; + long len; + + if (!PyArg_ParseTuple(args, "s#l:pwrite", &buf, &len, &offset)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + rv = jpwrite(fp->fs, buf, len, offset); + Py_END_ALLOW_THREADS + + return PyLong_FromLong(rv); +} + +/* truncate */ +PyDoc_STRVAR(jf_truncate__doc, +"truncate(lenght)\n\ +\n\ +Truncate the file to the given size.\n\ +It's a wrapper to jtruncate().\n"); + +static PyObject *jf_truncate(jfileobject *fp, PyObject *args) +{ + int rv; + long lenght; + + if (!PyArg_ParseTuple(args, "l:truncate", &lenght)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + rv = jtruncate(fp->fs, lenght); + Py_END_ALLOW_THREADS + + if (rv != 0) + return PyErr_SetFromErrno(PyExc_IOError); + + return PyInt_FromLong(rv); +} + +/* lseek */ +PyDoc_STRVAR(jf_lseek__doc, +"lseek(offset, whence)\n\ +\n\ +Reposition the file pointer to the given offset, according to the directive\n\ +whence as follows:\n\ +SEEK_SET The offset is set relative to the beginning of the file.\n\ +SEEK_CUR The offset is set relative to the current position.\n\ +SEEK_END The offset is set relative to the end of the file.\n\ +\n\ +These constants are defined in the module. See lseek's manpage for more\n\ +information.\n\ +It's a wrapper to jlseek().\n"); + +static PyObject *jf_lseek(jfileobject *fp, PyObject *args) +{ + long rv; + int whence; + long offset; + + if (!PyArg_ParseTuple(args, "li:lseek", &offset, &whence)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + rv = jlseek(fp->fs, offset, whence); + Py_END_ALLOW_THREADS + + if (rv == -1) + return PyErr_SetFromErrno(PyExc_IOError); + + return PyLong_FromLong(rv); +} + +/* jsync */ +PyDoc_STRVAR(jf_jsync__doc, +"jsync()\n\ +\n\ +Used with lingering transactions, see the library documentation for more\n\ +detailed information.\n\ +It's a wrapper to jsync().\n"); + +static PyObject *jf_jsync(jfileobject *fp, PyObject *args) +{ + long rv; + + if (!PyArg_ParseTuple(args, ":jsync")) + return NULL; + + Py_BEGIN_ALLOW_THREADS + rv = jsync(fp->fs); + Py_END_ALLOW_THREADS + + return PyLong_FromLong(rv); +} + +/* new_trans */ +PyDoc_STRVAR(jf_new_trans__doc, +"new_trans()\n\ +\n\ +Returns an object representing a new empty transaction.\n\ +It's a wrapper to jtrans_init().\n"); + +static PyObject *jf_new_trans(jfileobject *fp, PyObject *args) +{ + jtransobject *tp; + + if (!PyArg_ParseTuple(args, ":new_trans")) + return NULL; + + tp = PyObject_New(jtransobject, &JTransType); + if (tp == NULL) + return NULL; + + tp->ts = malloc(sizeof(struct jtrans)); + if(tp->ts == NULL) { + return PyErr_NoMemory(); + } + + /* increment the reference count, it's decremented on deletion */ + tp->jfile = fp; + Py_INCREF(fp); + + jtrans_init(fp->fs, tp->ts); + + return (PyObject *) tp; +} + + +/* method table */ +static PyMethodDef jfile_methods[] = { + { "read", (PyCFunction)jf_read, METH_VARARGS, jf_read__doc }, + { "pread", (PyCFunction)jf_pread, METH_VARARGS, jf_pread__doc }, + { "write", (PyCFunction)jf_write, METH_VARARGS, jf_write__doc }, + { "pwrite", (PyCFunction)jf_pwrite, METH_VARARGS, jf_pwrite__doc }, + { "truncate", (PyCFunction)jf_truncate, METH_VARARGS, jf_truncate__doc }, + { "lseek", (PyCFunction)jf_lseek, METH_VARARGS, jf_lseek__doc }, + { "jsync", (PyCFunction)jf_jsync, METH_VARARGS, jf_jsync__doc }, + { "new_trans", (PyCFunction)jf_new_trans, METH_VARARGS, jf_new_trans__doc }, + { NULL } +}; + +static PyObject *jf_getattr(jfileobject *fp, char *name) +{ + return Py_FindMethod(jfile_methods, (PyObject *)fp, name); +} + +static PyTypeObject JFileType = { + PyObject_HEAD_INIT(NULL) + 0, + "libjio.jfile", + sizeof(jfileobject), + 0, + (destructor)jf_dealloc, + 0, + (getattrfunc)jf_getattr, +}; + + +/* + * The jtrans object + */ + +/* delete */ +static void jt_dealloc(jtransobject *tp) +{ + if (tp->ts != NULL) { + jtrans_free(tp->ts); + free(tp->ts); + } + Py_DECREF(tp->jfile); + PyObject_Del(tp); +} + +/* add */ +PyDoc_STRVAR(jt_add__doc, +"add(buf, offset)\n\ +\n\ +Add an operation to write the given buffer at the given offset to the\n\ +transaction.\n\ +It's a wrapper to jtrans_add().\n"); + +static PyObject *jt_add(jtransobject *tp, PyObject *args) +{ + long rv; + long len; + long offset; + unsigned char *buf; + + if (!PyArg_ParseTuple(args, "s#l:add", &buf, &len, &offset)) + return NULL; + + rv = jtrans_add(tp->ts, buf, len, offset); + + return PyLong_FromLong(rv); +} + +/* commit */ +PyDoc_STRVAR(jt_commit__doc, +"commit()\n\ +\n\ +Commits a transaction.\n\ +It's a wrapper to jtrans_commit().\n"); + +static PyObject *jt_commit(jtransobject *tp, PyObject *args) +{ + long rv; + + if (!PyArg_ParseTuple(args, ":commit")) + return NULL; + + Py_BEGIN_ALLOW_THREADS + rv = jtrans_commit(tp->ts); + Py_END_ALLOW_THREADS + + if (rv < 0) + return PyErr_SetFromErrno(PyExc_IOError); + + return PyLong_FromLong(rv); +} + +/* rollback */ +PyDoc_STRVAR(jt_rollback__doc, +"rollback()\n\ +\n\ +Rollbacks a transaction.\n\ +It's a wrapper to jtrans_rollback().\n"); + +static PyObject *jt_rollback(jtransobject *tp, PyObject *args) +{ + long rv; + + if (!PyArg_ParseTuple(args, ":rollback")) + return NULL; + + Py_BEGIN_ALLOW_THREADS + rv = jtrans_rollback(tp->ts); + Py_END_ALLOW_THREADS + + if (rv < 0) + return PyErr_SetFromErrno(PyExc_IOError); + + return PyLong_FromLong(rv); +} + +/* method table */ +static PyMethodDef jtrans_methods[] = { + { "add", (PyCFunction)jt_add, METH_VARARGS, jt_add__doc }, + { "commit", (PyCFunction)jt_commit, METH_VARARGS, jt_commit__doc }, + { "rollback", (PyCFunction)jt_rollback, METH_VARARGS, jt_rollback__doc }, + { NULL } +}; + +static PyObject *jt_getattr(jtransobject *tp, char *name) +{ + return Py_FindMethod(jtrans_methods, (PyObject *)tp, name); +} + +static PyTypeObject JTransType = { + PyObject_HEAD_INIT(NULL) + 0, + "libjio.jtrans", + sizeof(jtransobject), + 0, + (destructor)jt_dealloc, + 0, + (getattrfunc)jt_getattr, +}; + + + +/* + * The module + */ + +/* open */ +PyDoc_STRVAR(jf_open__doc, +"open(name[, flags[, mode[, jflags]]])\n\ +\n\ +Opens a file, returns a file object.\n\ +The arguments flags, mode and jflags are the same as jopen(); the constants\n\ +needed are defined in the module.\n\ +It's a wrapper to jopen().\n"); + +static PyObject *jf_open(PyObject *self, PyObject *args) +{ + int rv; + char *file; + int flags, mode, jflags; + jfileobject *fp; + + flags = O_RDWR; + mode = 0666; + jflags = 0; + + if (!PyArg_ParseTuple(args, "s|iii:open", &file, &flags, &mode, + &jflags)) + return NULL; + + fp = PyObject_New(jfileobject, &JFileType); + if (fp == NULL) + return NULL; + + fp->fs = malloc(sizeof(struct jfs)); + if (fp->fs == NULL) { + return PyErr_NoMemory(); + } + + rv = jopen(fp->fs, file, flags, mode, jflags); + if (rv < 0) { + free(fp->fs); + return PyErr_SetFromErrno(PyExc_IOError); + } + + if (PyErr_Occurred()) { + free(fp->fs); + return NULL; + } + + return (PyObject *) fp; +} + +/* jfsck */ +PyDoc_STRVAR(jf_jfsck__doc, +"jfsck(name)\n\ +\n\ +Checks the integrity of the file with the given name; returns a dictionary\n\ +with all the different values of the check (equivalent to the 'struct\n\ +jfsck_result'), or None if there was nothing to check.\n\ +It's a wrapper to jfsck().\n"); + +static PyObject *jf_jfsck(PyObject *self, PyObject *args) +{ + int rv; + char *name; + struct jfsck_result res; + PyObject *dict; + + if (!PyArg_ParseTuple(args, "s:jfsck", &name)) + return NULL; + + dict = PyDict_New(); + if (dict == NULL) + return PyErr_NoMemory(); + + Py_BEGIN_ALLOW_THREADS + rv = jfsck(name, &res); + Py_END_ALLOW_THREADS + + if (rv == J_ENOMEM) { + return PyErr_NoMemory(); + } else if (rv != 0) { + return Py_None; + } + + PyDict_SetItemString(dict, "total", PyLong_FromLong(res.total)); + PyDict_SetItemString(dict, "invalid", PyLong_FromLong(res.invalid)); + PyDict_SetItemString(dict, "in_progress", PyLong_FromLong(res.in_progress)); + PyDict_SetItemString(dict, "broken", PyLong_FromLong(res.broken)); + PyDict_SetItemString(dict, "corrupt", PyLong_FromLong(res.corrupt)); + PyDict_SetItemString(dict, "apply_error", PyLong_FromLong(res.apply_error)); + PyDict_SetItemString(dict, "reapplied", PyLong_FromLong(res.reapplied)); + + return dict; +} + +/* jfsck_cleanup */ +PyDoc_STRVAR(jf_jfsck_cleanup__doc, +"jfsck_cleanup()\n\ +\n\ +Clean the journal directory and leave it ready to use.\n\ +It's a wrapper to jfsck_cleanup().\n"); + +static PyObject *jf_jfsck_cleanup(PyObject *self, PyObject *args) +{ + long rv; + char *name; + + if (!PyArg_ParseTuple(args, "s:jfsck_cleanup", &name)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + rv = jfsck_cleanup(name); + Py_END_ALLOW_THREADS + + return PyInt_FromLong(rv); +} + +/* function table */ +static PyMethodDef libjio_functions[] = { + { "open", (PyCFunction)jf_open, METH_VARARGS, jf_open__doc }, + { "jfsck", (PyCFunction)jf_jfsck, METH_VARARGS, jf_jfsck__doc }, + { "jfsck_cleanup", (PyCFunction)jf_jfsck_cleanup, METH_VARARGS, jf_jfsck_cleanup__doc }, + { NULL, }, +}; + +/* module initialization */ +PyDoc_STRVAR(libjio__doc, +"libjio is a library to do transactional, journaled I/O\n\ +You can find it at http://users.auriga.wearlab.de/~alb/libjio/\n\ +\n\ +Use the open() method to create a file object, and then operate on it.\n\ +Please read the documentation for more information.\n"); + +PyMODINIT_FUNC initlibjio(void) +{ + PyObject* m; + + JFileType.ob_type = &PyType_Type; + JTransType.ob_type = &PyType_Type; + + m = Py_InitModule3("libjio", libjio_functions, libjio__doc); + + Py_INCREF(&JFileType); + PyModule_AddObject(m, "jfile", (PyObject *) &JFileType); + + Py_INCREF(&JTransType); + PyModule_AddObject(m, "jtrans", (PyObject *) &JTransType); + + /* libjio's constants */ + PyModule_AddIntConstant(m, "J_NOLOCK", J_NOLOCK); + PyModule_AddIntConstant(m, "J_NOROLLBACK", J_NOROLLBACK); + PyModule_AddIntConstant(m, "J_LINGER", J_LINGER); + PyModule_AddIntConstant(m, "J_COMMITTED", J_COMMITTED); + PyModule_AddIntConstant(m, "J_ROLLBACKED", J_ROLLBACKED); + PyModule_AddIntConstant(m, "J_ROLLBACKING", J_ROLLBACKING); + PyModule_AddIntConstant(m, "J_ESUCCESS", J_ESUCCESS); + PyModule_AddIntConstant(m, "J_ENOENT", J_ENOENT); + PyModule_AddIntConstant(m, "J_ENOJOURNAL", J_ENOJOURNAL); + PyModule_AddIntConstant(m, "J_ENOMEM", J_ENOMEM); + + /* open constants (at least the POSIX ones) */ + PyModule_AddIntConstant(m, "O_RDONLY", O_RDONLY); + PyModule_AddIntConstant(m, "O_WRONLY", O_WRONLY); + PyModule_AddIntConstant(m, "O_RDWR", O_RDWR); + PyModule_AddIntConstant(m, "O_CREAT", O_CREAT); + PyModule_AddIntConstant(m, "O_EXCL", O_EXCL); + PyModule_AddIntConstant(m, "O_TRUNC", O_TRUNC); + PyModule_AddIntConstant(m, "O_APPEND", O_APPEND); + PyModule_AddIntConstant(m, "O_NONBLOCK", O_NONBLOCK); + PyModule_AddIntConstant(m, "O_NDELAY", O_NDELAY); + PyModule_AddIntConstant(m, "O_SYNC", O_SYNC); + PyModule_AddIntConstant(m, "O_ASYNC", O_ASYNC); + PyModule_AddIntConstant(m, "O_LARGEFILE", O_LARGEFILE); + + /* lseek constants */ + PyModule_AddIntConstant(m, "SEEK_SET", SEEK_SET); + PyModule_AddIntConstant(m, "SEEK_CUR", SEEK_CUR); + PyModule_AddIntConstant(m, "SEEK_END", SEEK_END); +} + unchanged: --- /dev/null 2004-04-13 23:59:22.000000000 -0300 +++ cur-root/bindings/python/setup.py 2004-08-30 12:24:49.156941776 -0300 @@ -0,0 +1,16 @@ + +from distutils.core import setup, Extension + +libjio = Extension("libjio", + libraries = ['jio'], + sources = ['libjio.c']) + +setup( + name = 'libjio', + description = "A library for journaled I/O", + author="Alberto Bertogli", + author_email="albertogli@telpin.com.ar", + url="http://users.auriga.wearlab.de/~alb/libjio", + ext_modules = [libjio] +) + unchanged: --- cur/Makefile~python_bindings 2004-08-30 12:25:11.000000000 -0300 +++ cur-root/Makefile 2004-08-30 12:29:29.145377064 -0300 @@ -33,6 +33,12 @@ install: all @echo "Please run ldconfig to update your library cache" @echo +python: all + cd bindings/python && python setup.py build + +python_install: python + cd bindings/python && python setup.py install + .c.o: $(CC) $(CFLAGS) $(INCLUDES) -c $< -o $@ @@ -40,6 +46,7 @@ install: all clean: rm -f $(OBJS) libjio.a libjio.so jiofsck.o jiofsck rm -f *.bb *.bbg *.da *.gcov gmon.out + rm -rf bindings/python/build/ .PHONY: default all shared static install clean unchanged: --- cur/README~mailing_list 2004-08-30 12:31:18.138807544 -0300 +++ cur-root/README 2004-08-30 12:32:32.113561672 -0300 @@ -28,8 +28,8 @@ To see how to install it, please read th It is licensed under the Open Software License version 2.0. -Comments and patches are always welcome; please send them to my email address, -albertogli@telpin.com.ar. +Comments and patches are always welcome; please send them to the mailing list, +libjio-devel@auriga.wearlab.de. Thanks, Alberto unchanged: --- cur/INSTALL~mailing_list 2004-08-30 12:33:04.466643256 -0300 +++ cur-root/INSTALL 2004-08-30 12:33:14.338142560 -0300 @@ -35,6 +35,6 @@ After installing, you're ready to use th at the manpage with "man libjio". -If you have any question, suggestion or comment, please send it to -albertogli@telpin.com.ar. +If you have any question, suggestion or comment, please send it to the mailing +list, at libjio-devel@auriga.wearlab.de. only in patch2: unchanged: --- cur/Make.conf~version-0.19 2004-08-30 12:36:28.110684656 -0300 +++ cur-root/Make.conf 2004-08-30 12:36:34.283746208 -0300 @@ -1,5 +1,5 @@ -VERSION="0.18" +VERSION="0.19" CC = gcc CFLAGS += -Wall -O6 \