.gitignore | 7 + INSTALL | 50 ++++---- LICENSE | 30 ++++ Makefile | 111 +++----------- README | 2 +- UPGRADING | 46 +++--- bindings/preload/Makefile | 39 +++++ bindings/{python => python2}/libjio.c | 114 ++++++++------- bindings/{python => python2}/setup.py | 0 bindings/{python => python3}/libjio.c | 190 ++++++++++++------------- bindings/{python => python3}/setup.py | 0 doc/LICENSE | 188 ------------------------ Makefile => libjio/Makefile | 51 +++----- ansi.c => libjio/ansi.c | 0 check.c => libjio/check.c | 38 +++-- checksum.c => libjio/checksum.c | 0 common.c => libjio/common.c | 0 common.h => libjio/common.h | 1 + libjio/compat.h | 19 +++ libjio/fiu-local.h | 37 +++++ jiofsck.c => libjio/jiofsck.c | 0 {doc => libjio}/libjio.3 | 90 +++++++----- libjio.h => libjio/libjio.h | 10 +-- libjio.skel.pc => libjio/libjio.skel.pc | 0 trans.c => libjio/trans.c | 33 ++++- unix.c => libjio/unix.c | 0 samples/Makefile | 22 +++ samples/build | 5 - samples/clean | 2 - samples/full.c | 6 +- samples/jio1.c | 33 ++--- samples/jio2.c | 52 +++---- samples/jio3.c | 16 +-- tests/README | 10 -- tests/behaviour/README | 15 ++ tests/behaviour/runtests | 19 +++ tests/behaviour/t_corruption.py | 89 ++++++++++++ tests/behaviour/t_fi.py | 226 +++++++++++++++++++++++++++++ tests/behaviour/t_normal.py | 173 ++++++++++++++++++++++ tests/behaviour/tf.py | 236 +++++++++++++++++++++++++++++++ tests/performance/Makefile | 18 ++-- tests/performance/parallel.c | 122 ---------------- tests/performance/performance.c | 141 ++++++++++++++++++ tests/performance/streaming.c | 82 ----------- tests/recovery/.1.jio/2 | Bin 49 -> 0 bytes tests/recovery/.1.jio/3 | Bin 51 -> 0 bytes tests/recovery/.1.jio/4 | 1 - tests/recovery/.1.jio/5 | 1 - tests/recovery/.1.jio/6 | 1 - tests/recovery/.1.jio/7 | Bin 35 -> 0 bytes tests/recovery/.1.jio/desc | 9 -- tests/recovery/.1.jio/lock | Bin 4 -> 0 bytes tests/recovery/1 | 1 - tests/recovery/mklock | 25 ---- tests/recovery/mktrans | 48 ------- tests/stress/jiostress | 203 ++++++++++++++++++++++++++ 56 files changed, 1669 insertions(+), 943 deletions(-) diff --git a/.gitignore b/.gitignore index 7857c16..2f09c27 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,10 @@ *.so jiofsck libjio.pc +samples/full +samples/jio1 +samples/jio2 +samples/jio3 +tests/performance/performance +*.pyc +*.pyo diff --git a/INSTALL b/INSTALL index 69b49bc..ff96287 100644 --- a/INSTALL +++ b/INSTALL @@ -1,38 +1,38 @@ -To install the library and the checker (called jiofsck), run as root +Building and installing +----------------------- -# make install - - -It will copy the library to /usr/local/lib, manpages to /usr/local/man, -binaries to /usr/local/bin, and headers to /usr/local/include. - -If you want to change the prefix directory (/usr/local by default), for -instance, to /home/myself/usr; just run: - -# make PREFIX=/home/myself/usr install - - -This will create inside PREFIX the directories "bin", "lib", "include" and -"man" if necesary, and put the required files in there. +To build and install the library and the checker (called jiofsck), run "make" +and then "make install" (usually as root). That installs everything using +/usr/local as the base directory (so the library gets installed in +/usr/local/lib, the manpage at /usr/local/man, and so on). +To use a different base directory, for example /home/myself, run +"make PREFIX=/home/myself install". After installing, you need to run "ldconfig" in order to update your dynamic library cache. -So, if you want to install to /usr (like most distro bundled software do), -this is a brief command line summary: +If the default "make" is not GNU make (like in BSD systems), use "gmake" +instead. -make PREFIX=/usr install -ldconfig +Special builds +-------------- -There are other small options that might be useful (like compiling with -debugging information), you can check the Make.conf file for more information. + - To build with debugging information: "make DEBUG=1". + - To build with profiling support: "make PROFILE=1". + - To build with fault injection support, if you have libfiu: "make FI=1". -If you use any BSD, or if the default "make" is not GNU make, use "gmake" -instead. -After installing, you're ready to use the library; you can see how by looking -at the manpage with "man libjio". +Python bindings +--------------- + +The library comes with bindings for Python 2 and Python 3. In order to build +them, you should have libjio already installed. + + - To build the Python 2 bindings, run "make python2". To install them, run + "make python2_install". + - To build the Python 3 bindings, run "make python3". To install them, run + "make python3_install". diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..edb80f0 --- /dev/null +++ b/LICENSE @@ -0,0 +1,30 @@ + +I don't like licenses, because I don't like having to worry about all this +legal stuff just for a simple piece of software I don't really mind anyone +using. But I also believe that it's important that people share and give back; +so I'm placing this work under the following license. + + +BOLA - Buena Onda License Agreement (v1.0) +------------------------------------------ + +This work is provided 'as-is', without any express or implied warranty. In no +event will the authors be held liable for any damages arising from the use of +this work. + +To all effects and purposes, this work is to be considered Public Domain. + + +However, if you want to be "buena onda", you should: + +1. Not take credit for it, and give proper recognition to the authors. +2. Share your modifications, so everybody benefits from them. +4. Do something nice for the authors. +5. Help someone who needs it: sign up for some volunteer work or help your + neighbour paint the house. +6. Don't waste. Anything, but specially energy that comes from natural + non-renewable resources. Extra points if you discover or invent something + to replace them. +7. Be tolerant. Everything that's good in nature comes from cooperation. + + diff --git a/Makefile b/Makefile index 6bad0ab..3c32641 100644 --- a/Makefile +++ b/Makefile @@ -1,108 +1,45 @@ -VERSION="0.24" - -CFLAGS = -std=c99 -pedantic -Wall -O3 - -MANDATORY_CFLAGS := \ - -D_LARGEFILE_SOURCE=1 -D_LARGEFILE64_SOURCE=1 \ - -D_LFS_LARGEFILE=1 -D_LFS64_LARGEFILE=1 \ - -D_FILE_OFFSET_BITS=64 $(shell getconf LFS_CFLAGS 2>/dev/null) \ - -D_XOPEN_SOURCE=500 - -ALL_CFLAGS += $(CFLAGS) $(MANDATORY_CFLAGS) -fPIC - -ifdef DEBUG -ALL_CFLAGS += -g -endif - -ifdef PROFILE -ALL_CFLAGS += -g -pg -fprofile-arcs -ftest-coverage -endif - - -# prefix for installing the binaries -PREFIX=/usr/local - - -ifneq ($(V), 1) - NICE_CC = @echo " CC $@"; $(CC) - NICE_AR = @echo " AR $@"; $(AR) -else - NICE_CC = $(CC) - NICE_AR = $(AR) -endif - - -# objects to build -OBJS = checksum.o common.o trans.o check.o unix.o ansi.o - -# rules default: all -all: libjio.so libjio.a libjio.pc jiofsck - -libjio.so: $(OBJS) - $(NICE_CC) -shared $(ALL_CFLAGS) $(OBJS) -o libjio.so +all: libjio -libjio.a: $(OBJS) - $(NICE_AR) cr libjio.a $(OBJS) -libjio.pc: libjio.skel.pc - @echo "generating libjio.pc" - @cat libjio.skel.pc | \ - sed 's@++PREFIX++@$(PREFIX)@g' | \ - sed 's@++CFLAGS++@$(MANDATORY_CFLAGS)@g' \ - > libjio.pc +libjio: + $(MAKE) -C libjio/ -jiofsck: jiofsck.o libjio.a - $(NICE_CC) $(ALL_CFLAGS) jiofsck.o libjio.a -lpthread -o jiofsck +install: + $(MAKE) -C libjio/ install -install: all - install -d $(PREFIX)/lib - install -m 0755 libjio.so $(PREFIX)/lib - install -m 0644 libjio.a $(PREFIX)/lib - install -d $(PREFIX)/include - install -m 0644 libjio.h $(PREFIX)/include - install -d $(PREFIX)/lib/pkgconfig - install -m 644 libjio.pc $(PREFIX)/lib/pkgconfig - install -d $(PREFIX)/bin - install -m 0775 jiofsck $(PREFIX)/bin - install -d $(PREFIX)/man/man3 - install -m 0644 doc/libjio.3 $(PREFIX)/man/man3/ - @echo - @echo "Please run ldconfig to update your library cache" - @echo -.c.o: - $(NICE_CC) $(ALL_CFLAGS) $(INCLUDES) -c $< -o $@ +python2: + cd bindings/python2 && python setup.py build +python2_install: python2 + cd bindings/python2 && python setup.py install -python: all - cd bindings/python && python setup.py build +python3: + cd bindings/python3 && python3 setup.py build -python_install: python - cd bindings/python && python setup.py install +python3_install: python3 + cd bindings/python3 && python3 setup.py install -preload: all - install -d bindings/preload/build/ - $(NICE_CC) $(INCLUDES) -Wall -O3 -shared -fPIC \ - -D_XOPEN_SOURCE=500 \ - -ldl -lpthread -L. -ljio -I. \ - bindings/preload/libjio_preload.c \ - -o bindings/preload/build/libjio_preload.so +preload: + $(MAKE) -C bindings/preload/ preload_install: preload - install -d $(PREFIX)/lib - install -m 0755 bindings/preload/build/libjio_preload.so $(PREFIX)/lib + $(MAKE) -C bindings/preload/ install clean: - rm -f $(OBJS) libjio.a libjio.so libjio.pc jiofsck.o jiofsck - rm -f *.bb *.bbg *.da *.gcov *.gcno *.gcda gmon.out - rm -rf bindings/python/build/ - rm -rf bindings/preload/build/ + $(MAKE) -C libjio/ clean + $(MAKE) -C bindings/preload clean + rm -rf bindings/python2/build/ + rm -rf bindings/python3/build/ -.PHONY: default all install python python_install preload preload_install clean +.PHONY: default all libjio install \ + python2 python2_install python3 python3_install \ + preload preload_install \ + clean diff --git a/README b/README index 1adc539..0413faf 100644 --- a/README +++ b/README @@ -22,7 +22,7 @@ to the design and inner workings, and the manpage; all in the doc/ directory. To see how to install it, please read the INSTALL file. -It is licensed under the Open Software License version 3.0. +It is in the public domain, see the LICENSE file for more details. Comments and patches are always welcome; please send them to my email address, albertito@blitiri.com.ar. diff --git a/UPGRADING b/UPGRADING index 72b9c9e..e5a4201 100644 --- a/UPGRADING +++ b/UPGRADING @@ -1,32 +1,34 @@ -Here are the notes for upgrading libjio from one version to another. - -While normally nothing should be done, sometimes things change and some -actions need to be done. Here's the listing for the releases, you should -always check it before upgrading. +Here you can find a summary of the changes in the library that could require +an effort from its users, like the ones affecting the API, ABI or semantics. You should always clean all your files before upgrading. While I don't expect the transaction on-disk format to change, it's a good practise and it doesn't -take much. If it's mandatory, it will be noted. +take much effort. When it's mandatory, it will be noted. + -If you want to see what motivated the changes, see the changelog or just ask. +-> 0.25 + - It is no longer necessary to pass O_SYNC to jopen() if lingering + transactions are not in use. + - libjio.h can no longer be included from C++ source without surrounding it + by an 'extern "C"'. This obviously should only affect C++ applications. -0.22 -> 0.24 -* The return values of jfsck() have changed, so applications using it need to - be recompiled. -* Python bindings' jfsck(), jfsck_cleanup(), jsync(), jwrite() and jpwrite() - now raise an IOError on failures. +-> 0.24 + - The return values of jfsck() have changed, so applications using it need + to be recompiled. + - Python bindings' jfsck(), jfsck_cleanup(), jsync(), jwrite() and jpwrite() + now raise an IOError on failures. -0.21 -> 0.22 -* Applications need to be recompiled due to a change in the jfs structure. +-> 0.22 + - Applications need to be recompiled due to a change in the jfs structure. -0.19 -> 0.20 -* Applications need to be recompiled due to a change in the jfs structure. -* When you link your applications with libjio, you need to make sure you - compile using large file support. +-> 0.20 + - Applications need to be recompiled due to a change in the jfs structure. + - When you link your applications with libjio, you need to make sure you + compile using large file support. -0.18 -> 0.19 -* Applications need to be recompiled due to a change in the flag numbering. -* A flag number has changed, from J_COMMITED to J_COMMITTED. If you used that - flag, you need to rename it. +-> 0.19 + - Applications need to be recompiled due to a change in the flag numbering. + - A flag number has changed, from J_COMMITED to J_COMMITTED. If you used + that flag, you need to rename it. diff --git a/bindings/preload/Makefile b/bindings/preload/Makefile new file mode 100644 index 0000000..bfe34b7 --- /dev/null +++ b/bindings/preload/Makefile @@ -0,0 +1,39 @@ + +CFLAGS = -Wall -O3 + +MANDATORY_CFLAGS := \ + -D_LARGEFILE_SOURCE=1 $(shell getconf LFS_CFLAGS 2>/dev/null) \ + -D_XOPEN_SOURCE=500 + +ALL_CFLAGS = $(CFLAGS) $(MANDATORY_CFLAGS) -fPIC + + +PREFIX = /usr/local + +ifneq ($(V), 1) + NICE_CC = @echo " CC $@"; $(CC) +else + NICE_CC = $(CC) +endif + + +default: all + +all: preload + +preload: libjio_preload.o + $(NICE_CC) $(ALL_CFLAGS) -ljio -shared -fPIC \ + libjio_preload.o -o libjio_preload.so + +.c.o: + $(NICE_CC) $(ALL_CFLAGS) -c $< -o $@ + +install: preload + install -d $(PREFIX)/lib + install -m 0755 libjio_preload.so $(PREFIX)/lib + +clean: + rm -f libjio_preload.o libjio_preload.so + +.PHONY: default all install clean + diff --git a/bindings/python/libjio.c b/bindings/python2/libjio.c similarity index 82% copy from bindings/python/libjio.c copy to bindings/python2/libjio.c index eb90e5a..a29f8f1 100644 --- a/bindings/python/libjio.c +++ b/bindings/python2/libjio.c @@ -2,7 +2,6 @@ /* * Python bindings for libjio * Alberto Bertogli (albertito@blitiri.com.ar) - * Aug/2004 */ @@ -25,7 +24,7 @@ * UNIX file. * * The second one represents a single transaction, which is composed of - * several operations that get added by its add() method. It gets commited + * several operations that get added by its add() method. It gets committed * with commit(), and rolled back with rollback(). * * There rest of the module's functions are related to file checking, called @@ -41,16 +40,18 @@ typedef struct { PyObject_HEAD struct jfs *fs; -} jfileobject; -static PyTypeObject JFileType; +} jfile_object; + +static PyTypeObject jfile_type; /* jtrans */ typedef struct { PyObject_HEAD struct jtrans *ts; - jfileobject *jfile; -} jtransobject; -static PyTypeObject JTransType; + jfile_object *jfile; +} jtrans_object; + +static PyTypeObject jtrans_type; /* @@ -58,7 +59,7 @@ static PyTypeObject JTransType; */ /* delete */ -static void jf_dealloc(jfileobject *fp) +static void jf_dealloc(jfile_object *fp) { if (fp->fs) { jclose(fp->fs); @@ -73,7 +74,7 @@ PyDoc_STRVAR(jf_fileno__doc, \n\ Return the file descriptor number for the file.\n"); -static PyObject *jf_fileno(jfileobject *fp, PyObject *args) +static PyObject *jf_fileno(jfile_object *fp, PyObject *args) { if (!PyArg_ParseTuple(args, ":fileno")) return NULL; @@ -89,7 +90,7 @@ Read at most size bytes from the file, returns the string with\n\ the contents.\n\ It's a wrapper to jread().\n"); -static PyObject *jf_read(jfileobject *fp, PyObject *args) +static PyObject *jf_read(jfile_object *fp, PyObject *args) { long rv; long len; @@ -125,7 +126,7 @@ Read size bytes from the file at the given offset, return a string with the\n\ contents.\n\ It's a wrapper to jpread().\n"); -static PyObject *jf_pread(jfileobject *fp, PyObject *args) +static PyObject *jf_pread(jfile_object *fp, PyObject *args) { long rv; long len; @@ -162,7 +163,7 @@ Write the contents of the given buffer (a string) to the file, returns the\n\ number of bytes written.\n\ It's a wrapper to jwrite().\n"); -static PyObject *jf_write(jfileobject *fp, PyObject *args) +static PyObject *jf_write(jfile_object *fp, PyObject *args) { long rv; unsigned char *buf; @@ -189,7 +190,7 @@ Write the contents of the given buffer (a string) to the file at the given\n\ offset, returns the number of bytes written.\n\ It's a wrapper to jpwrite().\n"); -static PyObject *jf_pwrite(jfileobject *fp, PyObject *args) +static PyObject *jf_pwrite(jfile_object *fp, PyObject *args) { long rv; unsigned char *buf; @@ -216,7 +217,7 @@ PyDoc_STRVAR(jf_truncate__doc, Truncate the file to the given size.\n\ It's a wrapper to jtruncate().\n"); -static PyObject *jf_truncate(jfileobject *fp, PyObject *args) +static PyObject *jf_truncate(jfile_object *fp, PyObject *args) { int rv; long long lenght; @@ -248,7 +249,7 @@ These constants are defined in the module. See lseek's manpage for more\n\ information.\n\ It's a wrapper to jlseek().\n"); -static PyObject *jf_lseek(jfileobject *fp, PyObject *args) +static PyObject *jf_lseek(jfile_object *fp, PyObject *args) { long long rv; int whence; @@ -275,7 +276,7 @@ Used with lingering transactions, see the library documentation for more\n\ detailed information.\n\ It's a wrapper to jsync().\n"); -static PyObject *jf_jsync(jfileobject *fp, PyObject *args) +static PyObject *jf_jsync(jfile_object *fp, PyObject *args) { long rv; @@ -300,7 +301,7 @@ Moves the journal directory to the new path; note that there MUST NOT BE\n\ anything else operating on the file.\n\ It's a wrapper to jmove_journal().\n"); -static PyObject *jf_jmove_journal(jfileobject *fp, PyObject *args) +static PyObject *jf_jmove_journal(jfile_object *fp, PyObject *args) { long rv; char *newpath; @@ -325,14 +326,14 @@ PyDoc_STRVAR(jf_new_trans__doc, Returns an object representing a new empty transaction.\n\ It's a wrapper to jtrans_init().\n"); -static PyObject *jf_new_trans(jfileobject *fp, PyObject *args) +static PyObject *jf_new_trans(jfile_object *fp, PyObject *args) { - jtransobject *tp; + jtrans_object *tp; if (!PyArg_ParseTuple(args, ":new_trans")) return NULL; - tp = PyObject_New(jtransobject, &JTransType); + tp = PyObject_New(jtrans_object, &jtrans_type); if (tp == NULL) return NULL; @@ -353,29 +354,32 @@ static PyObject *jf_new_trans(jfileobject *fp, PyObject *args) /* method table */ static PyMethodDef jfile_methods[] = { - { "fileno", (PyCFunction)jf_fileno, METH_VARARGS, jf_fileno__doc }, - { "read", (PyCFunction)jf_read, METH_VARARGS, jf_read__doc }, - { "pread", (PyCFunction)jf_pread, METH_VARARGS, jf_pread__doc }, - { "write", (PyCFunction)jf_write, METH_VARARGS, jf_write__doc }, - { "pwrite", (PyCFunction)jf_pwrite, METH_VARARGS, jf_pwrite__doc }, - { "truncate", (PyCFunction)jf_truncate, METH_VARARGS, jf_truncate__doc }, - { "lseek", (PyCFunction)jf_lseek, METH_VARARGS, jf_lseek__doc }, - { "jsync", (PyCFunction)jf_jsync, METH_VARARGS, jf_jsync__doc }, - { "jmove_journal", (PyCFunction)jf_jmove_journal, METH_VARARGS, jf_jmove_journal__doc }, - { "new_trans", (PyCFunction)jf_new_trans, METH_VARARGS, jf_new_trans__doc }, + { "fileno", (PyCFunction) jf_fileno, METH_VARARGS, jf_fileno__doc }, + { "read", (PyCFunction) jf_read, METH_VARARGS, jf_read__doc }, + { "pread", (PyCFunction) jf_pread, METH_VARARGS, jf_pread__doc }, + { "write", (PyCFunction) jf_write, METH_VARARGS, jf_write__doc }, + { "pwrite", (PyCFunction) jf_pwrite, METH_VARARGS, jf_pwrite__doc }, + { "truncate", (PyCFunction) jf_truncate, METH_VARARGS, + jf_truncate__doc }, + { "lseek", (PyCFunction) jf_lseek, METH_VARARGS, jf_lseek__doc }, + { "jsync", (PyCFunction) jf_jsync, METH_VARARGS, jf_jsync__doc }, + { "jmove_journal", (PyCFunction) jf_jmove_journal, METH_VARARGS, + jf_jmove_journal__doc }, + { "new_trans", (PyCFunction) jf_new_trans, METH_VARARGS, + jf_new_trans__doc }, { NULL } }; -static PyObject *jf_getattr(jfileobject *fp, char *name) +static PyObject *jf_getattr(jfile_object *fp, char *name) { return Py_FindMethod(jfile_methods, (PyObject *)fp, name); } -static PyTypeObject JFileType = { +static PyTypeObject jfile_type = { PyObject_HEAD_INIT(NULL) 0, "libjio.jfile", - sizeof(jfileobject), + sizeof(jfile_object), 0, (destructor)jf_dealloc, 0, @@ -388,7 +392,7 @@ static PyTypeObject JFileType = { */ /* delete */ -static void jt_dealloc(jtransobject *tp) +static void jt_dealloc(jtrans_object *tp) { if (tp->ts != NULL) { jtrans_free(tp->ts); @@ -406,7 +410,7 @@ Add an operation to write the given buffer at the given offset to the\n\ transaction.\n\ It's a wrapper to jtrans_add().\n"); -static PyObject *jt_add(jtransobject *tp, PyObject *args) +static PyObject *jt_add(jtrans_object *tp, PyObject *args) { long rv; int len; @@ -430,7 +434,7 @@ PyDoc_STRVAR(jt_commit__doc, Commits a transaction.\n\ It's a wrapper to jtrans_commit().\n"); -static PyObject *jt_commit(jtransobject *tp, PyObject *args) +static PyObject *jt_commit(jtrans_object *tp, PyObject *args) { long rv; @@ -454,7 +458,7 @@ PyDoc_STRVAR(jt_rollback__doc, Rollbacks a transaction.\n\ It's a wrapper to jtrans_rollback().\n"); -static PyObject *jt_rollback(jtransobject *tp, PyObject *args) +static PyObject *jt_rollback(jtrans_object *tp, PyObject *args) { long rv; @@ -473,22 +477,23 @@ static PyObject *jt_rollback(jtransobject *tp, PyObject *args) /* method table */ static PyMethodDef jtrans_methods[] = { - { "add", (PyCFunction)jt_add, METH_VARARGS, jt_add__doc }, - { "commit", (PyCFunction)jt_commit, METH_VARARGS, jt_commit__doc }, - { "rollback", (PyCFunction)jt_rollback, METH_VARARGS, jt_rollback__doc }, + { "add", (PyCFunction) jt_add, METH_VARARGS, jt_add__doc }, + { "commit", (PyCFunction) jt_commit, METH_VARARGS, jt_commit__doc }, + { "rollback", (PyCFunction) jt_rollback, METH_VARARGS, + jt_rollback__doc }, { NULL } }; -static PyObject *jt_getattr(jtransobject *tp, char *name) +static PyObject *jt_getattr(jtrans_object *tp, char *name) { return Py_FindMethod(jtrans_methods, (PyObject *)tp, name); } -static PyTypeObject JTransType = { +static PyTypeObject jtrans_type = { PyObject_HEAD_INIT(NULL) 0, "libjio.jtrans", - sizeof(jtransobject), + sizeof(jtrans_object), 0, (destructor)jt_dealloc, 0, @@ -515,7 +520,7 @@ static PyObject *jf_open(PyObject *self, PyObject *args) int rv; char *file; int flags, mode, jflags; - jfileobject *fp; + jfile_object *fp; flags = O_RDWR; mode = 0600; @@ -525,7 +530,7 @@ static PyObject *jf_open(PyObject *self, PyObject *args) &jflags)) return NULL; - fp = PyObject_New(jfileobject, &JFileType); + fp = PyObject_New(jfile_object, &jfile_type); if (fp == NULL) return NULL; @@ -627,9 +632,10 @@ static PyObject *jf_jfsck_cleanup(PyObject *self, PyObject *args) /* function table */ static PyMethodDef libjio_functions[] = { - { "open", (PyCFunction)jf_open, METH_VARARGS, jf_open__doc }, - { "jfsck", (PyCFunction)jf_jfsck, METH_VARARGS, jf_jfsck__doc }, - { "jfsck_cleanup", (PyCFunction)jf_jfsck_cleanup, METH_VARARGS, jf_jfsck_cleanup__doc }, + { "open", (PyCFunction) jf_open, METH_VARARGS, jf_open__doc }, + { "jfsck", (PyCFunction) jf_jfsck, METH_VARARGS, jf_jfsck__doc }, + { "jfsck_cleanup", (PyCFunction) jf_jfsck_cleanup, METH_VARARGS, + jf_jfsck_cleanup__doc }, { NULL, }, }; @@ -645,16 +651,16 @@ PyMODINIT_FUNC initlibjio(void) { PyObject* m; - JFileType.ob_type = &PyType_Type; - JTransType.ob_type = &PyType_Type; + jfile_type.ob_type = &PyType_Type; + jtrans_type.ob_type = &PyType_Type; m = Py_InitModule3("libjio", libjio_functions, libjio__doc); - Py_INCREF(&JFileType); - PyModule_AddObject(m, "jfile", (PyObject *) &JFileType); + Py_INCREF(&jfile_type); + PyModule_AddObject(m, "jfile", (PyObject *) &jfile_type); - Py_INCREF(&JTransType); - PyModule_AddObject(m, "jtrans", (PyObject *) &JTransType); + Py_INCREF(&jtrans_type); + PyModule_AddObject(m, "jtrans", (PyObject *) &jtrans_type); /* libjio's constants */ PyModule_AddIntConstant(m, "J_NOLOCK", J_NOLOCK); diff --git a/bindings/python/setup.py b/bindings/python2/setup.py similarity index 100% copy from bindings/python/setup.py copy to bindings/python2/setup.py diff --git a/bindings/python/libjio.c b/bindings/python3/libjio.c similarity index 75% rename from bindings/python/libjio.c rename to bindings/python3/libjio.c index eb90e5a..7d63691 100644 --- a/bindings/python/libjio.c +++ b/bindings/python3/libjio.c @@ -1,8 +1,7 @@ /* - * Python bindings for libjio + * Python 3 bindings for libjio * Alberto Bertogli (albertito@blitiri.com.ar) - * Aug/2004 */ @@ -25,7 +24,7 @@ * UNIX file. * * The second one represents a single transaction, which is composed of - * several operations that get added by its add() method. It gets commited + * several operations that get added by its add() method. It gets committed * with commit(), and rolled back with rollback(). * * There rest of the module's functions are related to file checking, called @@ -41,16 +40,19 @@ typedef struct { PyObject_HEAD struct jfs *fs; -} jfileobject; -static PyTypeObject JFileType; +} jfile_object; + +static PyTypeObject jfile_type; + /* jtrans */ typedef struct { PyObject_HEAD struct jtrans *ts; - jfileobject *jfile; -} jtransobject; -static PyTypeObject JTransType; + jfile_object *jfile; +} jtrans_object; + +static PyTypeObject jtrans_type; /* @@ -58,7 +60,7 @@ static PyTypeObject JTransType; */ /* delete */ -static void jf_dealloc(jfileobject *fp) +static void jf_dealloc(jfile_object *fp) { if (fp->fs) { jclose(fp->fs); @@ -73,12 +75,12 @@ PyDoc_STRVAR(jf_fileno__doc, \n\ Return the file descriptor number for the file.\n"); -static PyObject *jf_fileno(jfileobject *fp, PyObject *args) +static PyObject *jf_fileno(jfile_object *fp, PyObject *args) { if (!PyArg_ParseTuple(args, ":fileno")) return NULL; - return PyInt_FromLong(fp->fs->fd); + return PyLong_FromLong(fp->fs->fd); } /* read */ @@ -89,7 +91,7 @@ Read at most size bytes from the file, returns the string with\n\ the contents.\n\ It's a wrapper to jread().\n"); -static PyObject *jf_read(jfileobject *fp, PyObject *args) +static PyObject *jf_read(jfile_object *fp, PyObject *args) { long rv; long len; @@ -110,7 +112,7 @@ static PyObject *jf_read(jfileobject *fp, PyObject *args) if (rv < 0) { r = PyErr_SetFromErrno(PyExc_IOError); } else { - r = PyString_FromStringAndSize((char *) buf, rv); + r = PyBytes_FromStringAndSize((char *) buf, rv); } free(buf); @@ -125,7 +127,7 @@ Read size bytes from the file at the given offset, return a string with the\n\ contents.\n\ It's a wrapper to jpread().\n"); -static PyObject *jf_pread(jfileobject *fp, PyObject *args) +static PyObject *jf_pread(jfile_object *fp, PyObject *args) { long rv; long len; @@ -147,7 +149,7 @@ static PyObject *jf_pread(jfileobject *fp, PyObject *args) if (rv < 0) { r = PyErr_SetFromErrno(PyExc_IOError); } else { - r = PyString_FromStringAndSize((char *) buf, rv); + r = PyBytes_FromStringAndSize((char *) buf, rv); } free(buf); @@ -162,7 +164,7 @@ Write the contents of the given buffer (a string) to the file, returns the\n\ number of bytes written.\n\ It's a wrapper to jwrite().\n"); -static PyObject *jf_write(jfileobject *fp, PyObject *args) +static PyObject *jf_write(jfile_object *fp, PyObject *args) { long rv; unsigned char *buf; @@ -189,7 +191,7 @@ Write the contents of the given buffer (a string) to the file at the given\n\ offset, returns the number of bytes written.\n\ It's a wrapper to jpwrite().\n"); -static PyObject *jf_pwrite(jfileobject *fp, PyObject *args) +static PyObject *jf_pwrite(jfile_object *fp, PyObject *args) { long rv; unsigned char *buf; @@ -216,7 +218,7 @@ PyDoc_STRVAR(jf_truncate__doc, Truncate the file to the given size.\n\ It's a wrapper to jtruncate().\n"); -static PyObject *jf_truncate(jfileobject *fp, PyObject *args) +static PyObject *jf_truncate(jfile_object *fp, PyObject *args) { int rv; long long lenght; @@ -248,7 +250,7 @@ These constants are defined in the module. See lseek's manpage for more\n\ information.\n\ It's a wrapper to jlseek().\n"); -static PyObject *jf_lseek(jfileobject *fp, PyObject *args) +static PyObject *jf_lseek(jfile_object *fp, PyObject *args) { long long rv; int whence; @@ -275,7 +277,7 @@ Used with lingering transactions, see the library documentation for more\n\ detailed information.\n\ It's a wrapper to jsync().\n"); -static PyObject *jf_jsync(jfileobject *fp, PyObject *args) +static PyObject *jf_jsync(jfile_object *fp, PyObject *args) { long rv; @@ -300,7 +302,7 @@ Moves the journal directory to the new path; note that there MUST NOT BE\n\ anything else operating on the file.\n\ It's a wrapper to jmove_journal().\n"); -static PyObject *jf_jmove_journal(jfileobject *fp, PyObject *args) +static PyObject *jf_jmove_journal(jfile_object *fp, PyObject *args) { long rv; char *newpath; @@ -325,14 +327,14 @@ PyDoc_STRVAR(jf_new_trans__doc, Returns an object representing a new empty transaction.\n\ It's a wrapper to jtrans_init().\n"); -static PyObject *jf_new_trans(jfileobject *fp, PyObject *args) +static PyObject *jf_new_trans(jfile_object *fp, PyObject *args) { - jtransobject *tp; + jtrans_object *tp; if (!PyArg_ParseTuple(args, ":new_trans")) return NULL; - tp = PyObject_New(jtransobject, &JTransType); + tp = (jtrans_object *) jtrans_type.tp_alloc(&jtrans_type, 0); if (tp == NULL) return NULL; @@ -350,36 +352,30 @@ static PyObject *jf_new_trans(jfileobject *fp, PyObject *args) return (PyObject *) tp; } - /* method table */ static PyMethodDef jfile_methods[] = { - { "fileno", (PyCFunction)jf_fileno, METH_VARARGS, jf_fileno__doc }, - { "read", (PyCFunction)jf_read, METH_VARARGS, jf_read__doc }, - { "pread", (PyCFunction)jf_pread, METH_VARARGS, jf_pread__doc }, - { "write", (PyCFunction)jf_write, METH_VARARGS, jf_write__doc }, - { "pwrite", (PyCFunction)jf_pwrite, METH_VARARGS, jf_pwrite__doc }, - { "truncate", (PyCFunction)jf_truncate, METH_VARARGS, jf_truncate__doc }, - { "lseek", (PyCFunction)jf_lseek, METH_VARARGS, jf_lseek__doc }, - { "jsync", (PyCFunction)jf_jsync, METH_VARARGS, jf_jsync__doc }, - { "jmove_journal", (PyCFunction)jf_jmove_journal, METH_VARARGS, jf_jmove_journal__doc }, - { "new_trans", (PyCFunction)jf_new_trans, METH_VARARGS, jf_new_trans__doc }, + { "fileno", (PyCFunction) jf_fileno, METH_VARARGS, jf_fileno__doc }, + { "read", (PyCFunction) jf_read, METH_VARARGS, jf_read__doc }, + { "pread", (PyCFunction) jf_pread, METH_VARARGS, jf_pread__doc }, + { "write", (PyCFunction) jf_write, METH_VARARGS, jf_write__doc }, + { "pwrite", (PyCFunction) jf_pwrite, METH_VARARGS, jf_pwrite__doc }, + { "truncate", (PyCFunction) jf_truncate, METH_VARARGS, + jf_truncate__doc }, + { "lseek", (PyCFunction) jf_lseek, METH_VARARGS, jf_lseek__doc }, + { "jsync", (PyCFunction) jf_jsync, METH_VARARGS, jf_jsync__doc }, + { "jmove_journal", (PyCFunction) jf_jmove_journal, METH_VARARGS, + jf_jmove_journal__doc }, + { "new_trans", (PyCFunction) jf_new_trans, METH_VARARGS, + jf_new_trans__doc }, { NULL } }; -static PyObject *jf_getattr(jfileobject *fp, char *name) -{ - return Py_FindMethod(jfile_methods, (PyObject *)fp, name); -} - -static PyTypeObject JFileType = { +static PyTypeObject jfile_type = { PyObject_HEAD_INIT(NULL) - 0, - "libjio.jfile", - sizeof(jfileobject), - 0, - (destructor)jf_dealloc, - 0, - (getattrfunc)jf_getattr, + .tp_name = "libjio.jfile", + .tp_itemsize = sizeof(jfile_object), + .tp_dealloc = (destructor) jf_dealloc, + .tp_methods = jfile_methods, }; @@ -388,7 +384,7 @@ static PyTypeObject JFileType = { */ /* delete */ -static void jt_dealloc(jtransobject *tp) +static void jt_dealloc(jtrans_object *tp) { if (tp->ts != NULL) { jtrans_free(tp->ts); @@ -406,7 +402,7 @@ Add an operation to write the given buffer at the given offset to the\n\ transaction.\n\ It's a wrapper to jtrans_add().\n"); -static PyObject *jt_add(jtransobject *tp, PyObject *args) +static PyObject *jt_add(jtrans_object *tp, PyObject *args) { long rv; int len; @@ -430,7 +426,7 @@ PyDoc_STRVAR(jt_commit__doc, Commits a transaction.\n\ It's a wrapper to jtrans_commit().\n"); -static PyObject *jt_commit(jtransobject *tp, PyObject *args) +static PyObject *jt_commit(jtrans_object *tp, PyObject *args) { long rv; @@ -454,7 +450,7 @@ PyDoc_STRVAR(jt_rollback__doc, Rollbacks a transaction.\n\ It's a wrapper to jtrans_rollback().\n"); -static PyObject *jt_rollback(jtransobject *tp, PyObject *args) +static PyObject *jt_rollback(jtrans_object *tp, PyObject *args) { long rv; @@ -473,30 +469,21 @@ static PyObject *jt_rollback(jtransobject *tp, PyObject *args) /* method table */ static PyMethodDef jtrans_methods[] = { - { "add", (PyCFunction)jt_add, METH_VARARGS, jt_add__doc }, - { "commit", (PyCFunction)jt_commit, METH_VARARGS, jt_commit__doc }, - { "rollback", (PyCFunction)jt_rollback, METH_VARARGS, jt_rollback__doc }, + { "add", (PyCFunction) jt_add, METH_VARARGS, jt_add__doc }, + { "commit", (PyCFunction) jt_commit, METH_VARARGS, jt_commit__doc }, + { "rollback", (PyCFunction) jt_rollback, METH_VARARGS, jt_rollback__doc }, { NULL } }; -static PyObject *jt_getattr(jtransobject *tp, char *name) -{ - return Py_FindMethod(jtrans_methods, (PyObject *)tp, name); -} - -static PyTypeObject JTransType = { +static PyTypeObject jtrans_type = { PyObject_HEAD_INIT(NULL) - 0, - "libjio.jtrans", - sizeof(jtransobject), - 0, - (destructor)jt_dealloc, - 0, - (getattrfunc)jt_getattr, + .tp_name = "libjio.jtrans", + .tp_itemsize = sizeof(jtrans_object), + .tp_dealloc = (destructor) jt_dealloc, + .tp_methods = jtrans_methods, }; - /* * The module */ @@ -515,7 +502,7 @@ static PyObject *jf_open(PyObject *self, PyObject *args) int rv; char *file; int flags, mode, jflags; - jfileobject *fp; + jfile_object *fp; flags = O_RDWR; mode = 0600; @@ -525,7 +512,7 @@ static PyObject *jf_open(PyObject *self, PyObject *args) &jflags)) return NULL; - fp = PyObject_New(jfileobject, &JFileType); + fp = (jfile_object *) jfile_type.tp_alloc(&jfile_type, 0); if (fp == NULL) return NULL; @@ -582,7 +569,7 @@ static PyObject *jf_jfsck(PyObject *self, PyObject *args) return PyErr_NoMemory(); } else if (rv != 0) { Py_XDECREF(dict); - PyErr_SetObject(PyExc_IOError, PyInt_FromLong(rv)); + PyErr_SetObject(PyExc_IOError, PyLong_FromLong(rv)); return NULL; } @@ -618,43 +605,52 @@ static PyObject *jf_jfsck_cleanup(PyObject *self, PyObject *args) Py_END_ALLOW_THREADS if (rv != 1) { - PyErr_SetObject(PyExc_IOError, PyInt_FromLong(rv)); + PyErr_SetObject(PyExc_IOError, PyLong_FromLong(rv)); return NULL; } - return PyInt_FromLong(rv); + return PyLong_FromLong(rv); } -/* function table */ -static PyMethodDef libjio_functions[] = { - { "open", (PyCFunction)jf_open, METH_VARARGS, jf_open__doc }, - { "jfsck", (PyCFunction)jf_jfsck, METH_VARARGS, jf_jfsck__doc }, - { "jfsck_cleanup", (PyCFunction)jf_jfsck_cleanup, METH_VARARGS, jf_jfsck_cleanup__doc }, - { NULL, }, + +static PyMethodDef module_methods[] = { + { "open", jf_open, METH_VARARGS, jf_open__doc }, + { "jfsck", jf_jfsck, METH_VARARGS, jf_jfsck__doc }, + { "jfsck_cleanup", jf_jfsck_cleanup, METH_VARARGS, + jf_jfsck_cleanup__doc }, + { NULL, NULL, 0, NULL }, }; -/* module initialization */ -PyDoc_STRVAR(libjio__doc, -"libjio is a library to do transactional, journaled I/O\n\ -You can find it at http://blitiri.com.ar/p/libjio/\n\ -\n\ -Use the open() method to create a file object, and then operate on it.\n\ -Please read the documentation for more information.\n"); +#define module_doc "libjio is a library to do transactional, journaled I/O\n" \ + "You can find it at http://blitiri.com.ar/p/libjio/\n" \ + "\n" \ + "Use the open() method to create a file object, and then operate " \ + "on it.\n" \ + "Please read the documentation for more information.\n" + +static PyModuleDef libjio_module = { + PyModuleDef_HEAD_INIT, + .m_name = "libjio", + .m_doc = module_doc, + .m_size = -1, + .m_methods = module_methods, +}; -PyMODINIT_FUNC initlibjio(void) +PyMODINIT_FUNC PyInit_libjio(void) { - PyObject* m; + PyObject *m; - JFileType.ob_type = &PyType_Type; - JTransType.ob_type = &PyType_Type; + if (PyType_Ready(&jfile_type) < 0 || + PyType_Ready(&jtrans_type) < 0) + return NULL; - m = Py_InitModule3("libjio", libjio_functions, libjio__doc); + m = PyModule_Create(&libjio_module); - Py_INCREF(&JFileType); - PyModule_AddObject(m, "jfile", (PyObject *) &JFileType); + Py_INCREF(&jfile_type); + PyModule_AddObject(m, "jfile", (PyObject *) &jfile_type); - Py_INCREF(&JTransType); - PyModule_AddObject(m, "jtrans", (PyObject *) &JTransType); + Py_INCREF(&jtrans_type); + PyModule_AddObject(m, "jtrans", (PyObject *) &jtrans_type); /* libjio's constants */ PyModule_AddIntConstant(m, "J_NOLOCK", J_NOLOCK); @@ -686,5 +682,7 @@ PyMODINIT_FUNC initlibjio(void) PyModule_AddIntConstant(m, "SEEK_SET", SEEK_SET); PyModule_AddIntConstant(m, "SEEK_CUR", SEEK_CUR); PyModule_AddIntConstant(m, "SEEK_END", SEEK_END); + + return m; } diff --git a/bindings/python/setup.py b/bindings/python3/setup.py similarity index 100% rename from bindings/python/setup.py rename to bindings/python3/setup.py diff --git a/doc/LICENSE b/doc/LICENSE deleted file mode 100644 index 3a91342..0000000 --- a/doc/LICENSE +++ /dev/null @@ -1,188 +0,0 @@ - -This project, 'libjio', is copyrighted by Alberto Bertogli and licensed under -the Open Software License version 3.0 as obtained from www.opensource.org (and -included here-in for easy reference) (that license itself is copyrighted by -Larry Rosen). - -Note that the "Original Work" that this license covers is only the library -itself. Thus just the act of linking/importing this library into another -program does NOT in itself make that program considered a derivative work of -this Original Work. - - Alberto Bertogli - 15 July 2007 - -------------------------------------------------------------------------- - -This Open Software License (the "License") applies to any original work of -authorship (the "Original Work") whose owner (the "Licensor") has placed the -following licensing notice adjacent to the copyright notice for the Original -Work: - - Licensed under the Open Software License version 3.0 - -1. Grant of Copyright License. Licensor grants You a worldwide, royalty-free, -non-exclusive, sublicensable license, for the duration of the copyright, to do -the following: - - 1. to reproduce the Original Work in copies, either alone or as part of a - collective work; - - 2. to translate, adapt, alter, transform, modify, or arrange the Original - Work, thereby creating derivative works ("Derivative Works") based upon - the Original Work; - - 3. to distribute or communicate copies of the Original Work and Derivative - Works to the public, with the proviso that copies of Original Work or - Derivative Works that You distribute or communicate shall be licensed - under this Open Software License; - - 4. to perform the Original Work publicly; and - - 5. to display the Original Work publicly. - -2. Grant of Patent License. Licensor grants You a worldwide, royalty-free, -non-exclusive, sublicensable license, under patent claims owned or controlled -by the Licensor that are embodied in the Original Work as furnished by the -Licensor, for the duration of the patents, to make, use, sell, offer for sale, -have made, and import the Original Work and Derivative Works. - -3. Grant of Source Code License. The term "Source Code" means the preferred -form of the Original Work for making modifications to it and all available -documentation describing how to modify the Original Work. Licensor agrees to -provide a machine-readable copy of the Source Code of the Original Work along -with each copy of the Original Work that Licensor distributes. Licensor -reserves the right to satisfy this obligation by placing a machine-readable -copy of the Source Code in an information repository reasonably calculated to -permit inexpensive and convenient access by You for as long as Licensor -continues to distribute the Original Work. - -4. Exclusions From License Grant. Neither the names of Licensor, nor the names -of any contributors to the Original Work, nor any of their trademarks or -service marks, may be used to endorse or promote products derived from this -Original Work without express prior permission of the Licensor. Except as -expressly stated herein, nothing in this License grants any license to -Licensor's trademarks, copyrights, patents, trade secrets or any other -intellectual property. No patent license is granted to make, use, sell, offer -for sale, have made, or import embodiments of any patent claims other than the -licensed claims defined in Section 2. No license is granted to the trademarks -of Licensor even if such marks are included in the Original Work. Nothing in -this License shall be interpreted to prohibit Licensor from licensing under -terms different from this License any Original Work that Licensor otherwise -would have a right to license. - -5. External Deployment. The term "External Deployment" means the use, -distribution, or communication of the Original Work or Derivative Works in any -way such that the Original Work or Derivative Works may be used by anyone -other than You, whether those works are distributed or communicated to those -persons or made available as an application intended for use over a network. -As an express condition for the grants of license hereunder, You must treat -any External Deployment by You of the Original Work or a Derivative Work as a -distribution under section 1(c). - -6. Attribution Rights. You must retain, in the Source Code of any Derivative -Works that You create, all copyright, patent, or trademark notices from the -Source Code of the Original Work, as well as any notices of licensing and any -descriptive text identified therein as an "Attribution Notice." You must cause -the Source Code for any Derivative Works that You create to carry a prominent -Attribution Notice reasonably calculated to inform recipients that You have -modified the Original Work. - -7. Warranty of Provenance and Disclaimer of Warranty. Licensor warrants that -the copyright in and to the Original Work and the patent rights granted herein -by Licensor are owned by the Licensor or are sublicensed to You under the -terms of this License with the permission of the contributor(s) of those -copyrights and patent rights. Except as expressly stated in the immediately -preceding sentence, the Original Work is provided under this License on an "AS -IS" BASIS and WITHOUT WARRANTY, either express or implied, including, without -limitation, the warranties of non-infringement, merchantability or fitness for -a particular purpose. THE ENTIRE RISK AS TO THE QUALITY OF THE ORIGINAL WORK -IS WITH YOU. This DISCLAIMER OF WARRANTY constitutes an essential part of this -License. No license to the Original Work is granted by this License except -under this disclaimer. - -8. Limitation of Liability. Under no circumstances and under no legal theory, -whether in tort (including negligence), contract, or otherwise, shall the -Licensor be liable to anyone for any indirect, special, incidental, or -consequential damages of any character arising as a result of this License or -the use of the Original Work including, without limitation, damages for loss -of goodwill, work stoppage, computer failure or malfunction, or any and all -other commercial damages or losses. This limitation of liability shall not -apply to the extent applicable law prohibits such limitation. - -9. Acceptance and Termination. If, at any time, You expressly assented to this -License, that assent indicates your clear and irrevocable acceptance of this -License and all of its terms and conditions. If You distribute or communicate -copies of the Original Work or a Derivative Work, You must make a reasonable -effort under the circumstances to obtain the express assent of recipients to -the terms of this License. This License conditions your rights to undertake -the activities listed in Section 1, including your right to create Derivative -Works based upon the Original Work, and doing so without honoring these terms -and conditions is prohibited by copyright law and international treaty. -Nothing in this License is intended to affect copyright exceptions and -limitations (including 'fair use' or 'fair dealing'). This License shall -terminate immediately and You may no longer exercise any of the rights granted -to You by this License upon your failure to honor the conditions in Section -1(c). - -10. Termination for Patent Action. This License shall terminate automatically -and You may no longer exercise any of the rights granted to You by this -License as of the date You commence an action, including a cross-claim or -counterclaim, against Licensor or any licensee alleging that the Original Work -infringes a patent. This termination provision shall not apply for an action -alleging patent infringement by combinations of the Original Work with other -software or hardware. - -11. Jurisdiction, Venue and Governing Law. Any action or suit relating to this -License may be brought only in the courts of a jurisdiction wherein the -Licensor resides or in which Licensor conducts its primary business, and under -the laws of that jurisdiction excluding its conflict-of-law provisions. The -application of the United Nations Convention on Contracts for the -International Sale of Goods is expressly excluded. Any use of the Original -Work outside the scope of this License or after its termination shall be -subject to the requirements and penalties of copyright or patent law in the -appropriate jurisdiction. This section shall survive the termination of this -License. - -12. Attorneys' Fees. In any action to enforce the terms of this License or -seeking damages relating thereto, the prevailing party shall be entitled to -recover its costs and expenses, including, without limitation, reasonable -attorneys' fees and costs incurred in connection with such action, including -any appeal of such action. This section shall survive the termination of this -License. - -13. Miscellaneous. If any provision of this License is held to be -unenforceable, such provision shall be reformed only to the extent necessary -to make it enforceable. - -14. Definition of "You" in This License. "You" throughout this License, -whether in upper or lower case, means an individual or a legal entity -exercising rights under, and complying with all of the terms of, this License. -For legal entities, "You" includes any entity that controls, is controlled by, -or is under common control with you. For purposes of this definition, -"control" means (i) the power, direct or indirect, to cause the direction or -management of such entity, whether by contract or otherwise, or (ii) ownership -of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial -ownership of such entity. - -15. Right to Use. You may use the Original Work in all ways not otherwise -restricted or conditioned by this License or by law, and Licensor promises not -to interfere with or be responsible for such uses by You. - -16. Modification of This License. This License is Copyright © 2005 Lawrence -Rosen. Permission is granted to copy, distribute, or communicate this License -without modification. Nothing in this License permits You to modify this -License as applied to the Original Work or to Derivative Works. However, You -may modify the text of this License and copy, distribute or communicate your -modified version (the "Modified License") and apply it to other original works -of authorship subject to the following conditions: (i) You may not indicate in -any way that your Modified License is the "Open Software License" or "OSL" and -you may not use those names in the name of your Modified License; (ii) You -must replace the notice specified in the first paragraph above with the notice -"Licensed under " or with a notice of your own -that is not confusingly similar to the notice in this License; and (iii) You -may not claim that your original works are open source software unless your -Modified License has been approved by Open Source Initiative (OSI) and You -comply with its license review and certification process. - - diff --git a/Makefile b/libjio/Makefile similarity index 57% copy from Makefile copy to libjio/Makefile index 6bad0ab..c3ab13e 100644 --- a/Makefile +++ b/libjio/Makefile @@ -1,15 +1,16 @@ -VERSION="0.24" - CFLAGS = -std=c99 -pedantic -Wall -O3 MANDATORY_CFLAGS := \ - -D_LARGEFILE_SOURCE=1 -D_LARGEFILE64_SOURCE=1 \ - -D_LFS_LARGEFILE=1 -D_LFS64_LARGEFILE=1 \ - -D_FILE_OFFSET_BITS=64 $(shell getconf LFS_CFLAGS 2>/dev/null) \ - -D_XOPEN_SOURCE=500 + -D_LARGEFILE_SOURCE=1 $(shell getconf LFS_CFLAGS 2>/dev/null) \ + -D_XOPEN_SOURCE=600 + +MANDATORY_LDFLAGS := $(shell getconf LFS_LIBS 2>/dev/null) ALL_CFLAGS += $(CFLAGS) $(MANDATORY_CFLAGS) -fPIC +ALL_LDFLAGS += $(LDFLAGS) $(MANDATORY_LDFLAGS) -fPIC + +LIBS = -lpthread ifdef DEBUG ALL_CFLAGS += -g @@ -19,9 +20,13 @@ ifdef PROFILE ALL_CFLAGS += -g -pg -fprofile-arcs -ftest-coverage endif +ifdef FI +ALL_CFLAGS += -DFIU_ENABLE=1 +LIBS += -lfiu +endif # prefix for installing the binaries -PREFIX=/usr/local +PREFIX = /usr/local ifneq ($(V), 1) @@ -42,7 +47,7 @@ default: all all: libjio.so libjio.a libjio.pc jiofsck libjio.so: $(OBJS) - $(NICE_CC) -shared $(ALL_CFLAGS) $(OBJS) -o libjio.so + $(NICE_CC) -shared $(ALL_LDFLAGS) $(LIBS) $(OBJS) -o libjio.so libjio.a: $(OBJS) $(NICE_AR) cr libjio.a $(OBJS) @@ -55,7 +60,7 @@ libjio.pc: libjio.skel.pc > libjio.pc jiofsck: jiofsck.o libjio.a - $(NICE_CC) $(ALL_CFLAGS) jiofsck.o libjio.a -lpthread -o jiofsck + $(NICE_CC) $(ALL_LDFLAGS) jiofsck.o libjio.a $(LIBS) -o jiofsck install: all install -d $(PREFIX)/lib @@ -68,41 +73,19 @@ install: all install -d $(PREFIX)/bin install -m 0775 jiofsck $(PREFIX)/bin install -d $(PREFIX)/man/man3 - install -m 0644 doc/libjio.3 $(PREFIX)/man/man3/ + install -m 0644 libjio.3 $(PREFIX)/man/man3/ @echo @echo "Please run ldconfig to update your library cache" @echo .c.o: - $(NICE_CC) $(ALL_CFLAGS) $(INCLUDES) -c $< -o $@ - - -python: all - cd bindings/python && python setup.py build - -python_install: python - cd bindings/python && python setup.py install - - -preload: all - install -d bindings/preload/build/ - $(NICE_CC) $(INCLUDES) -Wall -O3 -shared -fPIC \ - -D_XOPEN_SOURCE=500 \ - -ldl -lpthread -L. -ljio -I. \ - bindings/preload/libjio_preload.c \ - -o bindings/preload/build/libjio_preload.so - -preload_install: preload - install -d $(PREFIX)/lib - install -m 0755 bindings/preload/build/libjio_preload.so $(PREFIX)/lib + $(NICE_CC) $(ALL_CFLAGS) -c $< -o $@ clean: rm -f $(OBJS) libjio.a libjio.so libjio.pc jiofsck.o jiofsck rm -f *.bb *.bbg *.da *.gcov *.gcno *.gcda gmon.out - rm -rf bindings/python/build/ - rm -rf bindings/preload/build/ -.PHONY: default all install python python_install preload preload_install clean +.PHONY: default all install clean diff --git a/ansi.c b/libjio/ansi.c similarity index 100% rename from ansi.c rename to libjio/ansi.c diff --git a/check.c b/libjio/check.c similarity index 90% rename from check.c rename to libjio/check.c index c430d14..2fabde5 100644 --- a/check.c +++ b/libjio/check.c @@ -1,8 +1,5 @@ /* - * libjio - A library for Journaled I/O - * Alberto Bertogli (albertito@blitiri.com.ar) - * * Recovery functions */ @@ -23,11 +20,12 @@ /* fill a transaction structure from a mmapped transaction file */ -static int fill_trans(unsigned char *map, off_t len, struct jtrans *ts) +static off_t fill_trans(unsigned char *map, off_t len, struct jtrans *ts) { int i; unsigned char *p; struct joper *op, *tmp; + off_t translen; if (len < J_DISKHEADSIZE) return 0; @@ -43,6 +41,8 @@ static int fill_trans(unsigned char *map, off_t len, struct jtrans *ts) ts->numops = *( (uint32_t *) p); p += 4; + translen = J_DISKHEADSIZE; + for (i = 0; i < ts->numops; i++) { if (p + J_DISKOPHEADSIZE > map + len) goto error; @@ -79,9 +79,11 @@ static int fill_trans(unsigned char *map, off_t len, struct jtrans *ts) op->prev = tmp; op->next = NULL; } + + translen += J_DISKOPHEADSIZE + op->len; } - return 1; + return translen; error: while (ts->op != NULL) { @@ -106,7 +108,7 @@ int jfsck(const char *name, const char *jdir, struct jfsck_result *res) DIR *dir; struct dirent *dent; unsigned char *map; - off_t filelen, lr; + off_t filelen, translen, lr; tfd = -1; filelen = 0; @@ -189,8 +191,8 @@ int jfsck(const char *name, const char *jdir, struct jfsck_result *res) goto exit; } - /* loop for each file in the journal directory to find out the greater - * transaction number */ + /* find the greatest transaction number by looking into the journal + * directory */ maxtid = 0; for (dent = readdir(dir); dent != NULL; dent = readdir(dir)) { /* see if the file is named like a transaction, ignore @@ -211,7 +213,7 @@ int jfsck(const char *name, const char *jdir, struct jfsck_result *res) goto exit; } - /* we loop all the way up to the max transaction id */ + /* verify (and possibly fix) all the transactions */ for (i = 1; i <= maxtid; i++) { curts = malloc(sizeof(struct jtrans)); if (curts == NULL) { @@ -224,7 +226,7 @@ int jfsck(const char *name, const char *jdir, struct jfsck_result *res) /* open the transaction file, using i as its name, so we are * really looping in order (recovering transaction in a - * different order as they were applied means instant + * different order as they were applied would result in * corruption) */ get_jtfile(&fs, i, tname); tfd = open(tname, O_RDWR | O_SYNC, 0600); @@ -250,8 +252,15 @@ int jfsck(const char *name, const char *jdir, struct jfsck_result *res) map = NULL; goto loop; } - rv = fill_trans(map, filelen, curts); - if (rv != 1) { + translen = fill_trans(map, filelen, curts); + if (translen == 0) { + res->broken++; + goto loop; + } + + /* see if there's enough room for the checksum after the + * transaction information */ + if (filelen != translen + sizeof(uint32_t)) { res->broken++; goto loop; } @@ -264,7 +273,8 @@ int jfsck(const char *name, const char *jdir, struct jfsck_result *res) goto loop; } - /* remove flags from the transaction */ + /* remove flags from the transaction, so we don't have issues + * re-committing */ curts->flags = 0; rv = jtrans_commit(curts); @@ -349,13 +359,11 @@ int jfsck_cleanup(const char *name, const char *jdir) strcat(tfile, "/"); strcat(tfile, dent->d_name); - /* the full filename is too large */ if (strlen(tfile) > PATH_MAX) { closedir(dir); return 0; } - /* and remove it */ if (unlink(tfile) != 0) { closedir(dir); return 0; diff --git a/checksum.c b/libjio/checksum.c similarity index 100% rename from checksum.c rename to libjio/checksum.c diff --git a/common.c b/libjio/common.c similarity index 100% rename from common.c rename to libjio/common.c diff --git a/common.h b/libjio/common.h similarity index 94% rename from common.h rename to libjio/common.h index b25ec26..c946591 100644 --- a/common.h +++ b/libjio/common.h @@ -13,6 +13,7 @@ #include /* for uint*_t */ #include "libjio.h" /* for struct jfs */ +#include "fiu-local.h" /* for fault injection functions */ #define _F_READ 0x00001 #define _F_WRITE 0x00010 diff --git a/libjio/compat.h b/libjio/compat.h new file mode 100644 index 0000000..ecc742a --- /dev/null +++ b/libjio/compat.h @@ -0,0 +1,19 @@ + +/* Header to provide fallbacks for compatibility purposes. */ + +#ifndef _COMPAT_H +#define _COMPAT_H + +/* posix_fadvise() was introduced in SUSv3. Because it's the only SUSv3 + * function we rely on so far (everything else is SUSv2), we define a void + * fallback for systems that do not implement it. + * + * The check is based on POSIX_FADV_WILLNEED being defined, which is not very + * nice, but it's simple, it works and should be reliable. */ +#include +#ifndef POSIX_FADV_WILLNEED +#define posix_fadvise(fd, offset, len, advise) +#endif + +#endif + diff --git a/libjio/fiu-local.h b/libjio/fiu-local.h new file mode 100644 index 0000000..b68327b --- /dev/null +++ b/libjio/fiu-local.h @@ -0,0 +1,37 @@ + +/* libfiu - Fault Injection in Userspace + * + * This header, part of libfiu, is meant to be included in your project to + * avoid having libfiu as a mandatory build-time dependency. + * + * You can add it to your project, and #include it instead of fiu.h. + * The real fiu.h will be used only when FIU_ENABLE is defined. + * + * This header, as the rest of libfiu, is in the public domain. + * + * You can find more information about libfiu at + * http://blitiri.com.ar/p/libfiu. + */ + +#ifndef _FIU_LOCAL_H +#define _FIU_LOCAL_H + +/* Only define the stubs when fiu is disabled, otherwise use the real fiu.h + * header */ +#ifndef FIU_ENABLE + +#define fiu_init(flags) 0 +#define fiu_fail(name) 0 +#define fiu_failinfo() NULL +#define fiu_do_on(name, action) +#define fiu_exit_on(name) +#define fiu_return_on(name, retval) + +#else + +#include + +#endif /* FIU_ENABLE */ + +#endif /* _FIU_LOCAL_H */ + diff --git a/jiofsck.c b/libjio/jiofsck.c similarity index 100% rename from jiofsck.c rename to libjio/jiofsck.c diff --git a/doc/libjio.3 b/libjio/libjio.3 similarity index 70% rename from doc/libjio.3 rename to libjio/libjio.3 index 414f47f..defac0e 100644 --- a/doc/libjio.3 +++ b/libjio/libjio.3 @@ -36,7 +36,7 @@ libjio - A library for Journaled I/O .BI "ssize_t jwritev(struct jfs *" fs ", const struct iovec *" vector "," .BI " int " count ");" .BI "int jtruncate(struct jfs *" fs ", off_t " lenght ");" - +.BI "off_t jlseek(struct jfs *" fs ", off_t " offset ", int " whence ");" .BI "int jclose(struct jfs *" fs ");" .BI "void jtrans_init(struct jfs *" fs " ,struct jtrans *" ts ");" @@ -46,8 +46,12 @@ libjio - A library for Journaled I/O .BI "int jtrans_rollback(struct jtrans *" ts ");" .BI "void jtrans_free(struct jtrans *" ts ");" -.BI "int jfsck(const char *" name ", struct jfsck_result *" res ");" -.BI "int jfsck_cleanup(const char *" name ");" +.BI "int jsync(struct jfs *" fs ");" +.BI "int jmove_journal(struct jfs *" fs ", const char *" newpath ");" + +.BI "int jfsck(const char *" name ", const char *" jdir "," +.BI " struct jfsck_result *" res ");" +.BI "int jfsck_cleanup(const char *" name ", const char *" jdir ");" .SH DESCRIPTION @@ -57,21 +61,26 @@ documentation that comes along with the library itself, or on the web at http://blitiri.com.ar/p/libjio. Functions can be grouped in three different groups: the common functions, the -UNIX-alike API, and the basic functions. +UNIX-alike API, and the basic functions. All return 0 upon successful +completion and < 0 upon failure, unless otherwise noted. The common functions provide functionality common to the other two: -.B jopen() -to open files in order to use them with the library, and +.BR jopen() " and " jclose() +to open and close files in order to use them with the library, and .BR "jfsck() " and " jfsck_cleanup()" to provide integrity checking. -The second group mimics somehow the traditional UNIX API by providing similar -interfaces to read(), write(), and their friends. +The UNIX-alike API mimics the traditional UNIX API by providing similar +interfaces to +.BR read(2) ", " write(2) , +and friends. -The basic functions consists of -.BR "jtrans_commit()" , " jtrans_add() " and " jtrans_rollback()" . -They provide a method for manipulating transactions, which are defined in the -.IR "jtrans structure" " (described above)." +The basic functions consist of +.BR jtrans_init() ", " jtrans_add() ", " jtrans_commit() " and " +.BR jtrans_rollback() . +They provide a lower-level method for manipulating transactions, which are +defined in the +.IR "jtrans structure" . .SS STRUCTURES @@ -87,26 +96,35 @@ run, see below for details. .SS COMMON FUNCTIONS -Most functions reference somehow the structures described avobe, specially +Most functions reference the structures described above, specially .IR "struct jfs" " and " "struct jtrans" . -They represent a file to operate on and a single transaction, respectively. To -open a file, you should use the -.B jopen() -call, which is just like the normal -.B open(3) +They represent, respectively, a file to operate on and a single transaction. +To open a file, you should use +.BR jopen() , +which is just like the normal +.B open(2) call but affects a pointer to a .IR struct jfs . To close a file, use .BR jclose() . They're exactly like the -.BR open(3) " and close() +.BR open(2) " and " close(2) functions but use a .I struct jfs instead of a file descriptor; take a look at their manpages if you have any doubts about how to use them. +.B jmove_journal() +can be used to move the journal directory to a new location. It can be called +only when nobody else is using the file. It is usually not used, except for +very special cases. + There are two functions that differ from the rest, which are .BR jfsck() " and " jfsck_cleanup() . +Both take as the first two parameters the path to the file to check and the +path to the journal directory (usually NULL for the default, unless you've +changed it manually using +.BR jmove_journal() ). The first one, .BR jfsck() , @@ -131,21 +149,24 @@ is intended to be used after by programs wanting to remove all the stall transaction files and leave the journal directory ready to use. After calling .BR jfsck() , -the transaction files will no longer be needed, so by cleaning up the +those transaction files will no longer be needed, so by cleaning up the directory you make sure you're starting over with a clean journal. It returns 0 if there was an error, or 1 if it succeeded. The aforementioned .I jiofsck can also optionally invoke this function after performing the regular checks. +Calling it after +.B jfsck() +is highly recommended. .SS UNIX-alike API The UNIX-alike API, as explained before, consists of the functions .BR jread() ", " jpread() ", " jreadv() ", " jwrite() ", " jpwrite() ", " -.BR jwritev() ", " jtruncate() . +.BR jwritev() ", " jtruncate() "and " jlseek() . -They are all exactly like the UNIX equivalent (if you still don't get it, take -the initial 'j' out), and behave the same way, with the only exception that -instead of a file descriptor you need to pass a pointer to a +They are all exactly like the UNIX equivalent, and behave the same way, with +the only exception that instead of a file descriptor you need to pass a +pointer to a .IR "struct jfs" . Again, I will not duplicate the manpage for all these functions, just refer to the regular UNIX versions to see how to use them, they all have the same @@ -153,18 +174,17 @@ semantics and behave the same way. .SS BASIC FUNCTIONS -The basic functions are the ones which manipulate transactions directly; they -are five: +The basic functions are the ones which manipulate transactions directly: .BR jtrans_init() ", " jtrans_add() ", " jtrans_commit() ", " jtrans_rollback() and .BR jtrans_free() . -These are intended to be use where your application requires direct control +These are intended to be use when your application requires direct control over the transactions. .BR jtrans_init() " and " jtrans_free() -just initialize and free a given transaction; the former should be called -prior any use, and the latter when you want to destroy a transaction. Note -that +just initialize and free a given transaction structure; the former should be +called prior any use, and the latter when you want to destroy a transaction. +Note that .B jtrans_free() is not a disk operation, but only frees the pointers that were previously allocated by the library; all disk operations are performed by the other two @@ -173,10 +193,10 @@ functions. .B jtrans_add() is used to add operations to a transaction, and it takes the same parameters as -.BR pwrite() . -It gets a buffer, its lenght and the offset where it should be applied, and -adds it to the transaction. You can add multiple operations to a transaction, -and they will be applied in order. +.BR pwrite() : +a buffer, its length and the offset where it should be applied, and adds it to +the transaction. You can add multiple operations to a transaction, and they +will be applied in order. .B jtrans_commit() commits the given transaction to disk. After it has returned, data has been @@ -202,11 +222,11 @@ was an error. .BR pread (2), .BR pwrite (2), .BR ftruncate (2), +.BR lseek (2), .BR close (2) .SH BUGS -None that I'm aware of, but if you find one please let me know at If you want to report bugs, or have any questions or comments, just let me know at albertito@blitiri.com.ar. diff --git a/libjio.h b/libjio/libjio.h similarity index 97% rename from libjio.h rename to libjio/libjio.h index 39be877..b092a48 100644 --- a/libjio.h +++ b/libjio/libjio.h @@ -19,14 +19,10 @@ * library (which uses LFS) and that's just begging for problems. There should * be a portable way for the C library to do some of this for us, but until I * find one, this is the best we can do */ -#if (!defined _FILE_OFFSET_BITS) || (_FILE_OFFSET_BITS != 64) +#ifndef _LARGEFILE_SOURCE #error "You must compile your application with Large File Support" #endif -#ifdef __cplusplus -extern "C" { -#endif - /* logical structures */ @@ -166,9 +162,5 @@ FILE *jfsopen(struct jfs *stream, const char *mode); #define J_ENOMEM -3 /* no enough free memory */ -#ifdef __cplusplus -} /* from extern "C" above */ -#endif - #endif diff --git a/libjio.skel.pc b/libjio/libjio.skel.pc similarity index 100% rename from libjio.skel.pc rename to libjio/libjio.skel.pc diff --git a/trans.c b/libjio/trans.c similarity index 95% rename from trans.c rename to libjio/trans.c index ba30674..6fb6885 100644 --- a/trans.c +++ b/libjio/trans.c @@ -21,6 +21,7 @@ #include "libjio.h" #include "common.h" +#include "compat.h" /* @@ -38,6 +39,8 @@ static unsigned int get_tid(struct jfs *fs) /* read the current max. curid */ curid = *(fs->jmap); + fiu_do_on("jio/get_tid/overflow", curid = -1); + /* increment it and handle overflows */ rv = curid + 1; if (rv == 0) @@ -201,6 +204,10 @@ int jtrans_add(struct jtrans *ts, const void *buf, size_t count, off_t offset) jop->pdata = NULL; jop->locked = 0; + /* it's highly likely that jtrans_commit() will want to read the + * current data, so we tell the kernel about that */ + posix_fadvise(ts->fs->fd, offset, count, POSIX_FADV_WILLNEED); + return 1; } @@ -241,6 +248,8 @@ ssize_t jtrans_commit(struct jtrans *ts) if (fd < 0) goto exit; + fiu_exit_on("jio/commit/created_tf"); + /* and lock it */ plockf(fd, F_LOCKW, 0, 0); @@ -269,6 +278,8 @@ ssize_t jtrans_commit(struct jtrans *ts) goto unlink_exit; } + fiu_exit_on("jio/commit/tf_header"); + free(buf_init); curpos = J_DISKHEADSIZE; @@ -334,6 +345,8 @@ ssize_t jtrans_commit(struct jtrans *ts) goto unlink_exit; } + fiu_exit_on("jio/commit/tf_ophdr"); + free(buf_init); curpos += J_DISKOPHEADSIZE; @@ -344,8 +357,12 @@ ssize_t jtrans_commit(struct jtrans *ts) goto unlink_exit; curpos += op->len; + + fiu_exit_on("jio/commit/tf_opdata"); } + fiu_exit_on("jio/commit/tf_data"); + /* compute and save the checksum (curpos is always small, so there's * no overflow possibility when we convert to size_t) */ if (!checksum(fd, curpos, &csum)) @@ -376,6 +393,8 @@ ssize_t jtrans_commit(struct jtrans *ts) } } + fiu_exit_on("jio/commit/tf_sync"); + /* now that we have a safe transaction file, let's apply it */ written = 0; for (op = ts->op; op != NULL; op = op->next) { @@ -384,8 +403,11 @@ ssize_t jtrans_commit(struct jtrans *ts) goto rollback_exit; written += rv; + fiu_exit_on("jio/commit/wrote_op"); } + fiu_exit_on("jio/commit/wrote_all_ops"); + if (ts->flags & J_LINGER) { linger = malloc(sizeof(struct jlinger)); if (linger == NULL) @@ -399,13 +421,17 @@ ssize_t jtrans_commit(struct jtrans *ts) ts->fs->ltrans = linger; pthread_mutex_unlock(&(ts->fs->ltlock)); } else { + if (fdatasync(ts->fs->fd) != 0) + goto rollback_exit; + /* the transaction has been applied, so we cleanup and remove * it from the disk */ unlink(name); + fiu_exit_on("jio/commit/pre_ok_free_tid"); free_tid(ts->fs, ts->id); } - /* mark the transaction as commited, _after_ it was removed */ + /* mark the transaction as committed, _after_ it was removed */ ts->flags = ts->flags | J_COMMITTED; @@ -454,12 +480,11 @@ unlink_exit: exit: pthread_mutex_unlock(&(ts->lock)); - /* return the length only if it was properly commited */ + /* return the length only if it was properly committed */ if (ts->flags & J_COMMITTED) return written; else return -1; - } /* rollback a transaction */ @@ -471,6 +496,7 @@ ssize_t jtrans_rollback(struct jtrans *ts) jtrans_init(ts->fs, &newts); newts.flags = ts->flags; + newts.numops = ts->numops; if (ts->op == NULL || ts->flags & J_NOROLLBACK) { rv = -1; @@ -672,6 +698,7 @@ int jsync(struct jfs *fs) pthread_mutex_lock(&(fs->ltlock)); while (fs->ltrans != NULL) { free_tid(fs, fs->ltrans->id); + fiu_exit_on("jio/jsync/pre_unlink"); unlink(fs->ltrans->name); free(fs->ltrans->name); diff --git a/unix.c b/libjio/unix.c similarity index 100% rename from unix.c rename to libjio/unix.c diff --git a/samples/Makefile b/samples/Makefile new file mode 100644 index 0000000..fd3d1f4 --- /dev/null +++ b/samples/Makefile @@ -0,0 +1,22 @@ + +CFLAGS := -Wall -O3 -D_XOPEN_SOURCE=500 \ + $(shell getconf LFS_CFLAGS 2>/dev/null) +LIBS = -ljio + +BINS = full jio1 jio2 jio3 + +default: all + +all: $(BINS) + +$(BINS): + $(CC) $(CFLAGS) $(LIBS) $@.c -o $@ + +clean: + rm -f $(BINS) + rm -f *.bb *.bbg *.da *.gcov gmon.out + rm -f test1 test2 test3 + rm -rf .test1.jio .test2.jio .test3.jio + +.PHONY: default all clean + diff --git a/samples/build b/samples/build deleted file mode 100755 index f846eda..0000000 --- a/samples/build +++ /dev/null @@ -1,5 +0,0 @@ -gcc -Wall -O6 -ljio -lpthread jio1.c -o jio1 -gcc -Wall -O6 -ljio -lpthread jio2.c -o jio2 -gcc -Wall -O6 -ljio -lpthread jio3.c -o jio3 -gcc -Wall -O6 -ljio full.c -o full - diff --git a/samples/clean b/samples/clean deleted file mode 100755 index b5715c8..0000000 --- a/samples/clean +++ /dev/null @@ -1,2 +0,0 @@ -rm -rf jio1 jio2 jio3 test1 .test1.jio - diff --git a/samples/full.c b/samples/full.c index 90a009e..83e069c 100644 --- a/samples/full.c +++ b/samples/full.c @@ -17,11 +17,11 @@ int main(void) struct jfsck_result result; /* check the file is OK */ - jfsck(FILENAME, &result); - jfsck_cleanup(FILENAME); + jfsck(FILENAME, NULL, &result); + jfsck_cleanup(FILENAME, NULL); /* and open it */ - r = jopen(&file, FILENAME, O_SYNC | O_CREAT | O_TRUNC, 0600, 0); + r = jopen(&file, FILENAME, O_RDWR | O_CREAT | O_TRUNC, 0600, 0); if (r < 0) { perror("jopen"); return 1; diff --git a/samples/jio1.c b/samples/jio1.c index 4296ac6..9fb4062 100644 --- a/samples/jio1.c +++ b/samples/jio1.c @@ -1,5 +1,4 @@ - #include #include #include @@ -9,43 +8,42 @@ #include -#define str "TESTTESTTEST1234\n" +#define STR "TESTTESTTEST1234\n" -int jio(void) +static int jio(void) { int fd, rv; struct jfs fs; - fd = jopen(&fs, "test1", O_RDWR | O_CREAT | O_TRUNC | O_SYNC, 0660, 0); + fd = jopen(&fs, "test1", O_RDWR | O_CREAT | O_TRUNC, 0660, 0); if (fd < 0) - perror("OPEN"); + perror("jopen()"); - rv = jwrite(&fs, str, strlen(str)); - if (rv != strlen(str)) - perror("WRITE"); + rv = jwrite(&fs, STR, strlen(STR)); + if (rv != strlen(STR)) + perror("jwrite()"); return 0; - } -int classic(void) +static int classic(void) { int fd, rv; - fd = open("test1", O_RDWR | O_CREAT | O_TRUNC | O_SYNC, 0660); + fd = open("test1", O_RDWR | O_CREAT | O_TRUNC, 0660); if (fd < 0) - perror("OPEN"); + perror("open()"); - rv = write(fd, str, strlen(str)); - if (rv != strlen(str)) - perror("WRITE"); + rv = write(fd, STR, strlen(STR)); + if (rv != strlen(STR)) + perror("write()"); return 0; - } -int main(int argc, char **argv) { +int main(int argc, char **argv) +{ if (argc != 2) { printf("Use: jio1 [c|j]\n"); return 1; @@ -60,3 +58,4 @@ int main(int argc, char **argv) { return 0; } + diff --git a/samples/jio2.c b/samples/jio2.c index 9d6d008..3296dab 100644 --- a/samples/jio2.c +++ b/samples/jio2.c @@ -1,5 +1,4 @@ - #include #include #include @@ -11,62 +10,59 @@ #include -#define str "TESTTESTTEST1234\n" +#define STR "TESTTESTTEST1234\n" -int jio(void) +static int jio(void) { int fd, rv; struct jfs fs; - fd = jopen(&fs, "test1", O_RDWR | O_CREAT | O_TRUNC | O_SYNC, 0660, 0); + fd = jopen(&fs, "test2", O_RDWR | O_CREAT | O_TRUNC, 0660, 0); if (fd < 0) - perror("OPEN"); + perror("jopen()"); - rv = jwrite(&fs, str, strlen(str)); - if (rv != strlen(str)) - perror("WRITE"); + rv = jwrite(&fs, STR, strlen(STR)); + if (rv != strlen(STR)) + perror("jwrite()"); return 0; - } -int classic(void) +static int classic(void) { int fd, rv; - fd = open("test1", O_RDWR | O_CREAT | O_TRUNC | O_SYNC, 0660); + fd = open("test2", O_RDWR | O_CREAT | O_TRUNC, 0660); if (fd < 0) - perror("OPEN"); + perror("open()"); - rv = write(fd, str, strlen(str)); - if (rv != strlen(str)) - perror("WRITE"); + rv = write(fd, STR, strlen(STR)); + if (rv != strlen(STR)) + perror("write()"); return 0; - } -int main(int argc, char **argv) { - int i; - int N; - - if (argc != 2) { - printf("Use: jio1 [c|j] N\n"); +int main(int argc, char **argv) +{ + int i, n; + + if (argc != 3) { + printf("Use: jio2 [c|j] N\n"); return 1; } - - N = 0; - N = atoi(argv[2]); + + n = atoi(argv[2]); if (*argv[1] == 'c') - for (i = 0; i < N; i++) + for (i = 0; i < n; i++) classic(); else if (*argv[1] == 'j') - for (i = 0; i < N; i++) + for (i = 0; i < n; i++) jio(); else - printf("Use: jio1 [c|j] N\n"); + printf("Use: jio2 [c|j] n\n"); return 0; } diff --git a/samples/jio3.c b/samples/jio3.c index b5a9f5d..d66ee8d 100644 --- a/samples/jio3.c +++ b/samples/jio3.c @@ -1,5 +1,4 @@ - #include #include #include @@ -16,9 +15,9 @@ int main(int argc, char **argv) struct jfs fs; struct jtrans ts; - fd = jopen(&fs, "test1", O_RDWR | O_CREAT | O_SYNC, 0660, 0); + fd = jopen(&fs, "test3", O_RDWR | O_CREAT, 0660, 0); if (fd < 0) - perror("OPEN"); + perror("jopen()"); jtrans_init(&fs, &ts); @@ -31,19 +30,16 @@ int main(int argc, char **argv) #define str3 "3ROLLBACKTEST3!\n" jtrans_add(&ts, str3, strlen(str3), strlen(str1) + strlen(str2)); - rv = jtrans_commit(&ts); if (rv != strlen(str1) + strlen(str2) + strlen(str3)) - perror("COMMIT"); - printf("COMMIT OK: %d\n", rv); - + perror("jtrans_commit()"); + printf("commit ok: %d\n", rv); rv = jtrans_rollback(&ts); if (rv < 0) - perror("ROLLBACK"); - printf("ROLLBACK OK: %d\n", rv); + perror("jtrans_rollback()"); + printf("rollback ok: %d\n", rv); return 0; - } diff --git a/tests/README b/tests/README deleted file mode 100644 index 9e5fd0a..0000000 --- a/tests/README +++ /dev/null @@ -1,10 +0,0 @@ - -Here you will find a small set of testing utilities for libjio that cover -performance testing and recovery testings. They're really simple so there's -not much documentation besides the code. - -Another really useful way of doing testing is using some of the well known -filesystem benchmarking applications and modify them to use libjio. I regulary -use dbench, fsx, tiobench and the tdb test suite, the patches are available on -the website. - diff --git a/tests/behaviour/README b/tests/behaviour/README new file mode 100644 index 0000000..e69380d --- /dev/null +++ b/tests/behaviour/README @@ -0,0 +1,15 @@ + +In this directory you'll find libjio's automated behaviour tests. + +They're split in three suites: normal, corruption and fi. The normal suite +tests normal, expected behaviour. The corruption suite checks how the library +behaves in presence of disk corruption. The fi suite uses libfiu to inject +faults in order to simulate unexpected interruptions (like power failures) and +checks how the library behaves in those cases. + +To run them, use "./runtests ". To run all tests, you can run +"./runtests all". + +Note that the corruption and fi suite depends on libfiu being installed, and +libjio having been built using FI=1. + diff --git a/tests/behaviour/runtests b/tests/behaviour/runtests new file mode 100755 index 0000000..645a3bf --- /dev/null +++ b/tests/behaviour/runtests @@ -0,0 +1,19 @@ +#!/usr/bin/env python + +import sys +import tf + +possible_tests = ('normal', 'corruption', 'fi') +if len(sys.argv) < 2 or sys.argv[1] not in possible_tests + ('all',): + print 'Usage: runtests', '|'.join(possible_tests + ('all',)) + sys.exit(1) + +if sys.argv[1] == 'all': + mnames = ('t_' + i for i in possible_tests) +else: + mnames = ('t_' + sys.argv[1],) + +for mn in mnames: + print '--', mn + tf.autorun(__import__(mn)) + diff --git a/tests/behaviour/t_corruption.py b/tests/behaviour/t_corruption.py new file mode 100644 index 0000000..35a1998 --- /dev/null +++ b/tests/behaviour/t_corruption.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python + +# Corruption tests using libfiu. libjio must have been built with libfiu +# enabled (using something like make FI=1) for them to work. + +from tf import * + +try: + import fiu +except ImportError: + print + print "Error: unable to load fiu module. Corruption tests need" + print "libfiu support. Please install libfiu and recompile libjio" + print "with FI=1. You can still run the other tests." + print + raise + + +def test_c01(): + "checksum (1 bit change)" + c = gencontent() + + def f1(f, jf): + fiu.enable("jio/commit/tf_sync") + jf.write(c) + + n = run_with_tmp(f1) + assert content(n) == '' + tc = open(transpath(n, 1)).read() + # flip just one bit of the first byte + tc = chr((ord(tc[0]) & 0xFE) | (~ ord(tc[0]) & 0x1) & 0xFF) + tc[1:] + open(transpath(n, 1), 'w').write(tc) + fsck_verify(n, corrupt = 1) + assert content(n) == '' + cleanup(n) + +def test_c02(): + "truncate trans" + c = gencontent() + + def f1(f, jf): + fiu.enable("jio/commit/tf_sync") + jf.write(c) + + n = run_with_tmp(f1) + assert content(n) == '' + tp = transpath(n, 1) + open(tp, 'r+').truncate(len(content(tp)) - 2) + fsck_verify(n, broken = 1) + assert content(n) == '' + cleanup(n) + +def test_c03(): + "op len too big" + c = gencontent(10) + + def f1(f, jf): + fiu.enable("jio/commit/tf_sync") + jf.write(c) + + n = run_with_tmp(f1) + assert content(n) == '' + + tf = TransFile(transpath(n, 1)) + tf.ops[0].tlen = 99 + tf.save() + fsck_verify(n, broken = 1) + assert content(n) == '' + cleanup(n) + +def test_c04(): + "op len too small" + c = gencontent(100) + + def f1(f, jf): + fiu.enable("jio/commit/tf_sync") + jf.write(c) + + n = run_with_tmp(f1) + assert content(n) == '' + + tf = TransFile(transpath(n, 1)) + tf.ops[0].tlen = 10 + tf.save() + fsck_verify(n, broken = 1) + assert content(n) == '' + cleanup(n) + + diff --git a/tests/behaviour/t_fi.py b/tests/behaviour/t_fi.py new file mode 100644 index 0000000..7490a08 --- /dev/null +++ b/tests/behaviour/t_fi.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python + +# General tests using libfiu. libjio must have been built with libfiu enabled +# (using something like make FI=1) for them to work. + +import struct +from tf import * +import libjio + +try: + import fiu +except ImportError: + print + print "Error: unable to load fiu module. Fault injection tests need" + print "libfiu support. Please install libfiu and recompile libjio" + print "with FI=1. You can still run the other tests." + print + raise + + +def test_f01(): + "fail jio/get_tid/overflow" + c = gencontent() + + def f1(f, jf): + jf.write(c) + fiu.enable("jio/get_tid/overflow") + try: + jf.write(c) + except IOError: + pass + + n = run_with_tmp(f1) + assert content(n) == c + assert struct.unpack("I", content(jiodir(n) + '/lock'))[0] == 0 + fsck_verify(n) + assert content(n) == c + cleanup(n) + +def test_f02(): + "fail jio/commit/created_tf" + c = gencontent() + + def f1(f, jf): + fiu.enable("jio/commit/created_tf") + jf.write(c) + + n = run_with_tmp(f1) + fsck_verify(n, broken = 1) + assert content(n) == '' + cleanup(n) + +def test_f03(): + "fail jio/commit/tf_header" + c = gencontent() + + def f1(f, jf): + fiu.enable("jio/commit/tf_header") + jf.write(c) + + n = run_with_tmp(f1) + assert content(n) == '' + fsck_verify(n, broken = 1) + assert content(n) == '' + cleanup(n) + +def test_f04(): + "fail jio/commit/tf_ophdr" + c = gencontent() + + def f1(f, jf): + fiu.enable_external("jio/commit/tf_ophdr", + gen_ret_after(1, 0, 1)) + t = jf.new_trans() + t.add(c, 0) + t.add(c, len(c) + 200) + t.commit() + + n = run_with_tmp(f1) + + assert len(content(transpath(n, 1))) == DHS + DOHS + len(c) + DOHS + assert content(n) == '' + fsck_verify(n, broken = 1) + assert content(n) == '' + cleanup(n) + +def test_f05(): + "fail jio/commit/tf_opdata" + c = gencontent() + + def f1(f, jf): + fiu.enable_external("jio/commit/tf_opdata", + gen_ret_after(1, 0, 1)) + t = jf.new_trans() + t.add(c, 0) + t.add(c, len(c) + 200) + t.commit() + + n = run_with_tmp(f1) + + assert len(content(transpath(n, 1))) == DHS + (DOHS + len(c)) * 2 + assert content(n) == '' + fsck_verify(n, broken = 1) + assert content(n) == '' + cleanup(n) + +def test_f06(): + "fail jio/commit/tf_data" + c = gencontent() + + def f1(f, jf): + fiu.enable("jio/commit/tf_data") + t = jf.new_trans() + t.add(c, 0) + t.add(c, len(c) + 200) + t.commit() + + n = run_with_tmp(f1) + + assert len(content(transpath(n, 1))) == DHS + (DOHS + len(c)) * 2 + assert content(n) == '' + fsck_verify(n, broken = 1) + assert content(n) == '' + cleanup(n) + +def test_f07(): + "fail jio/commit/tf_sync" + c = gencontent() + + def f1(f, jf): + fiu.enable("jio/commit/tf_sync") + jf.write(c) + + n = run_with_tmp(f1) + assert content(n) == '' + fsck_verify(n, reapplied = 1) + assert content(n) == c + cleanup(n) + +def test_f08(): + "fail jio/commit/wrote_op" + c = gencontent() + + def f1(f, jf): + fiu.enable("jio/commit/wrote_op") + t = jf.new_trans() + t.add(c, 0) + t.add(c, len(c) + 200) + t.commit() + + n = run_with_tmp(f1) + + assert content(n) == c + fsck_verify(n, reapplied = 1) + assert content(n) == c + '\0' * 200 + c + cleanup(n) + +def test_f09(): + "fail jio/commit/wrote_all_ops" + c = gencontent() + + def f1(f, jf): + fiu.enable("jio/commit/wrote_all_ops") + jf.write(c) + + n = run_with_tmp(f1) + assert content(n) == c + fsck_verify(n, reapplied = 1) + assert content(n) == c + cleanup(n) + +def test_f10(): + "fail jio/commit/pre_ok_free_tid" + c = gencontent() + + def f1(f, jf): + fiu.enable("jio/commit/pre_ok_free_tid") + jf.write(c) + + n = run_with_tmp(f1) + assert content(n) == c + assert struct.unpack("I", content(jiodir(n) + '/lock'))[0] == 1 + fsck_verify(n) + assert content(n) == c + assert not os.path.exists(jiodir(n)) + cleanup(n) + +def test_f11(): + "fail jio/commit/tf_sync in rollback" + c = gencontent() + + def f1(f, jf): + jf.write('x' * (80 + len(c))) + t = jf.new_trans() + t.add(c, 80) + t.commit() + assert content(f.name) == 'x' * 80 + c + fiu.enable("jio/commit/tf_sync") + t.rollback() + + n = run_with_tmp(f1) + + assert content(n) == 'x' * 80 + c + fsck_verify(n, reapplied = 1) + assert content(n) == 'x' * (80 + len(c)) + cleanup(n) + +def test_f12(): + "fail jio/jsync/pre_unlink" + c = gencontent() + + def f1(f, jf): + fiu.enable("jio/jsync/pre_unlink") + t = jf.new_trans() + t.add(c, 0) + t.commit() + jf.jsync() + + n = run_with_tmp(f1, libjio.J_LINGER) + + assert content(n) == c + fsck_verify(n, reapplied = 1) + assert content(n) == c + cleanup(n) + + diff --git a/tests/behaviour/t_normal.py b/tests/behaviour/t_normal.py new file mode 100644 index 0000000..520d3a4 --- /dev/null +++ b/tests/behaviour/t_normal.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python + +# Normal tests. + +import libjio +from tf import * + + +def test_n01(): + "open + close" + def f1(f, jf): + pass + + n = run_with_tmp(f1) + assert content(n) == '' + fsck_verify(n) + assert content(n) == '' + cleanup(n) + +def test_n02(): + "write + seek + read" + c = gencontent() + + def f1(f, jf): + jf.write(c) + jf.lseek(0, 0) + assert jf.read(len(c) * 2) == c + + n = run_with_tmp(f1) + assert content(n) == c + fsck_verify(n) + cleanup(n) + +def test_n03(): + "pwrite" + c = gencontent() + + def f1(f, jf): + jf.pwrite(c, 80) + + n = run_with_tmp(f1) + assert content(n) == '\0' * 80 + c + fsck_verify(n) + cleanup(n) + +def test_n04(): + "truncate" + def f1(f, jf): + jf.truncate(826) + + n = run_with_tmp(f1) + assert content(n) == '\0' * 826 + fsck_verify(n) + cleanup(n) + +def test_n05(): + "commit" + c = gencontent() + + def f1(f, jf): + t = jf.new_trans() + t.add(c, 80) + t.commit() + + n = run_with_tmp(f1) + assert content(n) == '\0' * 80 + c + fsck_verify(n) + cleanup(n) + +def test_n06(): + "empty, then rollback" + c = gencontent() + + def f1(f, jf): + t = jf.new_trans() + t.add(c, 80) + t.commit() + t.rollback() + + n = run_with_tmp(f1) + + # XXX: This is weird, because the file was empty at the beginning. + # However, making it go back to 0 is delicate and the current code + # doesn't implement it. It probably should. + assert content(n) == '\0' * 80 + fsck_verify(n) + cleanup(n) + +def test_n07(): + "extending, then rollback" + c1 = gencontent() + c2 = gencontent() + + def f1(f, jf): + jf.write(c1) + t = jf.new_trans() + t.add(c2, len(c1) - 973) + t.commit() + t.rollback() + + n = run_with_tmp(f1) + + assert content(n) == c1 + fsck_verify(n) + cleanup(n) + +def test_n08(): + "multiple overlapping ops" + c1 = gencontent(9345) + c2 = gencontent(len(c1)) + c3 = gencontent(len(c1)) + c4 = gencontent(len(c1)) + c5 = gencontent(len(c1)) + + def f1(f, jf): + jf.write(c1) + t = jf.new_trans() + t.add(c2, len(c1) - 973) + t.add(c3, len(c1) - 1041) + t.add(c4, len(c1) - 666) + t.add(c5, len(c1) - 3000) + t.commit() + + n = run_with_tmp(f1) + assert content(n) == c1[:-3000] + c5 + c4[- (- 666 + 3000):] + fsck_verify(n) + cleanup(n) + +def test_n09(): + "rollback multiple overlapping ops" + c1 = gencontent(9345) + c2 = gencontent(len(c1)) + c3 = gencontent(len(c1)) + c4 = gencontent(len(c1)) + c5 = gencontent(len(c1)) + + def f1(f, jf): + jf.write(c1) + t = jf.new_trans() + t.add(c2, len(c1) - 973) + t.add(c3, len(c1) - 1041) + t.add(c4, len(c1) - 666) + t.add(c5, len(c1) - 3000) + t.commit() + t.rollback() + + n = run_with_tmp(f1) + + assert content(n) == c1 + fsck_verify(n) + cleanup(n) + +def test_n10(): + "lingering transactions" + c = gencontent() + + def f1(f, jf): + t = jf.new_trans() + t.add(c, 0) + t.commit() + del t + assert content(f.name) == c + assert os.path.exists(transpath(f.name, 1)) + jf.jsync() + assert not os.path.exists(transpath(f.name, 1)) + + n = run_with_tmp(f1, libjio.J_LINGER) + + assert content(n) == c + fsck_verify(n) + cleanup(n) + + diff --git a/tests/behaviour/tf.py b/tests/behaviour/tf.py new file mode 100644 index 0000000..016fd1e --- /dev/null +++ b/tests/behaviour/tf.py @@ -0,0 +1,236 @@ + +""" +Our customized testing framework. + +While not as sophisticated as the unittest module, it's targeted to our +particular kind of tests. + +To that end, it has several simple but useful functions aimed to make tests +more easier to read and write. +""" + +import sys +import os +import time +import random +import struct +import libjio + + +# Useful constants, must match libjio.h +DHS = 12 # disk header size +DOHS = 16 # disk op header size + + +def tmppath(): + """Returns a temporary path. We could use os.tmpnam() if it didn't + print a warning, or os.tmpfile() if it allowed us to get its name. + Since we just need a stupid name, we got our own function. Truly a + shame. Yes, it's not safe; I know and I don't care.""" + tmpdir = os.environ.get('TMPDIR', '/tmp') + now = time.time() + now_s = str(int(now)) + now_f = str((now - int(now)) * 10000) + now_str = "%s.%s" % (now_s[-5:], now_f[:now_f.find('.')]) + return tmpdir + '/jiotest.%s.%s' % (now_str, os.getpid()) + + +def run_forked(f, *args, **kwargs): + """Runs the function in a different process.""" + pid = os.fork() + if pid == 0: + # child + f(*args, **kwargs) + sys.exit(0) + else: + # parent + id, status = os.waitpid(pid, 0) + if not os.WIFEXITED(status): + raise RuntimeError, (id, status) + +def forked(f): + "Decorator that makes the function run in a different process." + def newf(*args, **kwargs): + run_forked(f, *args, **kwargs) + return newf + + +def gencontent(size = 9377): + "Generates random content." + s = '' + a = "%.20f" % random.random() + while len(s) < size: + s += a + s = s[:size] + return s + +def content(path): + "Returns the content of the given path." + f = open(path) + return f.read() + + +def biopen(path, mode = 'w+', jflags = 0): + "Returns (open(path), libjio.open(path))." + if 'r' in mode: + flags = os.O_RDONLY + if '+' in mode: + flags = os.O_RDWR + elif 'w' in mode: + flags = os.O_RDWR + if '+' in mode: + flags = flags | os.O_CREAT | os.O_TRUNC + else: + raise RuntimeError + + return open(path, mode), libjio.open(path, flags, 0400, jflags) + +def bitmp(mode = 'w+', jflags = 0): + "Opens a temporary file with biopen()." + path = tmppath() + return biopen(path, mode, jflags) + + +def run_with_tmp(func, jflags = 0): + """Runs the given function, that takes a file and a jfile as + parameters, using a temporary file. Returns the path of the temporary + file. The function runs in a new process that exits afterwards.""" + f, jf = bitmp(jflags = jflags) + run_forked(func, f, jf) + return f.name + + +def jiodir(path): + return os.path.dirname(path) + '/.' + os.path.basename(path) + '.jio' + +def transpath(path, ntrans): + jpath = jiodir(path) + return jpath + '/' + str(ntrans) + +def fsck(path): + "Calls libjio's jfsck()." + res = libjio.jfsck(path) + return res + +def fsck_verify(n, **kwargs): + """Runs fsck(n), and verifies that the fsck result matches the given + values. The default is to match all elements except total to 0 (total + is calculated automatically from the sum of the others). Raises an + AssertionError if the given results were not the ones expected.""" + expected = { + 'invalid': 0, + 'broken': 0, + 'reapplied': 0, + 'corrupt': 0, + 'in_progress': 0, + 'apply_error': 0, + } + expected.update(kwargs) + expected['total'] = sum(expected.values()) + res = fsck(n) + + for k in expected: + if k not in res: + raise AssertionError, k + ' not in res' + if res[k] != expected[k]: + raise AssertionError, k + ' does not match: ' + \ + str(res) + + libjio.jfsck_cleanup(n) + +def cleanup(path): + """Unlinks the path and its temporary libjio directory. The libjio + directory must only have the 'lock' file in it.""" + os.unlink(path) + jpath = jiodir(path) + if os.path.isdir(jpath): + assert 'lock' in os.listdir(jpath) + os.unlink(jpath + '/lock') + os.rmdir(jpath) + + +class attrdict (dict): + def __getattr__(self, name): + return self[name] + + def __setattr__(self, name, value): + self[name] = value + + def __delattr__(self, name): + del self[name] + +class TransFile (object): + def __init__(self, path = ''): + self.id = -1 + self.flags = 0 + self.numops = -1 + self.ops = [] + self.path = path + if path: + self.load() + + def load(self): + fd = open(self.path) + + # header + hdrfmt = "III" + self.id, self.flags, self.numops = struct.unpack(hdrfmt, + fd.read(struct.calcsize(hdrfmt))) + + # operations (header only) + opfmt = "IIQ" + self.ops = [] + for i in range(self.numops): + tlen, plen, offset = struct.unpack(opfmt, + fd.read(struct.calcsize(opfmt))) + payload = fd.read(tlen) + assert len(payload) == tlen + self.ops.append(attrdict(tlen = tlen, plen = plen, + offset = offset, payload = payload)) + + def save(self): + # the lack of integrity checking in this function is + # intentional, so we can write broken transactions and see how + # jfsck() copes with them + fd = open(self.path, 'w') + fd.write(struct.pack("III", self.id, self.flags, self.numops)) + for o in self.ops: + fd.write(struct.pack("IIQs", o.tlen, o.plen, o.offset, + o.payload)) + + def __repr__(self): + return '' % \ + (self.path, self.id, hex(self.flags), self.numops, + self.ops) + + +def gen_ret_after(n, notyet, itstime): + """Returns a function that returns value of notyet the first n + invocations, and itstime afterwards.""" + holder = [n] + def newf(*args, **kwargs): + holder[0] -= 1 + if holder[0] >= 0: + return notyet + return itstime + return newf + + +def autorun(module): + "Runs all the functions in the given module that begin with 'test'." + for name in sorted(dir(module)): + if not name.startswith('test'): + continue + + obj = getattr(module, name) + if '__call__' in dir(obj): + name = name[len('test'):] + if name.startswith('_'): + name = name[1:] + desc = '' + if obj.__doc__: + desc = obj.__doc__ + print "%-10s %-60.60s" % (name, desc) + obj() + + diff --git a/tests/performance/Makefile b/tests/performance/Makefile index 36a953f..11ddd33 100644 --- a/tests/performance/Makefile +++ b/tests/performance/Makefile @@ -1,23 +1,23 @@ -LIBS = -ljio -lpthread -CFLAGS = -Wall -O6 +CFLAGS := -Wall -O3 -D_XOPEN_SOURCE=500 \ + $(shell getconf LFS_CFLAGS 2>/dev/null) +LIBS = -ljio default: all -all: parallel streaming +all: performance -parallell: parallel.o - $(CC) $(LIBS) parallel.o -o parallel - -streaming: streaming.o - $(CC) $(LIBS) streaming.o -o streaming +performance: performance.o + $(CC) $(LIBS) performance.o -o performance .c.o: $(CC) $(CFLAGS) -c $< -o $@ clean: - rm -f streaming.o streaming parallel.o parallel + rm -f performance.o performance rm -f *.bb *.bbg *.da *.gcov gmon.out + rm -f test_file + rm -rf .test_file.jio .PHONY: default all clean diff --git a/tests/performance/parallel.c b/tests/performance/parallel.c deleted file mode 100644 index 205bb90..0000000 --- a/tests/performance/parallel.c +++ /dev/null @@ -1,122 +0,0 @@ - -/* - * streaming.c - A program to test speed of parallel writes using libjio. - * Alberto Bertogli (albertito@blitiri.com.ar) - */ - -/* - * It creates a big file, extend it using truncate and fork N threads, which - * write the file in chunks (ie. if we have three threads, the first one - * writes the first 1/3rd of the file, and so on). - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#define FILENAME "test_file-parallel" - -/* Declare here what's shared among threads - * It's not the cleanest design ever, but let's face it, it's a simple - * benchmarking program, who cares? */ -struct jfs fs; -int blocksize, towrite, mb; - - -void help(void) -{ - printf("Use: parallel MBs_to_write_per_thread blocksize nthreads\n"); - exit(1); -} - -void *worker(void *tno) -{ - void *buf; - int tid, work_done, rv; - off_t localoffset; - long secs, usecs; - double seconds, mb_per_sec; - struct timeval tv1, tv2; - - tid = (int) tno; - - localoffset = tid * towrite; - - buf = malloc(blocksize); - work_done = 0; - - gettimeofday(&tv1, NULL); - - while (work_done < towrite) { - rv = jpwrite(&fs, buf, blocksize, localoffset + work_done ); - if (rv != blocksize) { - perror("jpwrite:"); - break; - } - - work_done += blocksize; - } - - gettimeofday(&tv2, NULL); - - secs = tv2.tv_sec - tv1.tv_sec; - usecs = tv2.tv_usec - tv1.tv_usec; - - if (usecs < 0) { - secs -= 1; - usecs = 1000000 + usecs; - } - - seconds = secs + (usecs / 1000000.0); - mb_per_sec = mb / seconds; - - printf("%d %d %d %f %f\n", tid, mb, blocksize, seconds, mb_per_sec); - - return NULL; - -} - -int main(int argc, char **argv) -{ - int rv, nthreads, i; - pthread_t *threads; - - if (argc != 4) - help(); - - mb = atoi(argv[1]); - blocksize = atoi(argv[2]); - nthreads = atoi(argv[3]); - towrite = mb * 1024 * 1024; - - threads = malloc(sizeof(pthread_t) * nthreads); - - - rv = jopen(&fs, FILENAME, O_RDWR | O_CREAT | O_SYNC | O_TRUNC, - 0600, 0); - if (rv < 0) { - perror("jopen():"); - exit(1); - } - - /* extend the file */ - jtruncate(&fs, towrite * nthreads); - - /* start the threads */ - for (i = 0; i < nthreads; i++) { - pthread_create(threads + i, NULL, &worker, (void *) i); - } - - for (i = 0; i < nthreads; i++) { - pthread_join(*(threads + i), NULL); - } - - jclose(&fs); - return 0; -} - diff --git a/tests/performance/performance.c b/tests/performance/performance.c new file mode 100644 index 0000000..849cd60 --- /dev/null +++ b/tests/performance/performance.c @@ -0,0 +1,141 @@ + +/* + * performance.c - A program to test speed of parallel writes using libjio. + * Alberto Bertogli (albertito@blitiri.com.ar) + * + * It creates a big file, extends it using truncate, and forks N threads which + * write the file in chunks (ie. if we have three threads, the first one + * writes the first 1/3rd of the file, and so on). + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define FILENAME "test_file" + +/* These are shared among threads, to make the code simpler */ +static struct jfs fs; +static unsigned long mb; +static ssize_t blocksize, towrite; + + +static void help(void) +{ + printf("Use: performance towrite blocksize nthreads\n"); + printf("\n"); + printf(" - towrite: how many MB to write per thread\n"); + printf(" - blocksize: size of blocks written, in KB\n"); + printf(" - nthreads: number of threads to use\n"); +} + +static void *worker(void *tno) +{ + void *buf; + unsigned long tid; + ssize_t work_done, rv; + off_t localoffset; + long secs, usecs; + double seconds, mb_per_sec; + struct timeval tv1, tv2; + + tid = (unsigned long) tno; + + localoffset = tid * towrite; + + buf = malloc(blocksize); + if (buf == NULL) { + perror("malloc()"); + return NULL; + } + + work_done = 0; + + gettimeofday(&tv1, NULL); + + while (work_done < towrite) { + rv = jpwrite(&fs, buf, blocksize, localoffset + work_done); + if (rv != blocksize) { + perror("jpwrite()"); + break; + } + + work_done += blocksize; + } + + gettimeofday(&tv2, NULL); + + secs = tv2.tv_sec - tv1.tv_sec; + usecs = tv2.tv_usec - tv1.tv_usec; + + if (usecs < 0) { + secs -= 1; + usecs = 1000000 + usecs; + } + + seconds = secs + (usecs / 1000000.0); + mb_per_sec = mb / seconds; + + printf("%lu %zd %zd %f %f\n", tid, mb, blocksize, seconds, mb_per_sec); + + return NULL; +} + +int main(int argc, char **argv) +{ + int rv, nthreads; + unsigned long i; + pthread_t *threads; + struct jfsck_result ckres; + + if (argc != 4) { + help(); + return 1; + } + + mb = atoi(argv[1]); + blocksize = atoi(argv[2]) * 1024; + nthreads = atoi(argv[3]); + towrite = mb * 1024 * 1024; + + threads = malloc(sizeof(pthread_t) * nthreads); + if (threads == NULL) { + perror("malloc()"); + return 1; + } + + rv = jopen(&fs, FILENAME, O_RDWR | O_CREAT | O_TRUNC, 0600, 0); + if (rv < 0) { + perror("jopen()"); + return 1; + } + + jtruncate(&fs, towrite * nthreads); + + for (i = 0; i < nthreads; i++) { + pthread_create(threads + i, NULL, &worker, (void *) i); + } + + for (i = 0; i < nthreads; i++) { + pthread_join(*(threads + i), NULL); + } + + jclose(&fs); + jfsck(FILENAME, NULL, &ckres); + if (ckres.total != 0) { + fprintf(stderr, "There were %d errors during the test\n", + ckres.total); + fprintf(stderr, "jfsck() was used to fix them, but that"); + fprintf(stderr, "shouldn't happen.\n"); + return 1; + } + + jfsck_cleanup(FILENAME, NULL); + return 0; +} + diff --git a/tests/performance/streaming.c b/tests/performance/streaming.c deleted file mode 100644 index 211b388..0000000 --- a/tests/performance/streaming.c +++ /dev/null @@ -1,82 +0,0 @@ - -/* - * streaming.c - A program to test speed of a streaming write using libjio. - * Alberto Bertogli (albertito@blitiri.com.ar) - */ - - -#include -#include -#include -#include -#include -#include -#include -#include - -#define FILENAME "test_file-streaming" - - -void help(void) -{ - printf("Use: streaming MBs_to_write blocksize\n"); - exit(1); -} - - -int main(int argc, char **argv) -{ - int towrite, blocksize, rv, mb; - long secs, usecs; - double seconds, mb_per_sec; - void *buf; - struct jfs fs; - struct timeval tv1, tv2; - - if (argc != 3) - help(); - - mb = atoi(argv[1]); - towrite = mb * 1024 * 1024; - blocksize = atoi(argv[2]); - - rv = jopen(&fs, FILENAME, O_RDWR | O_CREAT | O_SYNC | O_TRUNC, - 0600, 0); - if (rv < 0) { - perror("jopen():"); - exit(1); - } - - buf = malloc(blocksize); - - gettimeofday(&tv1, NULL); - - while (towrite > 0) { - rv = jwrite(&fs, buf, blocksize); - if (rv != blocksize) { - perror("jwrite:"); - break; - } - - towrite -= blocksize; - } - - gettimeofday(&tv2, NULL); - - secs = tv2.tv_sec - tv1.tv_sec; - usecs = tv2.tv_usec - tv1.tv_usec; - - if (usecs < 0) { - secs -= 1; - usecs = 1000000 + usecs; - } - - seconds = secs + (usecs / 1000000.0); - mb_per_sec = mb / seconds; - - printf("%d %d %f %f\n", mb, blocksize, seconds, mb_per_sec); - - jclose(&fs); - return 0; -} - diff --git a/tests/recovery/.1.jio/2 b/tests/recovery/.1.jio/2 deleted file mode 100644 index 4917df2..0000000 Binary files a/tests/recovery/.1.jio/2 and /dev/null differ diff --git a/tests/recovery/.1.jio/3 b/tests/recovery/.1.jio/3 deleted file mode 100644 index dba9ab9..0000000 Binary files a/tests/recovery/.1.jio/3 and /dev/null differ diff --git a/tests/recovery/.1.jio/4 b/tests/recovery/.1.jio/4 deleted file mode 100644 index e1a12b8..0000000 --- a/tests/recovery/.1.jio/4 +++ /dev/null @@ -1 +0,0 @@ -,*6@gi*7!2 󯩄 \ No newline at end of file diff --git a/tests/recovery/.1.jio/5 b/tests/recovery/.1.jio/5 deleted file mode 100644 index a781fd3..0000000 --- a/tests/recovery/.1.jio/5 +++ /dev/null @@ -1 +0,0 @@ -XUXP_,`\ߋ(R*C \ No newline at end of file diff --git a/tests/recovery/.1.jio/6 b/tests/recovery/.1.jio/6 deleted file mode 100644 index bd9a8b2..0000000 --- a/tests/recovery/.1.jio/6 +++ /dev/null @@ -1 +0,0 @@ -Ibǁ|X<[FH,긭Z K<\ \ No newline at end of file diff --git a/tests/recovery/.1.jio/7 b/tests/recovery/.1.jio/7 deleted file mode 100644 index 5733512..0000000 Binary files a/tests/recovery/.1.jio/7 and /dev/null differ diff --git a/tests/recovery/.1.jio/desc b/tests/recovery/.1.jio/desc deleted file mode 100644 index 7f6996f..0000000 --- a/tests/recovery/.1.jio/desc +++ /dev/null @@ -1,9 +0,0 @@ - -1 valid Rollbacked -2 pdata too big Broken body -3 udata too big, pdata too small Broken body -4 random data, short header Broken header -5 random data, header lenght ok Broken body -6 random data, long file Broken body -7 valid, shorten the file via ftruncate Rollbacked - diff --git a/tests/recovery/.1.jio/lock b/tests/recovery/.1.jio/lock deleted file mode 100644 index 593f470..0000000 Binary files a/tests/recovery/.1.jio/lock and /dev/null differ diff --git a/tests/recovery/1 b/tests/recovery/1 deleted file mode 100644 index 8b450bf..0000000 --- a/tests/recovery/1 +++ /dev/null @@ -1 +0,0 @@ -smaller \ No newline at end of file diff --git a/tests/recovery/mklock b/tests/recovery/mklock deleted file mode 100644 index d0ef9e3..0000000 --- a/tests/recovery/mklock +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python - -"Generate a lock file" - -import sys -import struct - -if sys.argv < 3: - print "Use: mklock filename number" - sys.exit(1) - -try: - n = int(sys.argv[2]) -except: - print "Use: mlock filename number" - sys.exit(1) - -fd = open(sys.argv[1], "w"); - -s = struct.pack("I", n) - -fd.write(s) - -fd.close() - diff --git a/tests/recovery/mktrans b/tests/recovery/mktrans deleted file mode 100644 index 1fa6894..0000000 --- a/tests/recovery/mktrans +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python - -"Generate a transaction file" - -import sys -import struct - -def help(): - print "Use: mktrans tfile tid flags len plen ulen offset udata pdata" - sys.exit(1) - - -if sys.argv < 10: - help() - -try: - file = sys.argv[1] - tid = int(sys.argv[2]) - flags = int(sys.argv[3]) - dlen = int(sys.argv[4]) - plen = int(sys.argv[5]) - ulen = int(sys.argv[6]) - offset = int(sys.argv[7]) - udata = sys.argv[8] - pdata = sys.argv[9] -except: - help() - -fd = open(file, "w"); - -s = struct.pack("IIIIIQ%ds%ds" % (len(udata), len(pdata)),\ - tid, flags, dlen, plen, ulen, offset, \ - udata, pdata) - -print 'tid:', tid -print 'flags:', flags -print 'dlen:', dlen -print 'plen:', plen -print 'ulen:', ulen -print 'offset:', offset -print 'udata: +%s+' % udata -print 'pdata: +%s+' % pdata -print 'total lenght:', len(s) - -fd.write(s) - -fd.close() - diff --git a/tests/stress/jiostress b/tests/stress/jiostress new file mode 100755 index 0000000..ca4a83a --- /dev/null +++ b/tests/stress/jiostress @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 + +""" +This application is a stress tester for libjio. It's not a traditional stress +test like fsx (which can be used to test libjio using the preloading library), +but uses fault injection to check how the library behaves under random +failures. +""" + +import sys +import os +import random +import traceback +import libjio + +try: + import fiu +except ImportError: + print() + print("Error: unable to load fiu module. This test needs libfiu") + print("support. Please install libfiu and recompile libjio with FI=1.") + print() + raise + +# +# Auxiliary stuff +# + +randsrc = open('/dev/urandom', mode = 'rb') +def randbytes(n): + "Returns n random bytes" + if n < 64: + return randsrc.read(n) + + # to avoid reading too much from urandom (which needlessly stresses + # the kernel), just get a small random stream and repeat it + s = randsrc.read(64) + while len(s) < n: + s += s + s[::-1] + + return s[:n] + +def randfrange(maxend, maxsize): + start = random.randint(0, maxend - 1) + size = random.randint(0, (maxend - 1) - start) % maxsize + return start, start + size + +class ConsistencyError (Exception): + pass + + +# +# The test itself +# + +class Stresser: + def __init__(self, fname, fsize, nops): + self.fname = fname + self.fsize = fsize + self.nops = nops + + self.maxoplen = min(int(self.fsize / 4), + 5 * 1024 * 1024) + + self.jf = libjio.open(fname, + libjio.O_RDWR | libjio.O_CREAT, 0o600) + self.f = open(fname, mode = 'rb') + + # data used for consistency checks + self.current_range = (0, 0) + self.prev_data = b"" + self.new_data = b"" + + def pread(self, start, end): + ppos = self.f.tell() + self.f.seek(start, 0) + r = self.f.read(end - start) + self.f.seek(ppos, 0) + return r + + def randwrite(self): + pid = os.fork() + if pid == 0: + # child + try: + start, end = randfrange(self.fsize, + self.maxoplen) + + # read an extended range so we can check we + # only wrote what we were supposed to + estart = max(0, start - 32) + eend = min(self.fsize, end + 32) + self.current_range = (estart, eend) + self.prev_data = self.pread(estart, eend) + + nd = randbytes(end - start) + self.new_data = self.prev_data[estart:start] \ + + nd + self.prev_data[end:eend] + self.jf.pwrite(nd, start) + except IOError: + sys.exit(1) + except: + traceback.print_exc() + sys.exit(1) + sys.exit(0) + else: + # parent + id, status = os.waitpid(pid, 0) + if not os.WIFEXITED(status): + raise RuntimeError(status) + + if os.WEXITSTATUS(status) != 0: + return False + return True + + def verify(self): + # NOTE: must not use self.jf + real_data = self.pread(self.current_range[0], + self.current_range[1]) + if real_data not in (self.prev_data, self.new_data): + raise ConsistencyError + + def reopen(self): + self.jf = None + r = libjio.jfsck(self.fname) + self.verify() + libjio.jfsck_cleanup(self.fname) + + self.jf = libjio.open(self.fname, + libjio.O_RDWR | libjio.O_CREAT, 0o600) + return r + + def fiu_enable(self): + fiu.enable_random('jio/*', probability = 0.02) + + def fiu_disable(self): + fiu.disable('jio/*') + + def run(self): + self.fiu_enable() + nfailures = 0 + sys.stdout.write(" ") + for i in range(1, self.nops + 1): + sys.stdout.write(".") + if i % 10 == 0: + sys.stdout.write(" ") + if i % 50 == 0: + sys.stdout.write(" %d\n" % i) + sys.stdout.write(" ") + sys.stdout.flush() + if not self.randwrite(): + nfailures += 1 + self.fiu_disable() + r = self.reopen() + assert r['total'] <= 1 + self.fiu_enable() + self.verify() + sys.stdout.write("\n") + sys.stdout.flush() + self.fiu_disable() + return nfailures + + +# +# Main +# + +def usage(): + print(""" +Use: jiostress [] + +If the number of operations is not provided, the default (1000) will be +used.""") + + +def main(): + try: + fname = sys.argv[1] + fsize = int(sys.argv[2]) * 1024 * 1024 + nops = 1000 + if len(sys.argv) >= 4: + nops = int(sys.argv[3]) + except: + usage() + sys.exit(1) + + s = Stresser(fname, fsize, nops) + print("Running stress test") + nfailures = s.run() + print("Stress test completed") + print(" %d operations" % nops) + print(" %d simulated failures" % nfailures) + + r = libjio.jfsck(fname) + assert r['total'] == 0 + libjio.jfsck_cleanup(fname) + print("Final check completed") + #os.unlink(fname) + + +if __name__ == '__main__': + main() +