.gitignore | 4 + Makefile | 11 +- UPGRADING | 13 + bindings/preload/libjio_preload.c | 54 +-- bindings/{python3 => python}/libjio.c | 254 +++++++--- bindings/python/setup.py | 23 + bindings/python2/libjio.c | 696 ------------------------- bindings/python2/setup.py | 16 - bindings/python3/setup.py | 16 - doc/guide.rst | 156 +++---- doc/libjio.rst | 5 - libjio/Makefile | 10 +- libjio/ansi.c | 61 +-- libjio/autosync.c | 140 +++++ libjio/check.c | 128 +++--- libjio/checksum.c | 7 +- libjio/common.c | 17 +- libjio/common.h | 49 ++- libjio/compat.c | 59 +++ libjio/compat.h | 10 +- libjio/doxygen/Doxyfile.base | 204 ++++++++ libjio/doxygen/Doxyfile.internal | 8 + libjio/doxygen/Doxyfile.public | 8 + libjio/doxygen/Makefile | 24 + libjio/doxygen/groups.doxy | 13 + libjio/jiofsck.c | 23 +- libjio/journal.c | 343 ++++++++++++ libjio/journal.h | 24 + libjio/libjio.3 | 181 ++++---- libjio/libjio.h | 548 +++++++++++++++----- libjio/trans.c | 399 ++++---------- libjio/trans.h | 75 +++ libjio/unix.c | 91 +++-- samples/full.c | 25 +- samples/jio1.c | 13 +- samples/jio2.c | 13 +- samples/jio3.c | 29 +- tests/behaviour/runtests | 9 +- tests/behaviour/tf.py | 8 +- tests/performance/Makefile | 6 +- tests/performance/performance.c | 23 +- tests/performance/{performance.c => random.c} | 30 +- tests/stress/jiostress | 112 +++-- 43 files changed, 2234 insertions(+), 1704 deletions(-) diff --git a/.gitignore b/.gitignore index 2f09c27..c009b08 100644 --- a/.gitignore +++ b/.gitignore @@ -8,5 +8,9 @@ samples/jio1 samples/jio2 samples/jio3 tests/performance/performance +tests/performance/random *.pyc *.pyo +libjio/doxygen/doc.internal +libjio/doxygen/doc.public + diff --git a/Makefile b/Makefile index 3c32641..a810bc8 100644 --- a/Makefile +++ b/Makefile @@ -12,16 +12,16 @@ install: python2: - cd bindings/python2 && python setup.py build + cd bindings/python && python setup.py build python2_install: python2 - cd bindings/python2 && python setup.py install + cd bindings/python && python setup.py install python3: - cd bindings/python3 && python3 setup.py build + cd bindings/python && python3 setup.py build python3_install: python3 - cd bindings/python3 && python3 setup.py install + cd bindings/python && python3 setup.py install preload: @@ -34,8 +34,7 @@ preload_install: preload clean: $(MAKE) -C libjio/ clean $(MAKE) -C bindings/preload clean - rm -rf bindings/python2/build/ - rm -rf bindings/python3/build/ + rm -rf bindings/python/build/ .PHONY: default all libjio install \ diff --git a/UPGRADING b/UPGRADING index e5a4201..dc7ded7 100644 --- a/UPGRADING +++ b/UPGRADING @@ -6,6 +6,19 @@ You should always clean all your files before upgrading. While I don't expect the transaction on-disk format to change, it's a good practise and it doesn't take much effort. When it's mandatory, it will be noted. +-> 0.50 (Big API change) + - Structures are now opaque types: + struct jfs -> jfs_t; jopen() returns a pointer to one, jclose() frees it. + struct jtrans -> jtrans_t; jtrans_new() returns a pointer to one, + jtrans_free() frees it. + - Renamed jtrans_init() to jtrans_new(). + - jtrans_commit() returns -1 on recovered errors, -2 on unrecovered errors + (which are an indication of a severe underlying condition). + - jtrans_add() returns 0 on success and -1 on errors (it used to return 1 on + success and 0 on errors). + - jfsck() now has an additional "flags" parameter, which should be set to 0 + to select the default behaviour. + - jfsck_cleanup() was removed, and now jfsck() cleans up by default. -> 0.25 - It is no longer necessary to pass O_SYNC to jopen() if lingering diff --git a/bindings/preload/libjio_preload.c b/bindings/preload/libjio_preload.c index d71aa36..ab61434 100644 --- a/bindings/preload/libjio_preload.c +++ b/bindings/preload/libjio_preload.c @@ -66,7 +66,7 @@ static int (*c_dup2)(int oldfd, int newfd); struct fd_entry { int fd; unsigned int *refcount; - struct jfs *fs; + jfs_t *fs; pthread_mutex_t lock; }; static struct fd_entry fd_table[MAXFD]; @@ -200,7 +200,7 @@ static int __attribute__((constructor)) init(void) int open(const char *pathname, int flags, ...) { int r, fd; - struct jfs *fs; + jfs_t *fs; mode_t mode; struct stat st; va_list l; @@ -239,25 +239,14 @@ int open(const char *pathname, int flags, ...) } rec_inc(); - fs = malloc(sizeof(struct jfs)); + fs = jopen(pathname, flags, mode, 0); if (fs == NULL) { rec_dec(); return -1; } - fd = jopen(fs, pathname, flags, mode, 0); - if (fd >= MAXFD) { - printd("too many open fds: %d\n", fd); - jclose(fs); - free(fs); - rec_dec(); - return -1; - } rec_dec(); - if (fd < 0) { - printd("return %d\n", fd); - return fd; - } + fd = jfileno(fs); fd_lock(fd); fd_table[fd].fd = fd; @@ -274,7 +263,7 @@ int open(const char *pathname, int flags, ...) int open64(const char *pathname, int flags, ...) { int r, fd; - struct jfs *fs; + jfs_t *fs; mode_t mode; struct stat st; va_list l; @@ -313,25 +302,14 @@ int open64(const char *pathname, int flags, ...) } rec_inc(); - fs = malloc(sizeof(struct jfs)); + fs = jopen(pathname, flags, mode, 0); if (fs == NULL) { rec_dec(); return -1; } - fd = jopen(fs, pathname, flags, mode, 0); - if (fd >= MAXFD) { - printd("too many open fds: %d\n", fd); - jclose(fs); - free(fs); - rec_dec(); - return -1; - } rec_dec(); - if (fd < 0) { - printd("return %d\n", fd); - return fd; - } + fd = jfileno(fs); fd_lock(fd); fd_table[fd].fd = fd; @@ -366,13 +344,10 @@ int unlocked_close(int fd) r = jclose(fd_table[fd].fs); rec_dec(); - if (fd_table[fd].fs != NULL) { - fd_table[fd].fd = -1; - free(fd_table[fd].refcount); - fd_table[fd].refcount = NULL; - free(fd_table[fd].fs); - fd_table[fd].fs = NULL; - } + fd_table[fd].fd = -1; + free(fd_table[fd].refcount); + fd_table[fd].refcount = NULL; + fd_table[fd].fs = NULL; return r; } @@ -380,7 +355,7 @@ int unlocked_close(int fd) int close(int fd) { int r; - struct jfs *fs; + jfs_t *fs; if (called) { printd("orig\n"); @@ -410,6 +385,7 @@ int close(int fd) int unlink(const char *pathname) { int r; + struct jfsck_result res; if (called) { printd("orig\n"); @@ -419,7 +395,7 @@ int unlink(const char *pathname) printd("libjio\n"); rec_inc(); - jfsck_cleanup(pathname, NULL); + r = jfsck(pathname, NULL, &res, 0); rec_dec(); r = (*c_unlink)(pathname); @@ -519,7 +495,7 @@ int dup2(int oldfd, int newfd) rtype name DEF \ { \ rtype r; \ - struct jfs *fs; \ + jfs_t *fs; \ \ if (called) { \ printd("orig\n"); \ diff --git a/bindings/python3/libjio.c b/bindings/python/libjio.c similarity index 80% rename from bindings/python3/libjio.c rename to bindings/python/libjio.c index 7d63691..bd44992 100644 --- a/bindings/python3/libjio.c +++ b/bindings/python/libjio.c @@ -1,6 +1,6 @@ /* - * Python 3 bindings for libjio + * Python (2 and 3) bindings for libjio * Alberto Bertogli (albertito@blitiri.com.ar) */ @@ -27,9 +27,8 @@ * several operations that get added by its add() method. It gets committed * with commit(), and rolled back with rollback(). * - * There rest of the module's functions are related to file checking, called - * jfsck() and jfsck_cleanup(), which are just wrappers to the real C - * functions. + * There rest of the module's functions are related to file checking, at the + * moment only jfsck(), which is just a wrapper to the real C functions. */ /* @@ -39,7 +38,7 @@ /* jfile */ typedef struct { PyObject_HEAD - struct jfs *fs; + jfs_t *fs; } jfile_object; static PyTypeObject jfile_type; @@ -48,7 +47,7 @@ static PyTypeObject jfile_type; /* jtrans */ typedef struct { PyObject_HEAD - struct jtrans *ts; + jtrans_t *ts; jfile_object *jfile; } jtrans_object; @@ -64,7 +63,6 @@ static void jf_dealloc(jfile_object *fp) { if (fp->fs) { jclose(fp->fs); - free(fp->fs); } PyObject_Del(fp); } @@ -80,7 +78,7 @@ static PyObject *jf_fileno(jfile_object *fp, PyObject *args) if (!PyArg_ParseTuple(args, ":fileno")) return NULL; - return PyLong_FromLong(fp->fs->fd); + return PyLong_FromLong(jfileno(fp->fs)); } /* read */ @@ -112,7 +110,11 @@ static PyObject *jf_read(jfile_object *fp, PyObject *args) if (rv < 0) { r = PyErr_SetFromErrno(PyExc_IOError); } else { +#ifdef PYTHON3 r = PyBytes_FromStringAndSize((char *) buf, rv); +#elif PYTHON2 + r = PyString_FromStringAndSize((char *) buf, rv); +#endif } free(buf); @@ -149,7 +151,11 @@ static PyObject *jf_pread(jfile_object *fp, PyObject *args) if (rv < 0) { r = PyErr_SetFromErrno(PyExc_IOError); } else { +#ifdef PYTHON3 r = PyBytes_FromStringAndSize((char *) buf, rv); +#elif PYTHON2 + r = PyString_FromStringAndSize((char *) buf, rv); +#endif } free(buf); @@ -320,12 +326,61 @@ static PyObject *jf_jmove_journal(jfile_object *fp, PyObject *args) return PyLong_FromLong(rv); } +/* jfs_autosync_start() */ +PyDoc_STRVAR(jf_autosync_start__doc, +"autosync_start(max_sec, max_bytes)\n\ +\n\ +Starts the automatic sync thread (only useful when using lingering\n\ +transactions).\n"); + +static PyObject *jf_autosync_start(jfile_object *fp, PyObject *args) +{ + int rv; + unsigned int max_sec, max_bytes; + + if (!PyArg_ParseTuple(args, "II:autosync_start", &max_sec, + &max_bytes)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + rv = jfs_autosync_start(fp->fs, max_sec, max_bytes); + Py_END_ALLOW_THREADS + + if (rv != 0) + return PyErr_SetFromErrno(PyExc_IOError); + + return PyLong_FromLong(rv); +} + +/* jfs_autosync_stop() */ +PyDoc_STRVAR(jf_autosync_stop__doc, +"autosync_stop()\n\ +\n\ +Stops the automatic sync thread started by autosync_start()\n"); + +static PyObject *jf_autosync_stop(jfile_object *fp, PyObject *args) +{ + int rv; + + if (!PyArg_ParseTuple(args, ":autosync_stop")) + return NULL; + + Py_BEGIN_ALLOW_THREADS + rv = jfs_autosync_stop(fp->fs); + Py_END_ALLOW_THREADS + + if (rv != 0) + return PyErr_SetFromErrno(PyExc_IOError); + + return PyLong_FromLong(rv); +} + /* new_trans */ PyDoc_STRVAR(jf_new_trans__doc, "new_trans()\n\ \n\ Returns an object representing a new empty transaction.\n\ -It's a wrapper to jtrans_init().\n"); +It's a wrapper to jtrans_new().\n"); static PyObject *jf_new_trans(jfile_object *fp, PyObject *args) { @@ -334,11 +389,15 @@ static PyObject *jf_new_trans(jfile_object *fp, PyObject *args) if (!PyArg_ParseTuple(args, ":new_trans")) return NULL; +#ifdef PYTHON3 tp = (jtrans_object *) jtrans_type.tp_alloc(&jtrans_type, 0); +#elif PYTHON2 + tp = PyObject_New(jtrans_object, &jtrans_type); +#endif if (tp == NULL) return NULL; - tp->ts = malloc(sizeof(struct jtrans)); + tp->ts = jtrans_new(fp->fs); if(tp->ts == NULL) { return PyErr_NoMemory(); } @@ -347,8 +406,6 @@ static PyObject *jf_new_trans(jfile_object *fp, PyObject *args) tp->jfile = fp; Py_INCREF(fp); - jtrans_init(fp->fs, tp->ts); - return (PyObject *) tp; } @@ -365,11 +422,16 @@ static PyMethodDef jfile_methods[] = { { "jsync", (PyCFunction) jf_jsync, METH_VARARGS, jf_jsync__doc }, { "jmove_journal", (PyCFunction) jf_jmove_journal, METH_VARARGS, jf_jmove_journal__doc }, + { "autosync_start", (PyCFunction) jf_autosync_start, METH_VARARGS, + jf_autosync_start__doc }, + { "autosync_stop", (PyCFunction) jf_autosync_stop, METH_VARARGS, + jf_autosync_stop__doc }, { "new_trans", (PyCFunction) jf_new_trans, METH_VARARGS, jf_new_trans__doc }, { NULL } }; +#ifdef PYTHON3 static PyTypeObject jfile_type = { PyObject_HEAD_INIT(NULL) .tp_name = "libjio.jfile", @@ -378,6 +440,25 @@ static PyTypeObject jfile_type = { .tp_methods = jfile_methods, }; +#elif PYTHON2 +static PyObject *jf_getattr(jfile_object *fp, char *name) +{ + return Py_FindMethod(jfile_methods, (PyObject *)fp, name); +} + +static PyTypeObject jfile_type = { + PyObject_HEAD_INIT(NULL) + 0, + "libjio.jfile", + sizeof(jfile_object), + 0, + (destructor)jf_dealloc, + 0, + (getattrfunc)jf_getattr, +}; + +#endif + /* * The jtrans object @@ -388,7 +469,6 @@ static void jt_dealloc(jtrans_object *tp) { if (tp->ts != NULL) { jtrans_free(tp->ts); - free(tp->ts); } Py_DECREF(tp->jfile); PyObject_Del(tp); @@ -413,7 +493,7 @@ static PyObject *jt_add(jtrans_object *tp, PyObject *args) return NULL; rv = jtrans_add(tp->ts, buf, len, offset); - if (rv == 0) + if (rv < 0) return PyErr_SetFromErrno(PyExc_IOError); return PyLong_FromLong(rv); @@ -475,6 +555,7 @@ static PyMethodDef jtrans_methods[] = { { NULL } }; +#ifdef PYTHON3 static PyTypeObject jtrans_type = { PyObject_HEAD_INIT(NULL) .tp_name = "libjio.jtrans", @@ -483,6 +564,25 @@ static PyTypeObject jtrans_type = { .tp_methods = jtrans_methods, }; +#elif PYTHON2 +static PyObject *jt_getattr(jtrans_object *tp, char *name) +{ + return Py_FindMethod(jtrans_methods, (PyObject *)tp, name); +} + +static PyTypeObject jtrans_type = { + PyObject_HEAD_INIT(NULL) + 0, + "libjio.jtrans", + sizeof(jtrans_object), + 0, + (destructor)jt_dealloc, + 0, + (getattrfunc)jt_getattr, +}; + +#endif + /* * The module @@ -499,7 +599,6 @@ It's a wrapper to jopen().\n"); static PyObject *jf_open(PyObject *self, PyObject *args) { - int rv; char *file; int flags, mode, jflags; jfile_object *fp; @@ -512,23 +611,22 @@ static PyObject *jf_open(PyObject *self, PyObject *args) &jflags)) return NULL; +#ifdef PYTHON3 fp = (jfile_object *) jfile_type.tp_alloc(&jfile_type, 0); +#elif PYTHON2 + fp = PyObject_New(jfile_object, &jfile_type); +#endif + if (fp == NULL) return NULL; - fp->fs = malloc(sizeof(struct jfs)); + fp->fs = jopen(file, flags, mode, jflags); if (fp->fs == NULL) { - return PyErr_NoMemory(); - } - - rv = jopen(fp->fs, file, flags, mode, jflags); - if (rv < 0) { - free(fp->fs); return PyErr_SetFromErrno(PyExc_IOError); } if (PyErr_Occurred()) { - free(fp->fs); + jclose(fp->fs); return NULL; } @@ -537,23 +635,26 @@ static PyObject *jf_open(PyObject *self, PyObject *args) /* jfsck */ PyDoc_STRVAR(jf_jfsck__doc, -"jfsck(name[, jdir])\n\ +"jfsck(name[, jdir] [, flags])\n\ \n\ Checks the integrity of the file with the given name, using (optionally) jdir\n\ -as the journal directory; returns a dictionary with all the different values\n\ -of the check (equivalent to the 'struct jfsck_result'). If the path is\n\ -incorrect, or there is no journal associated with it, an IOError will be\n\ -raised.\n\ +as the journal directory and the given flags; returns a dictionary with all\n\ +the different values of the check (equivalent to the 'struct jfsck_result').\n\ +If the path is incorrect, or there is no journal associated with it, an\n\ +IOError will be raised.\n\ It's a wrapper to jfsck().\n"); -static PyObject *jf_jfsck(PyObject *self, PyObject *args) +static PyObject *jf_jfsck(PyObject *self, PyObject *args, PyObject *kw) { int rv; + unsigned int flags; char *name, *jdir = NULL; struct jfsck_result res; PyObject *dict; + char *keywords[] = { "name", "jdir", "flags", NULL }; - if (!PyArg_ParseTuple(args, "s|s:jfsck", &name, &jdir)) + if (!PyArg_ParseTupleAndKeywords(args, kw, "s|sI:jfsck", + keywords, &name, &jdir, &flags)) return NULL; dict = PyDict_New(); @@ -561,7 +662,7 @@ static PyObject *jf_jfsck(PyObject *self, PyObject *args) return PyErr_NoMemory(); Py_BEGIN_ALLOW_THREADS - rv = jfsck(name, jdir, &res); + rv = jfsck(name, jdir, &res, flags); Py_END_ALLOW_THREADS if (rv == J_ENOMEM) { @@ -584,40 +685,10 @@ static PyObject *jf_jfsck(PyObject *self, PyObject *args) return dict; } -/* jfsck_cleanup */ -PyDoc_STRVAR(jf_jfsck_cleanup__doc, -"jfsck_cleanup(name[, jdir])\n\ -\n\ -Clean the journal directory for the given file using (optionally) jdir as the\n\ -journal directory, and leave it ready to use.\n\ -It's a wrapper to jfsck_cleanup().\n"); - -static PyObject *jf_jfsck_cleanup(PyObject *self, PyObject *args) -{ - long rv; - char *name, *jdir = NULL; - - if (!PyArg_ParseTuple(args, "s|s:jfsck_cleanup", &name, &jdir)) - return NULL; - - Py_BEGIN_ALLOW_THREADS - rv = jfsck_cleanup(name, jdir); - Py_END_ALLOW_THREADS - - if (rv != 1) { - PyErr_SetObject(PyExc_IOError, PyLong_FromLong(rv)); - return NULL; - } - - return PyLong_FromLong(rv); -} - - static PyMethodDef module_methods[] = { { "open", jf_open, METH_VARARGS, jf_open__doc }, - { "jfsck", jf_jfsck, METH_VARARGS, jf_jfsck__doc }, - { "jfsck_cleanup", jf_jfsck_cleanup, METH_VARARGS, - jf_jfsck_cleanup__doc }, + { "jfsck", (PyCFunction) jf_jfsck, METH_VARARGS | METH_KEYWORDS, + jf_jfsck__doc }, { NULL, NULL, 0, NULL }, }; @@ -628,24 +699,10 @@ static PyMethodDef module_methods[] = { "on it.\n" \ "Please read the documentation for more information.\n" -static PyModuleDef libjio_module = { - PyModuleDef_HEAD_INIT, - .m_name = "libjio", - .m_doc = module_doc, - .m_size = -1, - .m_methods = module_methods, -}; -PyMODINIT_FUNC PyInit_libjio(void) +/* fills the module with the objects and constants */ +static void populate_module(PyObject *m) { - PyObject *m; - - if (PyType_Ready(&jfile_type) < 0 || - PyType_Ready(&jtrans_type) < 0) - return NULL; - - m = PyModule_Create(&libjio_module); - Py_INCREF(&jfile_type); PyModule_AddObject(m, "jfile", (PyObject *) &jfile_type); @@ -682,7 +739,46 @@ PyMODINIT_FUNC PyInit_libjio(void) PyModule_AddIntConstant(m, "SEEK_SET", SEEK_SET); PyModule_AddIntConstant(m, "SEEK_CUR", SEEK_CUR); PyModule_AddIntConstant(m, "SEEK_END", SEEK_END); +} + + +#ifdef PYTHON3 +static PyModuleDef libjio_module = { + PyModuleDef_HEAD_INIT, + .m_name = "libjio", + .m_doc = module_doc, + .m_size = -1, + .m_methods = module_methods, +}; + +PyMODINIT_FUNC PyInit_libjio(void) +{ + PyObject *m; + + if (PyType_Ready(&jfile_type) < 0 || + PyType_Ready(&jtrans_type) < 0) + return NULL; + + m = PyModule_Create(&libjio_module); + + populate_module(m); return m; } +#elif PYTHON2 +PyMODINIT_FUNC initlibjio(void) +{ + PyObject* m; + + jfile_type.ob_type = &PyType_Type; + jtrans_type.ob_type = &PyType_Type; + + m = Py_InitModule3("libjio", module_methods, module_doc); + + populate_module(m); +} + +#endif + + diff --git a/bindings/python/setup.py b/bindings/python/setup.py new file mode 100644 index 0000000..3b6de68 --- /dev/null +++ b/bindings/python/setup.py @@ -0,0 +1,23 @@ + +import sys +from distutils.core import setup, Extension + +if sys.version_info[0] == 2: + ver_define = ('PYTHON2', '1') +elif sys.version_info[0] == 3: + ver_define = ('PYTHON3', '1') + +libjio = Extension("libjio", + libraries = ['jio'], + sources = ['libjio.c'], + define_macros = [ver_define] ) + +setup( + name = 'libjio', + description = "A library for journaled I/O", + author="Alberto Bertogli", + author_email="albertito@blitiri.com.ar", + url="http://blitiri.com.ar/p/libjio", + ext_modules = [libjio] +) + diff --git a/bindings/python2/libjio.c b/bindings/python2/libjio.c deleted file mode 100644 index a29f8f1..0000000 --- a/bindings/python2/libjio.c +++ /dev/null @@ -1,696 +0,0 @@ - -/* - * Python bindings for libjio - * Alberto Bertogli (albertito@blitiri.com.ar) - */ - - -#include - -#include -#include -#include -#include -#include - -/* - * This module provides two classes and some functions. - * - * The classes are jfile (created with open()) and jtrans (created with - * jfile.new_trans()). - * - * The first one represents a journaled file where you operate using read(), - * write() and so on; to close it, just call del(). This is similar to the - * UNIX file. - * - * The second one represents a single transaction, which is composed of - * several operations that get added by its add() method. It gets committed - * with commit(), and rolled back with rollback(). - * - * There rest of the module's functions are related to file checking, called - * jfsck() and jfsck_cleanup(), which are just wrappers to the real C - * functions. - */ - -/* - * Type definitions - */ - -/* jfile */ -typedef struct { - PyObject_HEAD - struct jfs *fs; -} jfile_object; - -static PyTypeObject jfile_type; - -/* jtrans */ -typedef struct { - PyObject_HEAD - struct jtrans *ts; - jfile_object *jfile; -} jtrans_object; - -static PyTypeObject jtrans_type; - - -/* - * The jfile object - */ - -/* delete */ -static void jf_dealloc(jfile_object *fp) -{ - if (fp->fs) { - jclose(fp->fs); - free(fp->fs); - } - PyObject_Del(fp); -} - -/* fileno */ -PyDoc_STRVAR(jf_fileno__doc, -"fileno()\n\ -\n\ -Return the file descriptor number for the file.\n"); - -static PyObject *jf_fileno(jfile_object *fp, PyObject *args) -{ - if (!PyArg_ParseTuple(args, ":fileno")) - return NULL; - - return PyInt_FromLong(fp->fs->fd); -} - -/* read */ -PyDoc_STRVAR(jf_read__doc, -"read(size)\n\ -\n\ -Read at most size bytes from the file, returns the string with\n\ -the contents.\n\ -It's a wrapper to jread().\n"); - -static PyObject *jf_read(jfile_object *fp, PyObject *args) -{ - long rv; - long len; - unsigned char *buf; - PyObject *r; - - if (!PyArg_ParseTuple(args, "i:read", &len)) - return NULL; - - buf = malloc(len); - if (buf == NULL) - return PyErr_NoMemory(); - - Py_BEGIN_ALLOW_THREADS - rv = jread(fp->fs, buf, len); - Py_END_ALLOW_THREADS - - if (rv < 0) { - r = PyErr_SetFromErrno(PyExc_IOError); - } else { - r = PyString_FromStringAndSize((char *) buf, rv); - } - - free(buf); - return r; -} - -/* pread */ -PyDoc_STRVAR(jf_pread__doc, -"pread(size, offset)\n\ -\n\ -Read size bytes from the file at the given offset, return a string with the\n\ -contents.\n\ -It's a wrapper to jpread().\n"); - -static PyObject *jf_pread(jfile_object *fp, PyObject *args) -{ - long rv; - long len; - long long offset; - unsigned char *buf; - PyObject *r; - - if (!PyArg_ParseTuple(args, "iL:pread", &len, &offset)) - return NULL; - - buf = malloc(len); - if (buf == NULL) - return PyErr_NoMemory(); - - Py_BEGIN_ALLOW_THREADS - rv = jpread(fp->fs, buf, len, offset); - Py_END_ALLOW_THREADS - - if (rv < 0) { - r = PyErr_SetFromErrno(PyExc_IOError); - } else { - r = PyString_FromStringAndSize((char *) buf, rv); - } - - free(buf); - return r; -} - -/* write */ -PyDoc_STRVAR(jf_write__doc, -"write(buf)\n\ -\n\ -Write the contents of the given buffer (a string) to the file, returns the\n\ -number of bytes written.\n\ -It's a wrapper to jwrite().\n"); - -static PyObject *jf_write(jfile_object *fp, PyObject *args) -{ - long rv; - unsigned char *buf; - int len; - - if (!PyArg_ParseTuple(args, "s#:write", &buf, &len)) - return NULL; - - Py_BEGIN_ALLOW_THREADS - rv = jwrite(fp->fs, buf, len); - Py_END_ALLOW_THREADS - - if (rv < 0) - return PyErr_SetFromErrno(PyExc_IOError); - - return PyLong_FromLong(rv); -} - -/* pwrite */ -PyDoc_STRVAR(jf_pwrite__doc, -"pwrite(buf, offset)\n\ -\n\ -Write the contents of the given buffer (a string) to the file at the given\n\ -offset, returns the number of bytes written.\n\ -It's a wrapper to jpwrite().\n"); - -static PyObject *jf_pwrite(jfile_object *fp, PyObject *args) -{ - long rv; - unsigned char *buf; - long long offset; - int len; - - if (!PyArg_ParseTuple(args, "s#L:pwrite", &buf, &len, &offset)) - return NULL; - - Py_BEGIN_ALLOW_THREADS - rv = jpwrite(fp->fs, buf, len, offset); - Py_END_ALLOW_THREADS - - if (rv < 0) - return PyErr_SetFromErrno(PyExc_IOError); - - return PyLong_FromLong(rv); -} - -/* truncate */ -PyDoc_STRVAR(jf_truncate__doc, -"truncate(lenght)\n\ -\n\ -Truncate the file to the given size.\n\ -It's a wrapper to jtruncate().\n"); - -static PyObject *jf_truncate(jfile_object *fp, PyObject *args) -{ - int rv; - long long lenght; - - if (!PyArg_ParseTuple(args, "L:truncate", &lenght)) - return NULL; - - Py_BEGIN_ALLOW_THREADS - rv = jtruncate(fp->fs, lenght); - Py_END_ALLOW_THREADS - - if (rv != 0) - return PyErr_SetFromErrno(PyExc_IOError); - - return PyLong_FromLongLong(rv); -} - -/* lseek */ -PyDoc_STRVAR(jf_lseek__doc, -"lseek(offset, whence)\n\ -\n\ -Reposition the file pointer to the given offset, according to the directive\n\ -whence as follows:\n\ -SEEK_SET The offset is set relative to the beginning of the file.\n\ -SEEK_CUR The offset is set relative to the current position.\n\ -SEEK_END The offset is set relative to the end of the file.\n\ -\n\ -These constants are defined in the module. See lseek's manpage for more\n\ -information.\n\ -It's a wrapper to jlseek().\n"); - -static PyObject *jf_lseek(jfile_object *fp, PyObject *args) -{ - long long rv; - int whence; - long long offset; - - if (!PyArg_ParseTuple(args, "Li:lseek", &offset, &whence)) - return NULL; - - Py_BEGIN_ALLOW_THREADS - rv = jlseek(fp->fs, offset, whence); - Py_END_ALLOW_THREADS - - if (rv == -1) - return PyErr_SetFromErrno(PyExc_IOError); - - return PyLong_FromLongLong(rv); -} - -/* jsync */ -PyDoc_STRVAR(jf_jsync__doc, -"jsync()\n\ -\n\ -Used with lingering transactions, see the library documentation for more\n\ -detailed information.\n\ -It's a wrapper to jsync().\n"); - -static PyObject *jf_jsync(jfile_object *fp, PyObject *args) -{ - long rv; - - if (!PyArg_ParseTuple(args, ":jsync")) - return NULL; - - Py_BEGIN_ALLOW_THREADS - rv = jsync(fp->fs); - Py_END_ALLOW_THREADS - - if (rv < 0) - return PyErr_SetFromErrno(PyExc_IOError); - - return PyLong_FromLong(rv); -} - -/* jmove_journal */ -PyDoc_STRVAR(jf_jmove_journal__doc, -"jmove_journal(newpath)\n\ -\n\ -Moves the journal directory to the new path; note that there MUST NOT BE\n\ -anything else operating on the file.\n\ -It's a wrapper to jmove_journal().\n"); - -static PyObject *jf_jmove_journal(jfile_object *fp, PyObject *args) -{ - long rv; - char *newpath; - - if (!PyArg_ParseTuple(args, "s:jmove_journal", &newpath)) - return NULL; - - Py_BEGIN_ALLOW_THREADS - rv = jmove_journal(fp->fs, newpath); - Py_END_ALLOW_THREADS - - if (rv != 0) - return PyErr_SetFromErrno(PyExc_IOError); - - return PyLong_FromLong(rv); -} - -/* new_trans */ -PyDoc_STRVAR(jf_new_trans__doc, -"new_trans()\n\ -\n\ -Returns an object representing a new empty transaction.\n\ -It's a wrapper to jtrans_init().\n"); - -static PyObject *jf_new_trans(jfile_object *fp, PyObject *args) -{ - jtrans_object *tp; - - if (!PyArg_ParseTuple(args, ":new_trans")) - return NULL; - - tp = PyObject_New(jtrans_object, &jtrans_type); - if (tp == NULL) - return NULL; - - tp->ts = malloc(sizeof(struct jtrans)); - if(tp->ts == NULL) { - return PyErr_NoMemory(); - } - - /* increment the reference count, it's decremented on deletion */ - tp->jfile = fp; - Py_INCREF(fp); - - jtrans_init(fp->fs, tp->ts); - - return (PyObject *) tp; -} - - -/* method table */ -static PyMethodDef jfile_methods[] = { - { "fileno", (PyCFunction) jf_fileno, METH_VARARGS, jf_fileno__doc }, - { "read", (PyCFunction) jf_read, METH_VARARGS, jf_read__doc }, - { "pread", (PyCFunction) jf_pread, METH_VARARGS, jf_pread__doc }, - { "write", (PyCFunction) jf_write, METH_VARARGS, jf_write__doc }, - { "pwrite", (PyCFunction) jf_pwrite, METH_VARARGS, jf_pwrite__doc }, - { "truncate", (PyCFunction) jf_truncate, METH_VARARGS, - jf_truncate__doc }, - { "lseek", (PyCFunction) jf_lseek, METH_VARARGS, jf_lseek__doc }, - { "jsync", (PyCFunction) jf_jsync, METH_VARARGS, jf_jsync__doc }, - { "jmove_journal", (PyCFunction) jf_jmove_journal, METH_VARARGS, - jf_jmove_journal__doc }, - { "new_trans", (PyCFunction) jf_new_trans, METH_VARARGS, - jf_new_trans__doc }, - { NULL } -}; - -static PyObject *jf_getattr(jfile_object *fp, char *name) -{ - return Py_FindMethod(jfile_methods, (PyObject *)fp, name); -} - -static PyTypeObject jfile_type = { - PyObject_HEAD_INIT(NULL) - 0, - "libjio.jfile", - sizeof(jfile_object), - 0, - (destructor)jf_dealloc, - 0, - (getattrfunc)jf_getattr, -}; - - -/* - * The jtrans object - */ - -/* delete */ -static void jt_dealloc(jtrans_object *tp) -{ - if (tp->ts != NULL) { - jtrans_free(tp->ts); - free(tp->ts); - } - Py_DECREF(tp->jfile); - PyObject_Del(tp); -} - -/* add */ -PyDoc_STRVAR(jt_add__doc, -"add(buf, offset)\n\ -\n\ -Add an operation to write the given buffer at the given offset to the\n\ -transaction.\n\ -It's a wrapper to jtrans_add().\n"); - -static PyObject *jt_add(jtrans_object *tp, PyObject *args) -{ - long rv; - int len; - long long offset; - unsigned char *buf; - - if (!PyArg_ParseTuple(args, "s#L:add", &buf, &len, &offset)) - return NULL; - - rv = jtrans_add(tp->ts, buf, len, offset); - if (rv == 0) - return PyErr_SetFromErrno(PyExc_IOError); - - return PyLong_FromLong(rv); -} - -/* commit */ -PyDoc_STRVAR(jt_commit__doc, -"commit()\n\ -\n\ -Commits a transaction.\n\ -It's a wrapper to jtrans_commit().\n"); - -static PyObject *jt_commit(jtrans_object *tp, PyObject *args) -{ - long rv; - - if (!PyArg_ParseTuple(args, ":commit")) - return NULL; - - Py_BEGIN_ALLOW_THREADS - rv = jtrans_commit(tp->ts); - Py_END_ALLOW_THREADS - - if (rv < 0) - return PyErr_SetFromErrno(PyExc_IOError); - - return PyLong_FromLong(rv); -} - -/* rollback */ -PyDoc_STRVAR(jt_rollback__doc, -"rollback()\n\ -\n\ -Rollbacks a transaction.\n\ -It's a wrapper to jtrans_rollback().\n"); - -static PyObject *jt_rollback(jtrans_object *tp, PyObject *args) -{ - long rv; - - if (!PyArg_ParseTuple(args, ":rollback")) - return NULL; - - Py_BEGIN_ALLOW_THREADS - rv = jtrans_rollback(tp->ts); - Py_END_ALLOW_THREADS - - if (rv < 0) - return PyErr_SetFromErrno(PyExc_IOError); - - return PyLong_FromLong(rv); -} - -/* method table */ -static PyMethodDef jtrans_methods[] = { - { "add", (PyCFunction) jt_add, METH_VARARGS, jt_add__doc }, - { "commit", (PyCFunction) jt_commit, METH_VARARGS, jt_commit__doc }, - { "rollback", (PyCFunction) jt_rollback, METH_VARARGS, - jt_rollback__doc }, - { NULL } -}; - -static PyObject *jt_getattr(jtrans_object *tp, char *name) -{ - return Py_FindMethod(jtrans_methods, (PyObject *)tp, name); -} - -static PyTypeObject jtrans_type = { - PyObject_HEAD_INIT(NULL) - 0, - "libjio.jtrans", - sizeof(jtrans_object), - 0, - (destructor)jt_dealloc, - 0, - (getattrfunc)jt_getattr, -}; - - - -/* - * The module - */ - -/* open */ -PyDoc_STRVAR(jf_open__doc, -"open(name[, flags[, mode[, jflags]]])\n\ -\n\ -Opens a file, returns a file object.\n\ -The arguments flags, mode and jflags are the same as jopen(); the constants\n\ -needed are defined in the module.\n\ -It's a wrapper to jopen().\n"); - -static PyObject *jf_open(PyObject *self, PyObject *args) -{ - int rv; - char *file; - int flags, mode, jflags; - jfile_object *fp; - - flags = O_RDWR; - mode = 0600; - jflags = 0; - - if (!PyArg_ParseTuple(args, "s|iii:open", &file, &flags, &mode, - &jflags)) - return NULL; - - fp = PyObject_New(jfile_object, &jfile_type); - if (fp == NULL) - return NULL; - - fp->fs = malloc(sizeof(struct jfs)); - if (fp->fs == NULL) { - return PyErr_NoMemory(); - } - - rv = jopen(fp->fs, file, flags, mode, jflags); - if (rv < 0) { - free(fp->fs); - return PyErr_SetFromErrno(PyExc_IOError); - } - - if (PyErr_Occurred()) { - free(fp->fs); - return NULL; - } - - return (PyObject *) fp; -} - -/* jfsck */ -PyDoc_STRVAR(jf_jfsck__doc, -"jfsck(name[, jdir])\n\ -\n\ -Checks the integrity of the file with the given name, using (optionally) jdir\n\ -as the journal directory; returns a dictionary with all the different values\n\ -of the check (equivalent to the 'struct jfsck_result'). If the path is\n\ -incorrect, or there is no journal associated with it, an IOError will be\n\ -raised.\n\ -It's a wrapper to jfsck().\n"); - -static PyObject *jf_jfsck(PyObject *self, PyObject *args) -{ - int rv; - char *name, *jdir = NULL; - struct jfsck_result res; - PyObject *dict; - - if (!PyArg_ParseTuple(args, "s|s:jfsck", &name, &jdir)) - return NULL; - - dict = PyDict_New(); - if (dict == NULL) - return PyErr_NoMemory(); - - Py_BEGIN_ALLOW_THREADS - rv = jfsck(name, jdir, &res); - Py_END_ALLOW_THREADS - - if (rv == J_ENOMEM) { - Py_XDECREF(dict); - return PyErr_NoMemory(); - } else if (rv != 0) { - Py_XDECREF(dict); - PyErr_SetObject(PyExc_IOError, PyInt_FromLong(rv)); - return NULL; - } - - PyDict_SetItemString(dict, "total", PyLong_FromLong(res.total)); - PyDict_SetItemString(dict, "invalid", PyLong_FromLong(res.invalid)); - PyDict_SetItemString(dict, "in_progress", PyLong_FromLong(res.in_progress)); - PyDict_SetItemString(dict, "broken", PyLong_FromLong(res.broken)); - PyDict_SetItemString(dict, "corrupt", PyLong_FromLong(res.corrupt)); - PyDict_SetItemString(dict, "apply_error", PyLong_FromLong(res.apply_error)); - PyDict_SetItemString(dict, "reapplied", PyLong_FromLong(res.reapplied)); - - return dict; -} - -/* jfsck_cleanup */ -PyDoc_STRVAR(jf_jfsck_cleanup__doc, -"jfsck_cleanup(name[, jdir])\n\ -\n\ -Clean the journal directory for the given file using (optionally) jdir as the\n\ -journal directory, and leave it ready to use.\n\ -It's a wrapper to jfsck_cleanup().\n"); - -static PyObject *jf_jfsck_cleanup(PyObject *self, PyObject *args) -{ - long rv; - char *name, *jdir = NULL; - - if (!PyArg_ParseTuple(args, "s|s:jfsck_cleanup", &name, &jdir)) - return NULL; - - Py_BEGIN_ALLOW_THREADS - rv = jfsck_cleanup(name, jdir); - Py_END_ALLOW_THREADS - - if (rv != 1) { - PyErr_SetObject(PyExc_IOError, PyInt_FromLong(rv)); - return NULL; - } - - return PyInt_FromLong(rv); -} - -/* function table */ -static PyMethodDef libjio_functions[] = { - { "open", (PyCFunction) jf_open, METH_VARARGS, jf_open__doc }, - { "jfsck", (PyCFunction) jf_jfsck, METH_VARARGS, jf_jfsck__doc }, - { "jfsck_cleanup", (PyCFunction) jf_jfsck_cleanup, METH_VARARGS, - jf_jfsck_cleanup__doc }, - { NULL, }, -}; - -/* module initialization */ -PyDoc_STRVAR(libjio__doc, -"libjio is a library to do transactional, journaled I/O\n\ -You can find it at http://blitiri.com.ar/p/libjio/\n\ -\n\ -Use the open() method to create a file object, and then operate on it.\n\ -Please read the documentation for more information.\n"); - -PyMODINIT_FUNC initlibjio(void) -{ - PyObject* m; - - jfile_type.ob_type = &PyType_Type; - jtrans_type.ob_type = &PyType_Type; - - m = Py_InitModule3("libjio", libjio_functions, libjio__doc); - - Py_INCREF(&jfile_type); - PyModule_AddObject(m, "jfile", (PyObject *) &jfile_type); - - Py_INCREF(&jtrans_type); - PyModule_AddObject(m, "jtrans", (PyObject *) &jtrans_type); - - /* libjio's constants */ - PyModule_AddIntConstant(m, "J_NOLOCK", J_NOLOCK); - PyModule_AddIntConstant(m, "J_NOROLLBACK", J_NOROLLBACK); - PyModule_AddIntConstant(m, "J_LINGER", J_LINGER); - PyModule_AddIntConstant(m, "J_COMMITTED", J_COMMITTED); - PyModule_AddIntConstant(m, "J_ROLLBACKED", J_ROLLBACKED); - PyModule_AddIntConstant(m, "J_ROLLBACKING", J_ROLLBACKING); - PyModule_AddIntConstant(m, "J_RDONLY", J_RDONLY); - PyModule_AddIntConstant(m, "J_ESUCCESS", J_ESUCCESS); - PyModule_AddIntConstant(m, "J_ENOENT", J_ENOENT); - PyModule_AddIntConstant(m, "J_ENOJOURNAL", J_ENOJOURNAL); - PyModule_AddIntConstant(m, "J_ENOMEM", J_ENOMEM); - - /* open constants (at least the POSIX ones) */ - PyModule_AddIntConstant(m, "O_RDONLY", O_RDONLY); - PyModule_AddIntConstant(m, "O_WRONLY", O_WRONLY); - PyModule_AddIntConstant(m, "O_RDWR", O_RDWR); - PyModule_AddIntConstant(m, "O_CREAT", O_CREAT); - PyModule_AddIntConstant(m, "O_EXCL", O_EXCL); - PyModule_AddIntConstant(m, "O_TRUNC", O_TRUNC); - PyModule_AddIntConstant(m, "O_APPEND", O_APPEND); - PyModule_AddIntConstant(m, "O_NONBLOCK", O_NONBLOCK); - PyModule_AddIntConstant(m, "O_NDELAY", O_NDELAY); - PyModule_AddIntConstant(m, "O_SYNC", O_SYNC); - PyModule_AddIntConstant(m, "O_ASYNC", O_ASYNC); - - /* lseek constants */ - PyModule_AddIntConstant(m, "SEEK_SET", SEEK_SET); - PyModule_AddIntConstant(m, "SEEK_CUR", SEEK_CUR); - PyModule_AddIntConstant(m, "SEEK_END", SEEK_END); -} - diff --git a/bindings/python2/setup.py b/bindings/python2/setup.py deleted file mode 100644 index 3332b82..0000000 --- a/bindings/python2/setup.py +++ /dev/null @@ -1,16 +0,0 @@ - -from distutils.core import setup, Extension - -libjio = Extension("libjio", - libraries = ['jio'], - sources = ['libjio.c']) - -setup( - name = 'libjio', - description = "A library for journaled I/O", - author="Alberto Bertogli", - author_email="albertito@blitiri.com.ar", - url="http://blitiri.com.ar/p/libjio", - ext_modules = [libjio] -) - diff --git a/bindings/python3/setup.py b/bindings/python3/setup.py deleted file mode 100644 index 3332b82..0000000 --- a/bindings/python3/setup.py +++ /dev/null @@ -1,16 +0,0 @@ - -from distutils.core import setup, Extension - -libjio = Extension("libjio", - libraries = ['jio'], - sources = ['libjio.c']) - -setup( - name = 'libjio', - description = "A library for journaled I/O", - author="Alberto Bertogli", - author_email="albertito@blitiri.com.ar", - url="http://blitiri.com.ar/p/libjio", - ext_modules = [libjio] -) - diff --git a/doc/guide.rst b/doc/guide.rst index 3f3bdef..5202ff2 100644 --- a/doc/guide.rst +++ b/doc/guide.rst @@ -5,23 +5,21 @@ libjio Programmer's Guide Introduction ------------ -This small document attempts serve as a guide to the programmer who wants to -make use of the library. It's not a replacement for the man page or reading -the code; but it's a good starting point for everyone who wants to get -involved with it. +This small document attempts to serve as a guide to the programmer who wants +to use the library. It's not a replacement for the man page or reading the +code; but it's a good starting point for everyone who wants to get involved +with it. The library is not complex to use at all, and the interfaces were designed to be as intuitive as possible, so the text is structured as a guide to present the reader all the common structures and functions the way they're normally used. + Definitions ----------- -This is a library which provides a journaled, transaction-oriented I/O API. -You've probably read this a hundred times already in the documents, and if you -haven't wondered yet what on earth does, this mean you should be reading -something else! +This is a library which provides a transaction-oriented I/O API. We say this is a transaction-oriented API because we make transactions the center of our operations, and journaled because we use a journal (which takes @@ -30,102 +28,88 @@ crash at any point. In this document, we think of a transaction as a list of *(buffer, length, offset)* to be applied to a file. That triplet is called an *operation*, so we -can say that a transaction represent an ordered group of operations on the +can say that a transaction represents an ordered group of operations on the same file. -The act of *committing* a transaction means writing all the elements of the -list; and rollbacking means to undo a previous commit, and leave the data just -as it was before doing the commit. While all this definitions may seem obvious -to some people, it requires special attention because there are a lot of -different definitions, and it's not that common to see "transaction" applied -to file I/O, because it's a term used mostly on database stuff. +The act of *committing* a transaction means writing all the elements of that +list; and *rolling back* means to undo a previous commit, and leave the data +just as it was before doing the commit. The library provides several guarantees, the most relevant and useful being that at any point of time, even if the machine crash horribly, a transaction will be either fully applied or not applied at all. -To achieve this, the library uses what is called a journal, a very vague (and -fashionable) term we use to describe a set of auxiliary files that get created +To achieve this, the library uses what is called a journal, a very vague and +fashionable term we use to describe a set of auxiliary files that get created to store temporary data at several stages. The proper definition and how we use them is outside the scope of this document, and you as a programmer shouldn't need to deal with it. In case you're curious, it's described in a bit more detail in another text which talks about how the library works -internally. Now let's get real. +internally. The data types -------------- -To understand any library, it's essential to be confident in the knowledge of -their data structures and how they relate each other. libjio has two basic -structures which have a very strong relationship, and represent the essential -objects it deals with. Note that you normally don't manipulate them directly, -because they have their own initializer functions, but they are the building -blocks for the rest of the text, which, once this is understood, should be -obvious and self-evident. +libjio has two basic opaque types which have a very strong relationship, and +represent the essential objects it deals with. Note that you don't manipulate +them directly, but use them through the API. -The first structure we face is *struct jfs*, usually called the file -structure, and it represents an open file, just like a regular file descriptor -or a FILE \*. +The first is *jfs_t*, usually called the file structure, and it represents an +open file, just like a regular file descriptor or a *FILE **. -Then you find *struct jtrans*, usually called the transaction structure, which +Then second is *jtrans_t*, usually called the transaction structure, which represents a single transaction. Basic operation --------------- -Now that we've described our data types, let's see how we can operate with the -library. - First of all, as with regular I/O, you need to open your files. This is done -with *jopen()*, which looks a lot like *open()* but takes a file structure +with *jopen()*, which looks a lot like *open()* but returns a file structure instead of a file descriptor (this will be very common among all the functions), and adds a new parameter *jflags* that can be used to modify some -subtle library behaviour we'll see later, and is normally not used. +library behaviour we'll see later, and is normally not used. -We have a happy file structure open now, and the next thing to do would be to -create a transaction. This is what *jtrans_init()* is for: it takes a file -structure and a transaction structure and initializes the latter, leaving it -ready to use. +Now that you have opened a file, the next thing to do would be to create a +transaction. This is what *jtrans_new()* is for: it takes a file structure and +returns a new transaction structure. -Now that we have our transaction, let's add a write operation to it; to do -this we use *jtrans_add()*. We could keep on adding operations to the -transaction by keep on calling jtrans_add() as many times as we want. -Operations within a transaction may overlap, and will be applied in order. +To add an operation to the transaction, use *jtrans_add()*. You can add as +many operations as you want. Operations within a transaction may overlap, and +will be applied in order. -Finally, we decide to apply our transaction to the file, that is, write all -the operations we've added. And this is the easiest part: we call -*jtrans_commit()*, and that's it! +Finally, to apply our transaction to the file, use *jtrans_commit()*. -When we're done using the file, we call *jclose()*, just like we would call -*close()*. +When you're done using the file, call *jclose()*. -Let's put it all together and code a nice "hello world" -program (return values are ignored for simplicity):: +Let's put it all together and code a nice "hello world" program (return values +are ignored for simplicity):: char buf[] = "Hello world!"; - struct jfs file; - struct jtrans trans; + jfs_t *file; + jtrans_t *trans; - jopen(&file, "filename", O_RDWR | O_CREAT, 0600, 0); + file = jopen("filename", O_RDWR | O_CREAT, 0600, 0); - jtrans_init(&file, &trans); - jtrans_add(&trans, buf, strlen(buf), 0); - jtrans_commit(&trans); + trans = jtrans_new(file); + jtrans_add(trans, buf, strlen(buf), 0); + jtrans_commit(trans); + jtrans_free(trans); - jclose(&file); + jclose(file); -As we've seen, we open the file and initialize the structure with *jopen()* +As we've seen, you open the file and initialize the structure with *jopen()* (with the parameter *jflags* being the last 0), create a new transaction with -*jtrans_init()*, then add an operation with *jtrans_add()* (the last 0 is the +*jtrans_new()*, then add an operation with *jtrans_add()* (the last 0 is the offset, in this case the beginning of the file), commit the transaction with -*jtrans_commit()*, and finally close the file with *jclose()*. +*jtrans_commit()*, free it with *jtrans_free()*, and finally close the file +with *jclose()*. Reading is much easier: the library provides three functions, *jread()*, *jpread()* and *jreadv()*, that behave exactly like *read()*, *pread()* and *readv()*, except that they play safe with libjio's writing code. You should -use these to read from files you're writing with libjio. +use these to read from files when using libjio. Integrity checking and recovery @@ -133,27 +117,25 @@ Integrity checking and recovery An essential part of the library is taking care of recovering from crashes and be able to assure a file is consistent. When you're working with the file, -this is taking care of; but what when you first open it? To answer that +this is taking care of; but what about when you first open it? To answer that question, the library provides you with a function named *jfsck()*, which checks the integrity of a file and makes sure that everything is consistent. It must be called "offline", that is when you are not actively committing and rollbacking; it is normally done before calling *jopen()* and is **very, very -important**. Another good practise is to call *jfsck_cleanup()* after calling -*jfsck()*, to make sure we're starting up with a fresh clean journal. After -both calls, it is safe to assume that the file is and ready to use. +important**. You can also do this manually with an utility named jiofsck, which can be used -from the shell to perform the checking and cleanup. +from the shell to perform the checking. Rollback -------- -There is a very nice and important feature in transactions, that allow them to -be "undone", which means that you can undo a transaction and leave the file +There is a very nice and important feature in transactions, that allows them +to be "undone", which means that you can undo a transaction and leave the file just as it was the moment before applying it. The action of undoing it is -called to rollback, and the function is called jtrans_rollback(), which takes +called *rollback*, and the function is called *jtrans_rollback()*, which takes the transaction as the only parameter. Be aware that rollbacking a transaction can be dangerous if you're not careful @@ -172,7 +154,7 @@ and so on) which make each operation a transaction. This can be useful if you don't need to have the full power of the transactions but only to provide guarantees between the different functions. They are a lot like the normal UNIX functions, but instead of getting a file descriptor as their first -parameter, they get a file structure. You can check out the manual page to see +parameter they get a file structure. You can check out the manual page to see the details, but they work just like their UNIX version, only that they preserve atomicity and thread-safety within each call. @@ -193,17 +175,14 @@ The library is completely safe to use in multithreaded applications; however, there are some very basic and intuitive locking rules you have to bear in mind. -Most is fully threadsafe so you don't need to worry about concurrency; in -fact, a lot of effort has been put in making parallel operation safe and fast. - -You need to care only when opening, closing and checking for integrity. In -practise, that means that you shouldn't call *jopen()*, *jclose()* in parallel -with the same jfs structure, or in the middle of an I/O operation, just like -you do when using the normal UNIX calls. In the case of *jfsck()*, you -shouldn't invoke it for the same file more than once at the time; while it -will cope with that situation, it's not recommended. +You need to care only when closing and checking for integrity. In +practise, that means that you shouldn't call *jclose()* in the middle of an +I/O operation, just like you do when using the normal UNIX calls. In the case +of *jfsck()*, you shouldn't invoke it for the same file more than once at the +time, or when the file is open by any other process (this requirement will be +lifted in future releases). -All other operations (commiting a transaction, rollbacking it, adding +All other operations (commiting a transaction, rolling it back, adding operations, etc.) and all the wrappers are safe and don't require any special considerations. @@ -215,7 +194,8 @@ If you need to increase performance, you can use lingering transactions. In this mode, transactions take up more disk space but allows you to do the synchronous write only once, making commits much faster. To use them, just add *J_LINGER* to the *jflags* parameter in *jopen()*. You should call *jsync()* -frequently to avoid using up too much space. +frequently to avoid using up too much space, or start an asynchronous thread +that calls *jsync()* automatically using *jfs_autosync_start()*. Disk layout @@ -254,20 +234,18 @@ uses the same data types as the library. For instance, on 32-bit platforms (like x86), when using LFS, offsets are usually 64 bits, as opposed to the usual 32. -The library is always built with LFS; however, link it against an application -without LFS support could lead to serious problems because this kind of size -differences and ABI compatibility. +The library is always built with LFS; however, linking it against an +application without LFS support could lead to serious problems because this +kind of size differences and ABI compatibility. The Single Unix Specification standard proposes a simple and practical way to get the flags you need to pass your C compiler to tell you want to compile your application with LFS: use a program called "getconf" which should be called like "getconf LFS_CFLAGS", and it outputs the appropiate parameters. -Sadly, not all platforms implement it, so it's also wise to pass -"-D_FILE_OFFSET_BITS=64" just in case. In the end, the command line would be something like:: - gcc `getconf LFS_CFLAGS` -D_FILE_OFFSET_BITS=64 app.c -ljio -o app + gcc `getconf LFS_CFLAGS` app.c -ljio -o app If you want more detailed information or examples, you can check out how the library and sample applications get built. @@ -277,9 +255,9 @@ Where to go from here --------------------- If you're still interested in learning more, you can find some small and clean -samples are in the "samples" directory (full.c is a simple and complete one), -other more advanced examples can be found in the web page, as well as +samples are in the *samples* directory (*full.c* is a simple and complete +one), other more advanced examples can be found in the web page, as well as modifications to well known software to make use of the library. For more information about the inner workings of the library, you can read the "libjio" -document, and the source code. +document, the internal API reference, and the source code. diff --git a/doc/libjio.rst b/doc/libjio.rst index 32de123..6bc4f3c 100644 --- a/doc/libjio.rst +++ b/doc/libjio.rst @@ -163,11 +163,6 @@ from the commit procedure, we only apply the transaction after saving it in the journal, so there is really nothing left to be done. So if the transaction is complete, we only need to rollback. -In any case, after making the recovery you can simply remove the journal -entirely and let the library create a new one, and you can be sure that -transaction atomicity was preserved. You can use jfsck_cleanup() for that -purpose. - UNIX-alike API -------------- diff --git a/libjio/Makefile b/libjio/Makefile index c3ab13e..fecbfda 100644 --- a/libjio/Makefile +++ b/libjio/Makefile @@ -10,7 +10,7 @@ MANDATORY_LDFLAGS := $(shell getconf LFS_LIBS 2>/dev/null) ALL_CFLAGS += $(CFLAGS) $(MANDATORY_CFLAGS) -fPIC ALL_LDFLAGS += $(LDFLAGS) $(MANDATORY_LDFLAGS) -fPIC -LIBS = -lpthread +LIBS = -lpthread -lrt ifdef DEBUG ALL_CFLAGS += -g @@ -39,7 +39,8 @@ endif # objects to build -OBJS = checksum.o common.o trans.o check.o unix.o ansi.o +OBJS = autosync.o checksum.o common.o compat.o trans.o check.o journal.o \ + unix.o ansi.o # rules default: all @@ -81,11 +82,14 @@ install: all .c.o: $(NICE_CC) $(ALL_CFLAGS) -c $< -o $@ +doxygen: + $(MAKE) -C doxygen clean: rm -f $(OBJS) libjio.a libjio.so libjio.pc jiofsck.o jiofsck rm -f *.bb *.bbg *.da *.gcov *.gcno *.gcda gmon.out + $(MAKE) -C doxygen $@ -.PHONY: default all install clean +.PHONY: default all install clean doxygen diff --git a/libjio/ansi.c b/libjio/ansi.c index 017bd66..41180b6 100644 --- a/libjio/ansi.c +++ b/libjio/ansi.c @@ -1,8 +1,5 @@ /* - * libjio - A library for Journaled I/O - * Alberto Bertogli (albertito@blitiri.com.ar) - * * ANSI C API wrappers */ @@ -16,26 +13,24 @@ #include #include "libjio.h" +#include "common.h" +#include "trans.h" /* * To avoid completely useless code duplication, this functions rely on the * UNIX wrappers from unix.c. * - * The API is not nice, and you I wouldn't recommend it for any serious I/O; - * this wrappers are done more as code samples than anything else. + * The API is not nice, and I wouldn't recommend it for any serious I/O. * - * TODO: it should be possible to implement something like this at some - * LD_PRELOAD level without too much harm for most apps. - * TODO: this is still experimental, it hasn't received too much testing + * Note that it is still experimental and it hasn't received too much testing * (that's why it's not even documented), so use it at your own risk. */ -/* fopen wrapper */ +/* fopen() wrapper */ struct jfs *jfopen(const char *path, const char *mode) { - int fd; int flags; int pos_at_the_beginning; struct jfs *fs; @@ -50,35 +45,31 @@ struct jfs *jfopen(const char *path, const char *mode) else flags = O_RDONLY; } else if (mode[0] == 'a') { - /* in this case, make no distinction between "a" and "a+" - * because the file is _always_ open for reading anyways */ - pos_at_the_beginning = 0; - flags = O_RDWR | O_CREAT; + if (strlen(mode) > 1 && strchr(mode, '+')) + pos_at_the_beginning = 1; + else + pos_at_the_beginning = 0; + flags = O_RDWR | O_CREAT | O_APPEND; } else if (mode[0] == 'w') { - /* the same as before */ pos_at_the_beginning = 1; flags = O_RDWR | O_CREAT | O_TRUNC; } else { return NULL; } - fs = malloc(sizeof(struct jfs)); - - fd = jopen(fs, path, flags, 0666, 0); - if (fd < 0) { - free(fs); + fs = jopen(path, flags, 0666, 0); + if (fs == NULL) return NULL; - } if (pos_at_the_beginning) - lseek(fd, 0, SEEK_SET); + lseek(fs->fd, 0, SEEK_SET); else - lseek(fd, 0, SEEK_END); + lseek(fs->fd, 0, SEEK_END); return fs; } -/* fclose wrapper */ +/* fclose() wrapper */ int jfclose(struct jfs *stream) { int rv; @@ -91,7 +82,7 @@ int jfclose(struct jfs *stream) return EOF; } -/* freopen wrapper */ +/* freopen() wrapper */ struct jfs *jfreopen(const char *path, const char *mode, struct jfs *stream) { if (stream) @@ -101,7 +92,7 @@ struct jfs *jfreopen(const char *path, const char *mode, struct jfs *stream) return stream; } -/* fread wrapper */ +/* fread() wrapper */ size_t jfread(void *ptr, size_t size, size_t nmemb, struct jfs *stream) { int rv; @@ -113,7 +104,7 @@ size_t jfread(void *ptr, size_t size, size_t nmemb, struct jfs *stream) return rv / size; } -/* fwrite wrapper */ +/* fwrite() wrapper */ size_t jfwrite(const void *ptr, size_t size, size_t nmemb, struct jfs *stream) { int rv; @@ -125,13 +116,13 @@ size_t jfwrite(const void *ptr, size_t size, size_t nmemb, struct jfs *stream) return rv / size; } -/* fileno wrapper */ +/* fileno() wrapper */ int jfileno(struct jfs *stream) { return stream->fd; } -/* feof wrapper */ +/* feof() wrapper */ int jfeof(struct jfs *stream) { /* ANSI expects that when an EOF is reached in any operation (like @@ -157,14 +148,14 @@ int jfeof(struct jfs *stream) return 0; } -/* clearerr wrapper */ +/* clearerr() wrapper */ void jclearerr(struct jfs *stream) { /* As we do not carry any kind of error state (like explained in * jfeof()), this function has no effect. */ } -/* ferror wrapper */ +/* ferror() wrapper */ int jferror(struct jfs *stream) { /* The same as the above; however not returning this might have some @@ -172,7 +163,7 @@ int jferror(struct jfs *stream) return 0; } -/* fseek wrapper */ +/* fseek() wrapper */ int jfseek(struct jfs *stream, long offset, int whence) { off_t pos; @@ -188,14 +179,14 @@ int jfseek(struct jfs *stream, long offset, int whence) return 0; } -/* ftell wrapper */ +/* ftell() wrapper */ long jftell(struct jfs *stream) { /* forced conversion to long to meet the prototype */ return (long) lseek(stream->fd, 0, SEEK_CUR); } -/* rewind wrapper */ +/* rewind() wrapper */ void jrewind(struct jfs *stream) { lseek(stream->fd, 0, SEEK_SET); @@ -203,7 +194,7 @@ void jrewind(struct jfs *stream) /* convert a struct jfs to a FILE so you can use it with other functions that * require a FILE pointer; be aware that you're bypassing the journaling layer - * and it can cause severe corruption if you're not extremely careful */ + * and it can cause corruption if you're not extremely careful */ FILE *jfsopen(struct jfs *stream, const char *mode) { return fdopen(stream->fd, mode); diff --git a/libjio/autosync.c b/libjio/autosync.c new file mode 100644 index 0000000..edc6ec3 --- /dev/null +++ b/libjio/autosync.c @@ -0,0 +1,140 @@ + +/* + * Autosync API + */ + +#include /* pthread_* */ +#include /* ETIMEDOUT */ +#include /* sig_atomic_t */ +#include /* malloc() and friends */ +#include /* clock_gettime() */ + +#include "common.h" +#include "libjio.h" + +/** Configuration of an autosync thread */ +struct autosync_cfg { + /** File structure to jsync() */ + struct jfs *fs; + + /** Thread id */ + pthread_t tid; + + /** Max number of seconds between each jsync() */ + time_t max_sec; + + /** Max number of bytes written between each jsync() */ + size_t max_bytes; + + /** When the thread must die, we set this to 1 */ + sig_atomic_t must_die; + + /** Condition variable to wake up the thread */ + pthread_cond_t cond; + + /** Mutex to use for the condition variable */ + pthread_mutex_t mutex; +}; + +/** Thread that performs the automatic syncing */ +static void *autosync_thread(void *arg) +{ + int rv; + void *had_errors; + struct timespec ts; + struct autosync_cfg *cfg; + + cfg = (struct autosync_cfg *) arg; + + /* had_errors is a void * just to avoid weird casts, since we want to + * return it, but it's used as a boolean */ + had_errors = (void *) 0; + + pthread_mutex_lock(&cfg->mutex); + for (;;) { + clock_gettime(CLOCK_REALTIME, &ts); + ts.tv_sec += cfg->max_sec; + + rv = pthread_cond_timedwait(&cfg->cond, &cfg->mutex, &ts); + if (rv != 0 && rv != ETIMEDOUT) + break; + + if (cfg->must_die) + break; + + /* cover from spurious wakeups */ + if (rv != ETIMEDOUT && cfg->fs->ltrans_len < cfg->max_bytes) + continue; + + rv = jsync(cfg->fs); + if (rv != 0) + had_errors = (void *) 1; + + } + pthread_mutex_unlock(&cfg->mutex); + + pthread_exit(had_errors); + return NULL; +} + +/* Starts the autosync thread, which will perform a jsync() every max_sec + * seconds, or every max_bytes written using lingering transactions. */ +int jfs_autosync_start(struct jfs *fs, time_t max_sec, size_t max_bytes) +{ + struct autosync_cfg *cfg; + + if (fs->as_cfg != NULL) + return -1; + + cfg = malloc(sizeof(struct autosync_cfg)); + if (cfg == NULL) + return -1; + + cfg->fs = fs; + cfg->max_sec = max_sec; + cfg->max_bytes = max_bytes; + cfg->must_die = 0; + pthread_cond_init(&cfg->cond, NULL); + pthread_mutex_init(&cfg->mutex, NULL); + + fs->as_cfg = cfg; + + return pthread_create(&cfg->tid, NULL, &autosync_thread, cfg); +} + +/* Stops the autosync thread started by jfs_autosync_start(). It's + * automatically called in jclose(). */ +int jfs_autosync_stop(struct jfs *fs) +{ + int rv = 0; + void *had_errors; + + if (fs->as_cfg == NULL) + return 0; + + fs->as_cfg->must_die = 1; + pthread_cond_signal(&fs->as_cfg->cond); + pthread_join(fs->as_cfg->tid, &had_errors); + + if (had_errors) + rv = -1; + + pthread_cond_destroy(&fs->as_cfg->cond); + pthread_mutex_destroy(&fs->as_cfg->mutex); + free(fs->as_cfg); + fs->as_cfg = NULL; + + return rv; +} + +/** Notify the autosync thread that it should check the number of bytes + * written. Must be called with fs' ltlock held. */ +void autosync_check(struct jfs *fs) +{ + if (fs->as_cfg == NULL) + return; + + if (fs->ltrans_len > fs->as_cfg->max_bytes) + pthread_cond_signal(&fs->as_cfg->cond); +} + diff --git a/libjio/check.c b/libjio/check.c index 2fabde5..03458ab 100644 --- a/libjio/check.c +++ b/libjio/check.c @@ -17,9 +17,10 @@ #include "libjio.h" #include "common.h" +#include "trans.h" -/* fill a transaction structure from a mmapped transaction file */ +/** Fill a transaction structure from a mmapped transaction file */ static off_t fill_trans(unsigned char *map, off_t len, struct jtrans *ts) { int i; @@ -94,8 +95,66 @@ error: return 0; } -/* check the journal and rollback incomplete transactions */ -int jfsck(const char *name, const char *jdir, struct jfsck_result *res) +/** Remove all the files in the journal directory (if any). + * + * @param name path to the file + * @param jdir path to the journal directory, use NULL for the default + * @returns 0 on success, < 0 on error + */ +static int jfsck_cleanup(const char *name, const char *jdir) +{ + char path[PATH_MAX], tfile[PATH_MAX*3]; + DIR *dir; + struct dirent *dent; + + if (jdir == NULL) { + if (!get_jdir(name, path)) + return -1; + } else { + strcpy(path, jdir); + } + + dir = opendir(path); + if (dir == NULL && errno == ENOENT) + /* it doesn't exist, so it's clean */ + return 0; + else if (dir == NULL) + return -1; + + for (dent = readdir(dir); dent != NULL; dent = readdir(dir)) { + /* we only care about transactions (named as numbers > 0) and + * the lockfile (named "lock"); ignore everything else */ + if (strcmp(dent->d_name, "lock") && atoi(dent->d_name) <= 0) + continue; + + /* build the full path to the transaction file */ + memset(tfile, 0, PATH_MAX * 3); + strcat(tfile, path); + strcat(tfile, "/"); + strcat(tfile, dent->d_name); + + if (strlen(tfile) > PATH_MAX) { + closedir(dir); + return -1; + } + + if (unlink(tfile) != 0) { + closedir(dir); + return -1; + } + } + if (closedir(dir) != 0) + return -1; + + if (rmdir(path) != 0) + return -1; + + return 0; +} + +/* Check the journal and fix the incomplete transactions */ +enum jfsck_return jfsck(const char *name, const char *jdir, + struct jfsck_result *res, unsigned int flags) { int tfd, rv, i, ret; unsigned int maxtid; @@ -215,13 +274,12 @@ int jfsck(const char *name, const char *jdir, struct jfsck_result *res) /* verify (and possibly fix) all the transactions */ for (i = 1; i <= maxtid; i++) { - curts = malloc(sizeof(struct jtrans)); + curts = jtrans_new(&fs); if (curts == NULL) { ret = J_ENOMEM; goto exit; } - jtrans_init(&fs, curts); curts->id = i; /* open the transaction file, using i as its name, so we are @@ -293,8 +351,6 @@ loop: if (map != NULL) munmap(map, filelen); - if (curts->name) - free(curts->name); while (curts->op != NULL) { tmpop = curts->op->next; if (curts->op->pdata) @@ -308,6 +364,12 @@ loop: res->total++; } + if ( !(flags & J_NOCLEANUP) ) { + if (jfsck_cleanup(name, jdir) < 0) { + ret = J_ECLEANUP; + } + } + exit: if (fs.fd >= 0) close(fs.fd); @@ -326,55 +388,3 @@ exit: } -/* remove all the files in the journal directory (if any) */ -int jfsck_cleanup(const char *name, const char *jdir) -{ - char path[PATH_MAX], tfile[PATH_MAX*3]; - DIR *dir; - struct dirent *dent; - - if (jdir == NULL) { - if (!get_jdir(name, path)) - return 0; - } else { - strcpy(path, jdir); - } - - dir = opendir(path); - if (dir == NULL && errno == ENOENT) - /* it doesn't exist, so it's clean */ - return 1; - else if (dir == NULL) - return 0; - - for (dent = readdir(dir); dent != NULL; dent = readdir(dir)) { - /* we only care about transactions (named as numbers > 0) and - * the lockfile (named "lock"); ignore everything else */ - if (strcmp(dent->d_name, "lock") && atoi(dent->d_name) <= 0) - continue; - - /* build the full path to the transaction file */ - memset(tfile, 0, PATH_MAX * 3); - strcat(tfile, path); - strcat(tfile, "/"); - strcat(tfile, dent->d_name); - - if (strlen(tfile) > PATH_MAX) { - closedir(dir); - return 0; - } - - if (unlink(tfile) != 0) { - closedir(dir); - return 0; - } - } - if (closedir(dir) != 0) - return 0; - - if (rmdir(path) != 0) - return 0; - - return 1; -} - diff --git a/libjio/checksum.c b/libjio/checksum.c index 92a9227..250a96d 100644 --- a/libjio/checksum.c +++ b/libjio/checksum.c @@ -1,8 +1,5 @@ /* - * libjio - A library for Journaled I/O - * Alberto Bertogli (albertito@blitiri.com.ar) - * * Checksum functions * Based on RFC 1071, "Computing the Internet Checksum" */ @@ -13,6 +10,8 @@ #include "common.h" +/** Reads the contents of the given fd, up to len bytes, and stores the + * checksum in csum. Returns 1 on success, 0 on error. */ int checksum(int fd, size_t len, uint32_t *csum) { uint8_t *map; @@ -27,6 +26,8 @@ int checksum(int fd, size_t len, uint32_t *csum) return 1; } +/** Calculates the checksum of the given buffer, up to count bytes. Returns + * the checksum. */ uint32_t checksum_map(uint8_t *map, size_t count) { uint32_t sum = 0; diff --git a/libjio/common.c b/libjio/common.c index c37f144..7a3cdf9 100644 --- a/libjio/common.c +++ b/libjio/common.c @@ -1,8 +1,5 @@ /* - * libjio - A library for Journaled I/O - * Alberto Bertogli (albertito@blitiri.com.ar) - * * Common functions */ @@ -19,7 +16,7 @@ #include "common.h" -/* like lockf, but lock always from the beginning of the file */ +/** Like lockf(), but lock always from the given offset */ off_t plockf(int fd, int cmd, off_t offset, off_t len) { struct flock fl; @@ -50,8 +47,8 @@ off_t plockf(int fd, int cmd, off_t offset, off_t len) return fcntl(fd, op, &fl); } -/* like pread but either fails, or return a complete read; if we return less - * than count is because EOF was reached */ +/** Like pread() but either fails, or return a complete read. If it returns + * less than count it's because EOF was reached */ ssize_t spread(int fd, void *buf, size_t count, off_t offset) { int rv, c; @@ -78,7 +75,7 @@ ssize_t spread(int fd, void *buf, size_t count, off_t offset) return count; } -/* like spread() but for pwrite() */ +/** Like spread() but for pwrite() */ ssize_t spwrite(int fd, const void *buf, size_t count, off_t offset) { int rv, c; @@ -102,7 +99,7 @@ ssize_t spwrite(int fd, const void *buf, size_t count, off_t offset) return count; } -/* build the journal directory name out of the filename */ +/** Store in jdir the default journal directory path of the given filename */ int get_jdir(const char *filename, char *jdir) { char *base, *baset; @@ -128,8 +125,8 @@ int get_jdir(const char *filename, char *jdir) return 1; } -/* build the filename of a given transaction; assumes jtfile can hold at least - * PATH_MAX bytes */ +/** Build the filename of a given transaction. Assumes jtfile can hold at + * least PATH_MAX bytes. */ void get_jtfile(struct jfs *fs, unsigned int tid, char *jtfile) { snprintf(jtfile, PATH_MAX, "%s/%u", fs->jdir, tid); diff --git a/libjio/common.h b/libjio/common.h index c946591..4879eb5 100644 --- a/libjio/common.h +++ b/libjio/common.h @@ -1,9 +1,6 @@ /* - * libjio - A library for Journaled I/O - * Alberto Bertogli (albertito@blitiri.com.ar) - * - * Header for internal functions + * Header for internal functions and definitions */ #ifndef _COMMON_H @@ -29,6 +26,48 @@ #define MAX_TSIZE (SSIZE_MAX) +/** The main file structure */ +struct jfs { + /** Real file fd */ + int fd; + + /** Real file path */ + char *name; + + /** Journal directory path */ + char *jdir; + + /** Journal directory file descriptor */ + int jdirfd; + + /** Journal's lock file descriptor */ + int jfd; + + /** Journal's lock file mmap */ + unsigned int *jmap; + + /** Journal flags */ + uint32_t flags; + + /** Flags passed to the real open() */ + uint32_t open_flags; + + /** Lingering transactions (linked list) */ + struct jlinger *ltrans; + + /** Length of all the lingered transactions */ + size_t ltrans_len; + + /** Lingering transactions' lock */ + pthread_mutex_t ltlock; + + /** A soft lock used in some operations */ + pthread_mutex_t lock; + + /** Autosync config */ + struct autosync_cfg *as_cfg; +}; + off_t plockf(int fd, int cmd, off_t offset, off_t len); ssize_t spread(int fd, void *buf, size_t count, off_t offset); @@ -39,5 +78,7 @@ void get_jtfile(struct jfs *fs, unsigned int tid, char *jtfile); int checksum(int fd, size_t len, uint32_t *csum); uint32_t checksum_map(uint8_t *map, size_t count); +void autosync_check(struct jfs *fs); + #endif diff --git a/libjio/compat.c b/libjio/compat.c new file mode 100644 index 0000000..8c7645b --- /dev/null +++ b/libjio/compat.c @@ -0,0 +1,59 @@ + +/* + * Compatibility functions + */ + +/* To get sync_file_range() we need to temporarily define _GNU_SOURCE, which + * is not the nicest thing, but is not worth defining globally */ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#define _REMOVE_GNU_SOURCE +#endif + +#include /* sync_range_submit(), if possible */ +#include "compat.h" + + +/* + * sync_file_range() support through an internal similar API + */ + +#ifdef SYNC_FILE_RANGE_WRITE +const int have_sync_range = 1; + +/** Initiate write-out of the dirty pages in the range */ +int sync_range_submit(int fd, off_t offset, size_t nbytes) +{ + /* We don't need SYNC_FILE_RANGE_WAIT_BEFORE because we have exclusive + * access to the range (guaranteed by the caller) */ + return sync_file_range(fd, offset, nbytes, SYNC_FILE_RANGE_WRITE); +} + +/** Wait for completion of the previously-submitted I/O in the given ranges. + * Does NOT force the submission of any new I/O. */ +int sync_range_wait(int fd, off_t offset, size_t nbytes) +{ + return sync_file_range(fd, offset, nbytes, SYNC_FILE_RANGE_WAIT_BEFORE); +} + +#else + +#warning "No sync_file_range()" +const int have_sync_range = 0; + +int sync_range_submit(int fd, off_t offset, size_t nbytes) +{ + return -1; +} + +int sync_range_wait(int fd, off_t offset, size_t nbytes) +{ + return -1; +} + +#endif /* defined SYNC_FILE_RANGE_WRITE */ + +#ifdef _REMOVE_GNU_SOURCE +#undef _GNU_SOURCE +#endif + diff --git a/libjio/compat.h b/libjio/compat.h index ecc742a..cf90798 100644 --- a/libjio/compat.h +++ b/libjio/compat.h @@ -1,9 +1,17 @@ -/* Header to provide fallbacks for compatibility purposes. */ +/* Header to provide fallbacks for compatibility purposes */ #ifndef _COMPAT_H #define _COMPAT_H +/* sync_file_range() is linux-specific, so we provide an internal similar API, + * with a constant to be able to check for its presence; the implementation is + * in compat.c */ +extern const int have_sync_range; +int sync_range_submit(int fd, off_t offset, size_t nbytes); +int sync_range_wait(int fd, off_t offset, size_t nbytes); + + /* posix_fadvise() was introduced in SUSv3. Because it's the only SUSv3 * function we rely on so far (everything else is SUSv2), we define a void * fallback for systems that do not implement it. diff --git a/libjio/doxygen/Doxyfile.base b/libjio/doxygen/Doxyfile.base new file mode 100644 index 0000000..9db9a87 --- /dev/null +++ b/libjio/doxygen/Doxyfile.base @@ -0,0 +1,204 @@ +DOXYFILE_ENCODING = UTF-8 +PROJECT_NAME = libjio +PROJECT_NUMBER = 0.50 +OUTPUT_DIRECTORY = +CREATE_SUBDIRS = NO +OUTPUT_LANGUAGE = English +BRIEF_MEMBER_DESC = YES +REPEAT_BRIEF = YES +ABBREVIATE_BRIEF = +ALWAYS_DETAILED_SEC = NO +INLINE_INHERITED_MEMB = NO +FULL_PATH_NAMES = NO +STRIP_FROM_PATH = +STRIP_FROM_INC_PATH = +SHORT_NAMES = NO +JAVADOC_AUTOBRIEF = YES +QT_AUTOBRIEF = YES +MULTILINE_CPP_IS_BRIEF = NO +INHERIT_DOCS = YES +SEPARATE_MEMBER_PAGES = NO +TAB_SIZE = 8 +ALIASES = +OPTIMIZE_OUTPUT_FOR_C = YES +OPTIMIZE_OUTPUT_JAVA = NO +OPTIMIZE_FOR_FORTRAN = NO +OPTIMIZE_OUTPUT_VHDL = NO +EXTENSION_MAPPING = +BUILTIN_STL_SUPPORT = NO +CPP_CLI_SUPPORT = NO +SIP_SUPPORT = NO +IDL_PROPERTY_SUPPORT = YES +DISTRIBUTE_GROUP_DOC = NO +SUBGROUPING = YES +TYPEDEF_HIDES_STRUCT = NO +SYMBOL_CACHE_SIZE = 0 +EXTRACT_ALL = NO +EXTRACT_PRIVATE = NO +EXTRACT_STATIC = NO +EXTRACT_LOCAL_CLASSES = YES +EXTRACT_LOCAL_METHODS = NO +EXTRACT_ANON_NSPACES = NO +HIDE_UNDOC_MEMBERS = NO +HIDE_UNDOC_CLASSES = NO +HIDE_FRIEND_COMPOUNDS = NO +HIDE_IN_BODY_DOCS = NO +INTERNAL_DOCS = NO +CASE_SENSE_NAMES = YES +HIDE_SCOPE_NAMES = NO +SHOW_INCLUDE_FILES = YES +INLINE_INFO = YES +SORT_MEMBER_DOCS = NO +SORT_BRIEF_DOCS = NO +SORT_GROUP_NAMES = NO +SORT_BY_SCOPE_NAME = NO +GENERATE_TODOLIST = YES +GENERATE_TESTLIST = YES +GENERATE_BUGLIST = YES +GENERATE_DEPRECATEDLIST= YES +ENABLED_SECTIONS = +MAX_INITIALIZER_LINES = 30 +SHOW_USED_FILES = YES +SHOW_DIRECTORIES = YES +SHOW_FILES = YES +SHOW_NAMESPACES = YES +FILE_VERSION_FILTER = +LAYOUT_FILE = +QUIET = YES +WARNINGS = YES +WARN_IF_UNDOCUMENTED = YES +WARN_IF_DOC_ERROR = YES +WARN_NO_PARAMDOC = YES +WARN_FORMAT = "$file:$line: $text" +WARN_LOGFILE = +INPUT = ../ +INPUT_ENCODING = UTF-8 +FILE_PATTERNS = *.c *.h *.doxy +RECURSIVE = YES +EXCLUDE = +EXCLUDE_SYMLINKS = NO +EXCLUDE_PATTERNS = +EXCLUDE_SYMBOLS = +EXAMPLE_PATH = +EXAMPLE_PATTERNS = +EXAMPLE_RECURSIVE = NO +IMAGE_PATH = +INPUT_FILTER = +FILTER_PATTERNS = +FILTER_SOURCE_FILES = NO +SOURCE_BROWSER = YES +INLINE_SOURCES = NO +STRIP_CODE_COMMENTS = YES +REFERENCED_BY_RELATION = YES +REFERENCES_RELATION = NO +REFERENCES_LINK_SOURCE = NO +USE_HTAGS = NO +VERBATIM_HEADERS = YES +ALPHABETICAL_INDEX = NO +COLS_IN_ALPHA_INDEX = 5 +IGNORE_PREFIX = +GENERATE_HTML = YES +HTML_OUTPUT = html +HTML_FILE_EXTENSION = .html +HTML_HEADER = +HTML_FOOTER = +HTML_STYLESHEET = +HTML_ALIGN_MEMBERS = YES +HTML_DYNAMIC_SECTIONS = YES +GENERATE_DOCSET = NO +DOCSET_FEEDNAME = "Doxygen generated docs" +DOCSET_BUNDLE_ID = org.doxygen.Project +GENERATE_HTMLHELP = NO +CHM_FILE = +HHC_LOCATION = +GENERATE_CHI = NO +CHM_INDEX_ENCODING = +BINARY_TOC = NO +TOC_EXPAND = YES +GENERATE_QHP = NO +QCH_FILE = +QHP_NAMESPACE = +QHP_VIRTUAL_FOLDER = doc +QHP_CUST_FILTER_NAME = +QHP_CUST_FILTER_ATTRS = +QHP_SECT_FILTER_ATTRS = +QHG_LOCATION = +DISABLE_INDEX = NO +ENUM_VALUES_PER_LINE = 4 +GENERATE_TREEVIEW = NONE +TREEVIEW_WIDTH = 250 +FORMULA_FONTSIZE = 10 +GENERATE_LATEX = NO +LATEX_OUTPUT = latex +LATEX_CMD_NAME = latex +MAKEINDEX_CMD_NAME = makeindex +COMPACT_LATEX = NO +PAPER_TYPE = a4wide +EXTRA_PACKAGES = +LATEX_HEADER = +PDF_HYPERLINKS = YES +USE_PDFLATEX = YES +LATEX_BATCHMODE = NO +LATEX_HIDE_INDICES = NO +GENERATE_RTF = NO +RTF_OUTPUT = rtf +COMPACT_RTF = NO +RTF_HYPERLINKS = NO +RTF_STYLESHEET_FILE = +RTF_EXTENSIONS_FILE = +GENERATE_MAN = NO +MAN_OUTPUT = man +MAN_EXTENSION = .3 +MAN_LINKS = NO +GENERATE_XML = NO +XML_OUTPUT = xml +XML_SCHEMA = +XML_DTD = +XML_PROGRAMLISTING = YES +GENERATE_AUTOGEN_DEF = NO +GENERATE_PERLMOD = NO +PERLMOD_LATEX = NO +PERLMOD_PRETTY = YES +PERLMOD_MAKEVAR_PREFIX = +ENABLE_PREPROCESSING = YES +MACRO_EXPANSION = NO +EXPAND_ONLY_PREDEF = NO +SEARCH_INCLUDES = YES +INCLUDE_PATH = +INCLUDE_FILE_PATTERNS = +PREDEFINED = +EXPAND_AS_DEFINED = +SKIP_FUNCTION_MACROS = YES +TAGFILES = +GENERATE_TAGFILE = +ALLEXTERNALS = NO +EXTERNAL_GROUPS = YES +PERL_PATH = /usr/bin/perl +CLASS_DIAGRAMS = YES +MSCGEN_PATH = +HIDE_UNDOC_RELATIONS = YES +HAVE_DOT = YES +DOT_FONTNAME = FreeSans +DOT_FONTSIZE = 10 +DOT_FONTPATH = +CLASS_GRAPH = YES +COLLABORATION_GRAPH = YES +GROUP_GRAPHS = YES +UML_LOOK = NO +TEMPLATE_RELATIONS = NO +INCLUDE_GRAPH = YES +INCLUDED_BY_GRAPH = YES +CALL_GRAPH = YES +CALLER_GRAPH = YES +GRAPHICAL_HIERARCHY = YES +DIRECTORY_GRAPH = YES +DOT_IMAGE_FORMAT = png +DOT_PATH = +DOTFILE_DIRS = +DOT_GRAPH_MAX_NODES = 50 +MAX_DOT_GRAPH_DEPTH = 0 +DOT_TRANSPARENT = YES +DOT_MULTI_TARGETS = NO +GENERATE_LEGEND = YES +DOT_CLEANUP = YES +SEARCHENGINE = NO diff --git a/libjio/doxygen/Doxyfile.internal b/libjio/doxygen/Doxyfile.internal new file mode 100644 index 0000000..581bdb3 --- /dev/null +++ b/libjio/doxygen/Doxyfile.internal @@ -0,0 +1,8 @@ +@INCLUDE = Doxyfile.base +PROJECT_NAME = "libjio (internal)" +OUTPUT_DIRECTORY = doc.internal +EXTRACT_ALL = YES +EXTRACT_STATIC = YES +INTERNAL_DOCS = YES +EXCLUDE = jiofsck.c + diff --git a/libjio/doxygen/Doxyfile.public b/libjio/doxygen/Doxyfile.public new file mode 100644 index 0000000..98d1b37 --- /dev/null +++ b/libjio/doxygen/Doxyfile.public @@ -0,0 +1,8 @@ +@INCLUDE = Doxyfile.base +PROJECT_NAME = "libjio (public)" +OUTPUT_DIRECTORY = doc.public +EXTRACT_ALL = YES +EXTRACT_STATIC = NO +INTERNAL_DOCS = NO +INPUT = ../libjio.h groups.doxy + diff --git a/libjio/doxygen/Makefile b/libjio/doxygen/Makefile new file mode 100644 index 0000000..3b020d0 --- /dev/null +++ b/libjio/doxygen/Makefile @@ -0,0 +1,24 @@ + +ifneq ($(V), 1) + NICE_DOXYGEN = @echo " DOXYGEN $@"; doxygen +else + NICE_DOXYGEN = doxygen +endif + + +default: all + +all: public internal + +public: + $(NICE_DOXYGEN) Doxyfile.public + +internal: + $(NICE_DOXYGEN) Doxyfile.internal + +clean: + rm -rf doc.internal doc.public + + +.PHONY: all clean default doxygen internal public + diff --git a/libjio/doxygen/groups.doxy b/libjio/doxygen/groups.doxy new file mode 100644 index 0000000..e6275bf --- /dev/null +++ b/libjio/doxygen/groups.doxy @@ -0,0 +1,13 @@ + +/** +@addtogroup basic Basic API +This is the basic, lower-level API. + +@addtogroup check Check API +This is the integrity-checking API. + +@addtogroup unix UNIX-alike API +This is the UNIX-alike API, built on top of the @ref basic "basic API". + +*/ + diff --git a/libjio/jiofsck.c b/libjio/jiofsck.c index a22dcd7..8cfe00f 100644 --- a/libjio/jiofsck.c +++ b/libjio/jiofsck.c @@ -30,9 +30,13 @@ Examples:\n\ int main(int argc, char **argv) { - int i, rv, do_cleanup; + int i, do_cleanup; + unsigned int flags; char *file, *jdir; struct jfsck_result res; + enum jfsck_return rv; + + file = jdir = NULL; do_cleanup = 0; @@ -54,9 +58,13 @@ int main(int argc, char **argv) memset(&res, 0, sizeof(res)); + flags = 0; + if (!do_cleanup) + flags |= J_NOCLEANUP; + printf("Checking journal: "); fflush(stdout); - rv = jfsck(file, jdir, &res); + rv = jfsck(file, jdir, &res, flags); if (rv == J_ENOENT) { printf("No such file or directory\n"); @@ -69,17 +77,6 @@ int main(int argc, char **argv) printf("done\n"); - if (do_cleanup) { - printf("Cleaning journal: "); - fflush(stdout); - if (!jfsck_cleanup(file, jdir)) { - printf("Error cleaning journal\n"); - return 1; - } - - printf("done\n"); - } - printf("Journal checking results\n"); printf("------------------------\n\n"); diff --git a/libjio/journal.c b/libjio/journal.c new file mode 100644 index 0000000..be9e04b --- /dev/null +++ b/libjio/journal.c @@ -0,0 +1,343 @@ + +/* + * Internal journal + */ + +#include /* [s]size_t */ +#include /* open() */ +#include /* open() */ +#include /* f[data]sync(), close() */ +#include /* malloc() and friends */ +#include /* MAX_PATH */ +#include /* memcpy() */ +#include /* basename(), dirname() */ +#include /* fprintf() */ +#include /* readdir() and friends */ +#include /* errno */ +#include /* mmap() */ + +#include "libjio.h" +#include "common.h" +#include "compat.h" +#include "journal.h" +#include "trans.h" + + +/* + * helper functions + */ + +/** Get a new transaction id */ +static unsigned int get_tid(struct jfs *fs) +{ + unsigned int curid, rv; + + /* lock the whole file */ + plockf(fs->jfd, F_LOCKW, 0, 0); + + /* read the current max. curid */ + curid = *(fs->jmap); + + fiu_do_on("jio/get_tid/overflow", curid = -1); + + /* increment it and handle overflows */ + rv = curid + 1; + if (rv == 0) + goto exit; + + /* write to the file descriptor */ + *(fs->jmap) = rv; + +exit: + plockf(fs->jfd, F_UNLOCK, 0, 0); + return rv; +} + +/** Free a transaction id */ +static void free_tid(struct jfs *fs, unsigned int tid) +{ + unsigned int curid, i; + char name[PATH_MAX]; + + /* lock the whole file */ + plockf(fs->jfd, F_LOCKW, 0, 0); + + /* read the current max. curid */ + curid = *(fs->jmap); + + /* if we're the max tid, scan the directory looking up for the new + * max; the detailed description can be found in the "doc/" dir */ + if (tid == curid) { + /* look up the new max. */ + for (i = curid - 1; i > 0; i--) { + get_jtfile(fs, i, name); + if (access(name, R_OK | W_OK) == 0) { + break; + } else if (errno != EACCES) { + /* Real error, stop looking for a new max. It + * doesn't hurt us because it's ok if the max + * is higher than it could be */ + break; + } + } + + /* and save it */ + *(fs->jmap) = i; + } + + plockf(fs->jfd, F_UNLOCK, 0, 0); + return; +} + + +static int already_warned_about_sync = 0; + +/** fsync() a directory */ +static int fsync_dir(int fd) +{ + int rv; + + rv = fsync(fd); + + if (rv != 0 && (errno == EINVAL || errno == EBADF)) { + /* it seems to be legal that fsync() on directories is not + * implemented, so if this fails with EINVAL or EBADF, just + * call a global sync(); which is awful (and might still + * return before metadata is done) but it seems to be the + * saner choice; otherwise we just fail */ + sync(); + rv = 0; + + if (!already_warned_about_sync) { + fprintf(stderr, "libjio warning: falling back on " \ + "sync() for directory syncing\n"); + already_warned_about_sync = 1; + } + } + + return rv; +} + + +/* + * Journal functions + */ + +/** Create a new transaction in the journal. Returns a pointer to an opaque + * jop_t (that is freed using journal_free), or NULL if there was an error. */ +struct journal_op *journal_new(struct jtrans *ts) +{ + int fd, id; + ssize_t rv; + char *name = NULL; + unsigned char buf_init[J_DISKHEADSIZE]; + unsigned char *bufp; + struct journal_op *jop = NULL; + + jop = malloc(sizeof(struct journal_op)); + if (jop == NULL) + goto error; + + name = (char *) malloc(PATH_MAX); + if (name == NULL) + goto error; + + id = get_tid(ts->fs); + if (id == 0) + goto error; + + /* open the transaction file */ + get_jtfile(ts->fs, id, name); + fd = open(name, O_RDWR | O_CREAT | O_TRUNC, 0600); + if (fd < 0) + goto error; + + jop->id = id; + jop->fd = fd; + jop->name = name; + jop->curpos = 0; + jop->ts = ts; + jop->fs = ts->fs; + + fiu_exit_on("jio/commit/created_tf"); + + /* and lock it, just in case */ + plockf(fd, F_LOCKW, 0, 0); + + ts->id = id; + + /* save the header */ + bufp = buf_init; + + memcpy(bufp, (void *) &(ts->id), 4); + bufp += 4; + + memcpy(bufp, (void *) &(ts->flags), 4); + bufp += 4; + + memcpy(bufp, (void *) &(ts->numops), 4); + bufp += 4; + + rv = spwrite(fd, buf_init, J_DISKHEADSIZE, 0); + if (rv != J_DISKHEADSIZE) { + free(buf_init); + goto unlink_error; + } + + jop->curpos = J_DISKHEADSIZE; + + fiu_exit_on("jio/commit/tf_header"); + + return jop; + +unlink_error: + unlink(name); + free_tid(ts->fs, ts->id); + close(fd); + +error: + if (name) + free(name); + if (jop) + free(jop); + + return NULL; +} + +/** Save the given transaction in the journal */ +int journal_save(struct journal_op *jop) +{ + ssize_t rv; + uint32_t csum; + struct joper *op; + unsigned char hdr[J_DISKOPHEADSIZE]; + unsigned char *hdrp; + const struct jtrans *ts = jop->ts; + + /* save each transacion in the file */ + for (op = ts->op; op != NULL; op = op->next) { + /* read the current content only if the transaction is not + * marked as NOROLLBACK, and if the data is not there yet, + * which is the normal case, but for rollbacking we fill it + * ourselves */ + if (!(ts->flags & J_NOROLLBACK) && (op->pdata == NULL)) { + op->pdata = malloc(op->len); + if (op->pdata == NULL) + goto error; + + op->plen = op->len; + + rv = spread(ts->fs->fd, op->pdata, op->len, + op->offset); + if (rv < 0) + goto error; + if (rv < op->len) { + /* we are extending the file! */ + /* ftruncate(ts->fs->fd, op->offset + op->len); */ + op->plen = rv; + } + } + + /* save the operation's header */ + hdrp = hdr; + + memcpy(hdrp, (void *) &(op->len), 4); + hdrp += 4; + + memcpy(hdrp, (void *) &(op->plen), 4); + hdrp += 4; + + memcpy(hdrp, (void *) &(op->offset), 8); + hdrp += 8; + + rv = spwrite(jop->fd, hdr, J_DISKOPHEADSIZE, jop->curpos); + if (rv != J_DISKOPHEADSIZE) + goto error; + + fiu_exit_on("jio/commit/tf_ophdr"); + + jop->curpos += J_DISKOPHEADSIZE; + + /* and save it to the disk */ + rv = spwrite(jop->fd, op->buf, op->len, jop->curpos); + if (rv != op->len) + goto error; + + jop->curpos += op->len; + + fiu_exit_on("jio/commit/tf_opdata"); + } + + fiu_exit_on("jio/commit/tf_data"); + + /* compute and save the checksum (curpos is always small, so there's + * no overflow possibility when we convert to size_t) */ + if (!checksum(jop->fd, jop->curpos, &csum)) + goto error; + + rv = spwrite(jop->fd, &csum, sizeof(uint32_t), jop->curpos); + if (rv != sizeof(uint32_t)) + goto error; + jop->curpos += sizeof(uint32_t); + + /* this is a simple but efficient optimization: instead of doing + * everything O_SYNC, we sync at this point only, this way we avoid + * doing a lot of very small writes; in case of a crash the + * transaction file is only useful if it's complete (ie. after this + * point) so we only flush here (both data and metadata) */ + if (fsync(jop->fd) != 0) + goto error; + if (fsync_dir(ts->fs->jdirfd) != 0) + goto error; + + fiu_exit_on("jio/commit/tf_sync"); + + return 0; + +error: + return -1; +} + +/** Free a journal operation. + * NOTE: It can't assume the save completed successfuly, so we can call it + * when journal_save() fails. */ +int journal_free(struct journal_op *jop) +{ + int rv; + + rv = -1; + + if (unlink(jop->name)) { + /* we do not want to leave a possibly complete transaction + * file around when the transaction was not commited and the + * unlink failed, so we attempt to truncate it, and if that + * fails we corrupt the checksum as a last resort */ + if (ftruncate(jop->fd, 0) != 0) { + if (pwrite(jop->fd, "\0\0\0\0", 4, jop->curpos - 4) + != 4) + goto exit; + if (fdatasync(jop->fd) != 0) + goto exit; + } + } + + if (fsync_dir(jop->fs->jdirfd) != 0) + goto exit; + + fiu_exit_on("jio/commit/pre_ok_free_tid"); + free_tid(jop->fs, jop->id); + + rv = 0; + +exit: + close(jop->fd); + + if (jop->name) + free(jop->name); + + free(jop); + + return rv; +} + + diff --git a/libjio/journal.h b/libjio/journal.h new file mode 100644 index 0000000..5a4666c --- /dev/null +++ b/libjio/journal.h @@ -0,0 +1,24 @@ + +#ifndef _JOURNAL_H +#define _JOURNAL_H + +#include "libjio.h" + + +struct journal_op { + int id; + int fd; + char *name; + off_t curpos; + struct jtrans *ts; + struct jfs *fs; +}; + +typedef struct journal_op jop_t; + +struct journal_op *journal_new(struct jtrans *ts); +int journal_save(struct journal_op *jop); +int journal_free(struct journal_op *jop); + +#endif + diff --git a/libjio/libjio.3 b/libjio/libjio.3 index defac0e..4de5881 100644 --- a/libjio/libjio.3 +++ b/libjio/libjio.3 @@ -1,17 +1,41 @@ .TH libjio 3 "21/Feb/2004" .SH NAME libjio - A library for Journaled I/O -.SH SYNOPSYS +.SH SYNOPSIS .nf .B #include -.B struct jfs; +.BI "jfs_t *jopen(const char *" name ", int " flags ", int " mode ", +.BI " int " jflags ");" +.BI "ssize_t jread(jfs_t *" fs ", void *" buf ", size_t " count ");" +.BI "ssize_t jpread(jfs_t *" fs ", void *" buf ", size_t " count "," +.BI " off_t " offset ");" +.BI "ssize_t jreadv(jfs_t *" fs ", struct iovec *" vector "," +.BI " int " count ");" +.BI "ssize_t jwrite(jfs_t *" fs ", const void *" buf ", size_t " count ");" +.BI "ssize_t jpwrite(jfs_t *" fs ", const void *" buf ", size_t " count "," +.BI " off_t " offset ");" +.BI "ssize_t jwritev(jfs_t *" fs ", const struct iovec *" vector "," +.BI " int " count ");" +.BI "int jtruncate(jfs_t *" fs ", off_t " lenght ");" +.BI "off_t jlseek(jfs_t *" fs ", off_t " offset ", int " whence ");" +.BI "int jclose(jfs_t *" fs ");" -.BR "struct jtrans " { - ... - unsigned int flags; - ... -}; +.BI "jtrans_t *jtrans_new(jfs_t *" fs ");" +.BI "int jtrans_commit(jtrans_t *" ts ");" +.BI "int jtrans_add(jtrans_t *" ts ", const void *" buf "," +.BI " size_t " count ", off_t " offset ");" +.BI "int jtrans_rollback(jtrans_t *" ts ");" +.BI "void jtrans_free(jtrans_t *" ts ");" + +.BI "int jsync(jfs_t *" fs ");" +.BI "int jfs_autosync_start(jfs_t *" fs ", time_t " max_sec "," +.BI " size_t " max_bytes ");" +.BI "int jfs_autosync_stop(jfs_t *" fs ");" +.BI "int jmove_journal(jfs_t *" fs ", const char *" newpath ");" + +.BI "enum jfsck_return jfsck(const char *" name ", const char *" jdir "," +.BI " jfsck_result *" res ", unsigned int " flags ");" .BR "struct jfsck_result" " {" int total; /* total transactions files we looked at */ @@ -23,35 +47,14 @@ libjio - A library for Journaled I/O ... }; -.BI "int jopen(struct jfs *" fs ", const char *" name "," -.BI " int " flags ", int " mode ", int " jflags ");" -.BI "ssize_t jread(struct jfs *" fs ", void *" buf ", size_t " count ");" -.BI "ssize_t jpread(struct jfs *" fs ", void *" buf ", size_t " count "," -.BI " off_t " offset ");" -.BI "ssize_t jreadv(struct jfs *" fs ", struct iovec *" vector "," -.BI " int " count ");" -.BI "ssize_t jwrite(struct jfs *" fs ", const void *" buf ", size_t " count ");" -.BI "ssize_t jpwrite(struct jfs *" fs ", const void *" buf ", size_t " count "," -.BI " off_t " offset ");" -.BI "ssize_t jwritev(struct jfs *" fs ", const struct iovec *" vector "," -.BI " int " count ");" -.BI "int jtruncate(struct jfs *" fs ", off_t " lenght ");" -.BI "off_t jlseek(struct jfs *" fs ", off_t " offset ", int " whence ");" -.BI "int jclose(struct jfs *" fs ");" - -.BI "void jtrans_init(struct jfs *" fs " ,struct jtrans *" ts ");" -.BI "int jtrans_commit(struct jtrans *" ts ");" -.BI "int jtrans_add(struct jtrans *" ts ", const void * " buf "," -.BI " size_t " count ", off_t " offset ");" -.BI "int jtrans_rollback(struct jtrans *" ts ");" -.BI "void jtrans_free(struct jtrans *" ts ");" - -.BI "int jsync(struct jfs *" fs ");" -.BI "int jmove_journal(struct jfs *" fs ", const char *" newpath ");" +.BR "enum jfsck_return" " {" + J_ESUCCESS = 0, /* Success */ + J_ENOENT = -1, /* No such file or directory */ + J_ENOJOURNAL = -2, /* No journal associated with the given file */ + J_ENOMEM = -3, /* Not enough free memory */ + J_ECLEANUP = -4, /* Error cleaning the journal directory */ +}; -.BI "int jfsck(const char *" name ", const char *" jdir "," -.BI " struct jfsck_result *" res ");" -.BI "int jfsck_cleanup(const char *" name ", const char *" jdir ");" .SH DESCRIPTION @@ -67,7 +70,7 @@ completion and < 0 upon failure, unless otherwise noted. The common functions provide functionality common to the other two: .BR jopen() " and " jclose() to open and close files in order to use them with the library, and -.BR "jfsck() " and " jfsck_cleanup()" +.B jfsck() to provide integrity checking. The UNIX-alike API mimics the traditional UNIX API by providing similar @@ -76,41 +79,35 @@ interfaces to and friends. The basic functions consist of -.BR jtrans_init() ", " jtrans_add() ", " jtrans_commit() " and " +.BR jtrans_new() ", " jtrans_add() ", " jtrans_commit() " and " .BR jtrans_rollback() . -They provide a lower-level method for manipulating transactions, which are -defined in the -.IR "jtrans structure" . +They provide a lower-level method for manipulating transactions. -.SS STRUCTURES +.SS TYPES AND STRUCTURES -Both -.IR "struct jfs" " and " "struct jtrans" -are meant to be treated as opaque types, except for the fields documented -above, which you should treat as read-only. +.I jfs_t +represents an open file, and +.I jtrans_t +represents a transaction. Both are meant to be treated as opaque types. .B struct jfsck_result holds the results of a .B jfsck() -run, see below for details. +run. The fields are described in the synopsis section. .SS COMMON FUNCTIONS -Most functions reference the structures described above, specially -.IR "struct jfs" " and " "struct jtrans" . -They represent, respectively, a file to operate on and a single transaction. To open a file, you should use .BR jopen() , which is just like the normal .B open(2) -call but affects a pointer to a -.IR struct jfs . +call but returns an opaque pointer. To close a file, use .BR jclose() . They're exactly like the .BR open(2) " and " close(2) functions but use a -.I struct jfs +.I jfs_t instead of a file descriptor; take a look at their manpages if you have any doubts about how to use them. @@ -119,17 +116,30 @@ can be used to move the journal directory to a new location. It can be called only when nobody else is using the file. It is usually not used, except for very special cases. -There are two functions that differ from the rest, which are -.BR jfsck() " and " jfsck_cleanup() . -Both take as the first two parameters the path to the file to check and the -path to the journal directory (usually NULL for the default, unless you've -changed it manually using -.BR jmove_journal() ). - -The first one, -.BR jfsck() , -is used to perform journal checking and recovery in case of a crash. It must -be performed when nobody else is using the file (like in the case of a +.B jfs_autosync_start() +can be used to start a thread which will automatically perform a +.B jsync() +after the given number of seconds or the given number of bytes written using +lingering transactions (whatever comes first). It's very useful when using +lingering transactions. +.B jfs_autosync_stop() +stops the thread started by +.BR jfs_autosync_start() . +The thread is also stopped automatically when +.B jclose() +is called. + +.B jfsck() +takes as the first two parameters the path to the file to check and the path +to the journal directory (usually NULL for the default, unless you've changed +it manually using +.BR jmove_journal() ), +and optionally a flags parameter, which can be 0 for the default behaviour, or +J_NOCLEANUP to indicate that the journal should not be cleaned up after +successful recovery. + +It is used to perform journal checking and recovery in case of a crash. It +must be performed when nobody else is using the file (like in the case of a filesystem which can't be mounted), and it returns 0 if success or an error code != 0 in case of a failure. If it succeeded, it will fill jfsck_result summarizing the outcome of the operation. The error codes can be either @@ -138,25 +148,12 @@ summarizing the outcome of the operation. The error codes can be either .I J_ENOJOURNAL (no journal associated with that file) or .I J_ENOMEM -(not enough free memory). There is also a program named +(not enough free memory), and +.I J_ECLEANUP +(error cleaning the journal directory). There is also a program named .I jiofsck which is just a simple human frontend to this function. -The second, -.BR jfsck_cleanup() , -is intended to be used after -.B jfsck() -by programs wanting to remove all the stall transaction files and leave the -journal directory ready to use. After calling -.BR jfsck() , -those transaction files will no longer be needed, so by cleaning up the -directory you make sure you're starting over with a clean journal. It returns -0 if there was an error, or 1 if it succeeded. The aforementioned -.I jiofsck -can also optionally invoke this function after performing the regular checks. -Calling it after -.B jfsck() -is highly recommended. .SS UNIX-alike API @@ -167,7 +164,7 @@ The UNIX-alike API, as explained before, consists of the functions They are all exactly like the UNIX equivalent, and behave the same way, with the only exception that instead of a file descriptor you need to pass a pointer to a -.IR "struct jfs" . +.IR "jfs_t" . Again, I will not duplicate the manpage for all these functions, just refer to the regular UNIX versions to see how to use them, they all have the same semantics and behave the same way. @@ -175,16 +172,17 @@ semantics and behave the same way. .SS BASIC FUNCTIONS The basic functions are the ones which manipulate transactions directly: -.BR jtrans_init() ", " jtrans_add() ", " jtrans_commit() ", " jtrans_rollback() +.BR jtrans_new() ", " jtrans_add() ", " jtrans_commit() ", " jtrans_rollback() and .BR jtrans_free() . These are intended to be use when your application requires direct control over the transactions. -.BR jtrans_init() " and " jtrans_free() -just initialize and free a given transaction structure; the former should be -called prior any use, and the latter when you want to destroy a transaction. -Note that +.BR jtrans_new() " and " jtrans_free() +just return a new +.I jtrans_t +and free a given one; the former should be called prior any use, and the +latter when you want to destroy a transaction. Note that .B jtrans_free() is not a disk operation, but only frees the pointers that were previously allocated by the library; all disk operations are performed by the other two @@ -200,17 +198,20 @@ will be applied in order. .B jtrans_commit() commits the given transaction to disk. After it has returned, data has been -saved to the disk. It returns the number of bytes written or -1 if there was -an error. The commit operation is atomic with regards to other read or write -operations on different processes, as long as they all access it via libjio. +saved to the disk. The commit operation is atomic with regards to other read +or write operations on different processes, as long as they all access it via +libjio. It returns the number of bytes written, -1 if there was an error but +atomic warantees were preserved, or -2 if there was an error and there is a +possible break of atomic warantees (which is an indication of a severe +underlying condition). .B jtrans_rollback() reverses a transaction that was applied with .BR jtrans_commit() , and leaves the file as it was before applying it. Be very very careful with this function, it's quite dangerous if you don't know for sure that you're -doing the right thing. It returns the number of bytes written or -1 if there -was an error. +doing the right thing. It returns the same values as +.BR jtrans_commit() . .SH SEE ALSO diff --git a/libjio/libjio.h b/libjio/libjio.h index b092a48..4421d25 100644 --- a/libjio/libjio.h +++ b/libjio/libjio.h @@ -23,144 +23,446 @@ #error "You must compile your application with Large File Support" #endif +/* + * Opaque types, the API does not expose these + */ -/* logical structures */ - -/* the main file structure */ -struct jfs { - int fd; /* main file descriptor */ - char *name; /* and its name */ - char *jdir; /* journal directory */ - int jdirfd; /* journal directory file descriptor */ - int jfd; /* journal's lock file descriptor */ - unsigned int *jmap; /* journal's lock file mmap area */ - uint32_t flags; /* journal flags */ - struct jlinger *ltrans; /* lingered transactions */ - pthread_mutex_t ltlock; /* lingered transactions' lock */ - pthread_mutex_t lock; /* a soft lock used in some operations */ -}; +/** An open file, similar to a file descriptor. */ +typedef struct jfs jfs_t; -/* a single operation */ -struct joper { - int locked; /* is the region is locked? */ - off_t offset; /* operation's offset */ - size_t len; /* data length */ - void *buf; /* data */ - size_t plen; /* previous data length */ - void *pdata; /* previous data */ - struct joper *prev; - struct joper *next; -}; +/** A single transaction. */ +typedef struct jtrans jtrans_t; -/* a transaction */ -struct jtrans { - struct jfs *fs; /* journal file structure to operate on */ - char *name; /* name of the transaction file */ - int id; /* transaction id */ - uint32_t flags; /* transaction flags */ - unsigned int numops; /* quantity of operations in the list */ - ssize_t len; /* transaction's length */ - pthread_mutex_t lock; /* used to modify the operation list */ - struct joper *op; /* list of operations */ -}; -/* lingered transaction */ -struct jlinger { - int id; /* transaction id */ - char *name; /* name of the transaction file */ - struct jlinger *next; -}; +/* + * Public types + */ +/** The result of a jfsck() run. + * + * @see jfsck() + * @ingroup check + */ struct jfsck_result { - int total; /* total transactions files we looked at */ - int invalid; /* invalid files in the journal directory */ - int in_progress; /* transactions in progress */ - int broken; /* transactions broken */ - int corrupt; /* corrupt transactions */ - int apply_error; /* errors applying the transaction */ - int reapplied; /* transactions that were reapplied */ -}; + /** Total transactions files processed */ + int total; + + /** Number of invalid transactions */ + int invalid; + /** Number of transactions in progress */ + int in_progress; -/* on-disk structures */ + /** Number of broken transactions */ + int broken; -/* header (fixed length, defined below) */ -struct disk_header { - uint32_t id; /* id */ - uint32_t flags; /* flags about this transaction */ - uint32_t numops; /* number of operations */ + /** Number of corrupt transactions */ + int corrupt; + + /** Number of errors applying transactions */ + int apply_error; + + /** Number of transactions successfully reapplied */ + int reapplied; }; -/* operation */ -struct disk_operation { - uint32_t len; /* data length */ - uint32_t plen; /* previous data length */ - uint64_t offset; /* offset relative to the BOF */ - char *prevdata; /* previous data for rollback */ +/** jfsck() return values. + * + * @see jfsck() + * @ingroup check + */ +enum jfsck_return { + /** Success */ + J_ESUCCESS = 0, + + /** No such file or directory */ + J_ENOENT = -1, + + /** No journal associated with the given file */ + J_ENOJOURNAL = -2, + + /** Not enough free memory */ + J_ENOMEM = -3, + + /** Error cleaning the journal directory */ + J_ECLEANUP = -4, }; -/* core functions */ -int jopen(struct jfs *fs, const char *name, int flags, int mode, int jflags); -void jtrans_init(struct jfs *fs, struct jtrans *ts); -int jtrans_add(struct jtrans *ts, const void *buf, size_t count, off_t offset); -ssize_t jtrans_commit(struct jtrans *ts); -ssize_t jtrans_rollback(struct jtrans *ts); -void jtrans_free(struct jtrans *ts); -int jsync(struct jfs *fs); -int jmove_journal(struct jfs *fs, const char *newpath); -int jclose(struct jfs *fs); - - -/* journal checker */ -int jfsck(const char *name, const char *jdir, struct jfsck_result *res); -int jfsck_cleanup(const char *name, const char *jdir); - -/* UNIX API wrappers */ -ssize_t jread(struct jfs *fs, void *buf, size_t count); -ssize_t jpread(struct jfs *fs, void *buf, size_t count, off_t offset); -ssize_t jreadv(struct jfs *fs, const struct iovec *vector, int count); -ssize_t jwrite(struct jfs *fs, const void *buf, size_t count); -ssize_t jpwrite(struct jfs *fs, const void *buf, size_t count, off_t offset); -ssize_t jwritev(struct jfs *fs, const struct iovec *vector, int count); -int jtruncate(struct jfs *fs, off_t length); -off_t jlseek(struct jfs *fs, off_t offset, int whence); - -/* ANSI C stdio wrappers */ -struct jfs *jfopen(const char *path, const char *mode); -int jfclose(struct jfs *stream); -struct jfs *jfreopen(const char *path, const char *mode, struct jfs *stream); -size_t jfread(void *ptr, size_t size, size_t nmemb, struct jfs *stream); -size_t jfwrite(const void *ptr, size_t size, size_t nmemb, struct jfs *stream); -int jfileno(struct jfs *stream); -int jfeof(struct jfs *stream); -void jclearerr(struct jfs *stream); -int jferror(struct jfs *stream); -int jfseek(struct jfs *stream, long offset, int whence); -long jftell(struct jfs *stream); -void jrewind(struct jfs *stream); -FILE *jfsopen(struct jfs *stream, const char *mode); - - -/* jfs and jtrans constants */ -#define J_NOLOCK 1 /* don't lock the file before operating on it */ -#define J_NOROLLBACK 2 /* no need to read rollback information */ -#define J_LINGER 4 /* use lingering transactions */ -#define J_COMMITTED 8 /* mark a transaction as committed */ -#define J_ROLLBACKED 16 /* mark a transaction as rollbacked */ -#define J_ROLLBACKING 32 /* mark a transaction as rollbacking */ -#define J_RDONLY 64 /* mark a file as read-only */ - -/* disk constants */ -#define J_DISKHEADSIZE 12 /* length of disk_header */ -#define J_DISKOPHEADSIZE 16 /* length of disk_operation header */ - -/* jfsck constants (return values) */ -#define J_ESUCCESS 0 /* success - shouldn't be used */ -#define J_ENOENT -1 /* no such file */ -#define J_ENOJOURNAL -2 /* no journal associated */ -#define J_ENOMEM -3 /* no enough free memory */ +/* + * Core functions + */ + +/** Open a file. + * + * Takes the same parameters as the UNIX open(2), with an additional one for + * internal flags. + * + * The only supported internal flag is J_LINGER, which enables lingering + * transactions. + * + * @param name path to the file to open + * @param flags flags to pass to open(2) + * @param mode mode to pass to open(2) + * @param jflags journal flags + * @returns a new jfs_t that identifies the open file on success, or NULL on + * error + * @see jclose(), open() + * @ingroup basic + */ +jfs_t *jopen(const char *name, int flags, int mode, int jflags); + +/** Close a file opened with jopen(). + * + * After a call to this function, the memory allocated for the open file will + * be freed. + * + * If there was an autosync thread started for this file, it will be stopped. + * + * @param fs open file + * @returns 0 on success, -1 on error + * @see jopen(), jfs_autosync_start() + * @ingroup basic + */ +int jclose(jfs_t *fs); + +/** Sync a file. Makes sense only when using lingering transactions. + * + * @param fs open file + * @returns 0 on success, -1 on error + * @ingroup basic + */ +int jsync(jfs_t *fs); + +/** Create a new transaction. + * + * @param fs open file the transaction will apply to + * @returns a new transaction (must be freed using jtrans_free()) + * @see jtrans_free() + * @ingroup basic + */ +jtrans_t *jtrans_new(jfs_t *fs); + +/** Add an operation to a transaction. + * + * An operation consists of a buffer, its length, and the offset to write it + * to. + * + * The file will not be touched (not even locked) until commit time, where the + * first count bytes of buf will be written at offset. + * + * Transactions will be applied in order, and overlapping operations are + * permitted, in which case the latest one will prevail. + * + * @param ts transaction + * @param buf buffer to write + * @param count how many bytes from the buffer to write + * @param offset offset to write at + * @returns 0 on success, -1 on error + * @ingroup basic + */ +int jtrans_add(jtrans_t *ts, const void *buf, size_t count, off_t offset); + +/** Commit a transaction. + * + * All the operations added to it using jtrans_add() will be written to disk, + * in the same order they were added. + * + * After this function returns successfully, all the data can be trusted to be + * on the disk. The commit is atomic with regards to other processes using + * libjio, but not accessing directly to the file. + * + * @param ts transaction + * @returns the amount of bytes written to disk, or -1 if there was an error + * but atomic warranties were preserved, or -2 if there was an error and + * there is a possible break of atomic warranties (which is an indication + * of a severe underlying condition). + * @ingroup basic + */ +ssize_t jtrans_commit(jtrans_t *ts); + +/** Rollback a transaction. + * + * This function atomically undoes a previous committed transaction. After its + * successful return, the data can be trusted to be on disk. + * + * Use with care. + * + * @param ts transaction + * @returns the same as jtrans_commit() + * @see jtrans_commit() + * @ingroup basic + */ +ssize_t jtrans_rollback(jtrans_t *ts); + +/** Free a transaction structure. + * + * @param ts transaction to free + * @see jtrans_new() + * @ingroup basic + */ +void jtrans_free(jtrans_t *ts); + +/** Change the location of the journal directory. + * + * The file MUST NOT be in use by any other thread or process. The older + * journal directory will be removed. + * + * @param fs open file + * @param newpath path to the new journal directory, which will be created if + * it doesn't exist + * @returns 0 on success, -1 on error + * @ingroup basic + */ +int jmove_journal(jfs_t *fs, const char *newpath); + + +/* + * Autosync + */ + +/** Start an autosync thread. + * + * The thread will call jsync(fs) every max_sec seconds, or every max_bytes + * have been written. Only one autosync thread per open file is allowed. + * + * @param fs open file + * @param max_sec maximum number of seconds that should pass between each + * call to jsync() + * @param max_bytes maximum number of bytes that should be written between + * each call to jsync() + * @returns 0 on success, -1 on error + * @ingroup basic + */ +int jfs_autosync_start(jfs_t *fs, time_t max_sec, size_t max_bytes); + +/** Stop an autosync thread that was started using jfs_autosync_start(fs). + * + * @param fs open file + * @returns 0 on success, -1 on error + * @ingroup basic + */ +int jfs_autosync_stop(jfs_t *fs); + + +/* + * Journal checker + */ + +/** Check and repair the given path. + * + * The file MUST NOT be in use by any other thread or process. This + * requirement will be lifted in future releases. + * + * @param name path to the file to check + * @param jdir journal directory of the given file, use NULL for the default + * @param res structure where to store the result + * @param flags flags that change the checking behaviour, currently only + * J_NOCLEANUP is supported, which avoids cleaning up the journal + * directory after a successful recovery + * @see struct jfsck_result + * @returns 0 on success, < 0 on error, with the following possible negative + * values from enum jfsck_return: J_ENOENT if there was no such file with + * the given name, J_ENOJOURNAL if there was no journal at the given + * jdir, J_ENOMEM if memory could not be allocated, J_ECLEANUP if there + * was an error cleaning the journal. + * @ingroup check + */ +enum jfsck_return jfsck(const char *name, const char *jdir, + struct jfsck_result *res, unsigned int flags); + + +/* + * UNIX API wrappers + */ + +/** Read from the file. Works just like UNIX read(2). + * + * @param fs file to read from + * @param buf buffer used to store the data + * @param count maximum number of bytes to read + * @returns number of bytes read on success, or -1 on error + * @see read(2) + * @ingroup unix + */ +ssize_t jread(jfs_t *fs, void *buf, size_t count); + +/** Read from the file at the given offset. Works just like UNIX pread(2). + * + * @param fs file to read from + * @param buf buffer used to store the data + * @param count maximum number of bytes to read + * @param offset offset to read at + * @returns number of bytes read on success, or -1 on error + * @see pread(2) + * @ingroup unix + */ +ssize_t jpread(jfs_t *fs, void *buf, size_t count, off_t offset); + +/** Read from the file into multiple buffers. Works just like UNIX readv(2). + * + * @param fs file to read from + * @param vector buffers used to store the data + * @param count maximum number of bytes to read + * @returns number of bytes read on success, or -1 on error + * @see readv(2) + * @ingroup unix + */ +ssize_t jreadv(jfs_t *fs, const struct iovec *vector, int count); + +/** Write to the file. Works just like UNIX write(2). + * + * @param fs file to write to + * @param buf buffer used to read the data from + * @param count maximum number of bytes to write + * @returns number of bytes written on success, or -1 on error + * @see write(2) + * @ingroup unix + */ +ssize_t jwrite(jfs_t *fs, const void *buf, size_t count); + +/** Write to the file at the given offset. Works just like UNIX pwrite(2). + * + * @param fs file to write to + * @param buf buffer used to read the data from + * @param count maximum number of bytes to write + * @param offset offset to write at + * @returns number of bytes written on success, or -1 on error + * @see pwrite(2) + * @ingroup unix + */ +ssize_t jpwrite(jfs_t *fs, const void *buf, size_t count, off_t offset); + +/** Write to the file from multiple buffers. Works just like UNIX writev(2). + * + * @param fs file to write to + * @param vector buffers used to read the data from + * @param count maximum number of bytes to write + * @returns number of bytes written on success, or -1 on error + * @see writev(2) + * @ingroup unix + */ +ssize_t jwritev(jfs_t *fs, const struct iovec *vector, int count); + +/** Truncates the file to the given length. Works just like UNIX ftruncate(2). + * + * @param fs file to truncate + * @param length lenght to truncate to + * @returns 0 on success, -1 on error + * @see ftruncate(2) + * @ingroup unix + */ +int jtruncate(jfs_t *fs, off_t length); + +/** Reposition read/write file offset. Works just like UNIX lseek(2). + * + * @param fs file to change the offset to + * @param offset offset to set + * @param whence where to count offset from, can be SEEK_SET to count from the + * beginning of the file, SEEK_CUR to count from the current position, or + * SEEK_END to count from the end. + * @returns the new offset counted from the beginning of the file, or -1 on + * error. + * @ingroup unix + */ +off_t jlseek(jfs_t *fs, off_t offset, int whence); + + +/* + * ANSI C stdio wrappers + */ + +jfs_t *jfopen(const char *path, const char *mode); + +int jfclose(jfs_t *stream); + +jfs_t *jfreopen(const char *path, const char *mode, jfs_t *stream); + +size_t jfread(void *ptr, size_t size, size_t nmemb, jfs_t *stream); + +size_t jfwrite(const void *ptr, size_t size, size_t nmemb, jfs_t *stream); + +int jfileno(jfs_t *stream); + +int jfeof(jfs_t *stream); + +void jclearerr(jfs_t *stream); + +int jferror(jfs_t *stream); + +int jfseek(jfs_t *stream, long offset, int whence); + +long jftell(jfs_t *stream); + +void jrewind(jfs_t *stream); + +FILE *jfsopen(jfs_t *stream, const char *mode); + + +/* + * jopen() flags. + * + * Internally used also for jtrans_t flags. + */ + +/** Don't lock the file before operating on it. + * + * @see jopen() + * @ingroup basic */ +#define J_NOLOCK 1 + +/** No need to read rollback information. + * + * @see jopen() + * @ingroup basic */ +#define J_NOROLLBACK 2 + +/** Use lingering transactions. + * + * @see jopen() + * @ingroup basic */ +#define J_LINGER 4 + +/* Range 8-256 is reserved for future public use */ + +/** Marks a file as read-only. + * + * For internal use only, automatically set when O_RDONLY is passed to + * jopen(). + * + * @internal */ +#define J_RDONLY 512 + + +/* + * jtrans_t flags. + * + * For internal use only, but must be in the same space as the jopen() flags. + */ + +/** Marks a transaction as committed. + * @internal */ +#define J_COMMITTED 1024 + +/** Marks a transaction as rollbacked. + * @internal */ +#define J_ROLLBACKED 2048 + +/** Marks a transaction as rollbacking. + * @internal */ +#define J_ROLLBACKING 4096 + + +/* + * jfsck() flags + */ + +/** Do not perform a journal cleanup. Used in jfsck(). + * + * @see jfsck() + * @ingroup check */ +#define J_NOCLEANUP 1 #endif diff --git a/libjio/trans.c b/libjio/trans.c index 6fb6885..07d7bfe 100644 --- a/libjio/trans.c +++ b/libjio/trans.c @@ -1,9 +1,6 @@ /* - * libjio - A library for Journaled I/O - * Alberto Bertogli (albertito@blitiri.com.ar) - * - * Core transaction API and recovery functions + * Core transaction API */ #include @@ -22,86 +19,25 @@ #include "libjio.h" #include "common.h" #include "compat.h" +#include "journal.h" +#include "trans.h" /* - * helper functions + * Transaction functions */ -/* gets a new transaction id */ -static unsigned int get_tid(struct jfs *fs) -{ - unsigned int curid, rv; - - /* lock the whole file */ - plockf(fs->jfd, F_LOCKW, 0, 0); - - /* read the current max. curid */ - curid = *(fs->jmap); - - fiu_do_on("jio/get_tid/overflow", curid = -1); - - /* increment it and handle overflows */ - rv = curid + 1; - if (rv == 0) - goto exit; - - /* write to the file descriptor */ - *(fs->jmap) = rv; - -exit: - plockf(fs->jfd, F_UNLOCK, 0, 0); - return rv; -} - -/* frees a transaction id */ -static void free_tid(struct jfs *fs, unsigned int tid) -{ - unsigned int curid, i; - char name[PATH_MAX]; - - /* lock the whole file */ - plockf(fs->jfd, F_LOCKW, 0, 0); - - /* read the current max. curid */ - curid = *(fs->jmap); - - /* if we're the max tid, scan the directory looking up for the new - * max; the detailed description can be found in the "doc/" dir */ - if (tid == curid) { - /* look up the new max. */ - for (i = curid - 1; i > 0; i--) { - get_jtfile(fs, i, name); - if (access(name, R_OK | W_OK) == 0) { - break; - } else if (errno != EACCES) { - /* Real error, stop looking for a new max. It - * doesn't hurt us because it's ok if the max - * is higher than it could be */ - break; - } - } - - /* and save it */ - *(fs->jmap) = i; - } - - plockf(fs->jfd, F_UNLOCK, 0, 0); - return; -} - - -/* - * transaction functions - */ - -/* initialize a transaction structure */ -void jtrans_init(struct jfs *fs, struct jtrans *ts) +/* Initialize a transaction structure */ +struct jtrans *jtrans_new(struct jfs *fs) { pthread_mutexattr_t attr; + struct jtrans *ts; + + ts = malloc(sizeof(struct jtrans)); + if (ts == NULL) + return NULL; ts->fs = fs; - ts->name = NULL; ts->id = 0; ts->flags = fs->flags; ts->op = NULL; @@ -111,19 +47,17 @@ void jtrans_init(struct jfs *fs, struct jtrans *ts) pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_NORMAL); pthread_mutex_init( &(ts->lock), &attr); pthread_mutexattr_destroy(&attr); -} + return ts; +} -/* free the contents of a transaction structure */ +/* Free the contents of a transaction structure */ void jtrans_free(struct jtrans *ts) { struct joper *tmpop; ts->fs = NULL; - if (ts->name) - free(ts->name); - while (ts->op != NULL) { tmpop = ts->op->next; @@ -136,9 +70,11 @@ void jtrans_free(struct jtrans *ts) ts->op = tmpop; } pthread_mutex_destroy(&(ts->lock)); -} + free(ts); +} +/* Add an operation to a transaction */ int jtrans_add(struct jtrans *ts, const void *buf, size_t count, off_t offset) { struct joper *jop, *tmpop; @@ -148,12 +84,12 @@ int jtrans_add(struct jtrans *ts, const void *buf, size_t count, off_t offset) /* fail for read-only accesses */ if (ts->flags & J_RDONLY) { pthread_mutex_unlock(&(ts->lock)); - return 0; + return -1; } if ((long long) ts->len + count > MAX_TSIZE) { pthread_mutex_unlock(&(ts->lock)); - return 0; + return -1; } /* find the last operation in the transaction and create a new one at @@ -162,7 +98,7 @@ int jtrans_add(struct jtrans *ts, const void *buf, size_t count, off_t offset) ts->op = malloc(sizeof(struct joper)); if (ts->op == NULL) { pthread_mutex_unlock(&(ts->lock)); - return 0; + return -1; } jop = ts->op; jop->prev = NULL; @@ -172,7 +108,7 @@ int jtrans_add(struct jtrans *ts, const void *buf, size_t count, off_t offset) tmpop->next = malloc(sizeof(struct joper)); if (tmpop->next == NULL) { pthread_mutex_unlock(&(ts->lock)); - return 0; + return -1; } tmpop->next->prev = tmpop; jop = tmpop->next; @@ -188,7 +124,7 @@ int jtrans_add(struct jtrans *ts, const void *buf, size_t count, off_t offset) } free(jop); pthread_mutex_unlock(&(ts->lock)); - return 0; + return -1; } ts->numops++; @@ -204,24 +140,22 @@ int jtrans_add(struct jtrans *ts, const void *buf, size_t count, off_t offset) jop->pdata = NULL; jop->locked = 0; - /* it's highly likely that jtrans_commit() will want to read the - * current data, so we tell the kernel about that */ - posix_fadvise(ts->fs->fd, offset, count, POSIX_FADV_WILLNEED); + if (!(ts->flags & J_NOROLLBACK)) { + /* jtrans_commit() will want to read the current data, so we + * tell the kernel about that */ + posix_fadvise(ts->fs->fd, offset, count, POSIX_FADV_WILLNEED); + } - return 1; + return 0; } -/* commit a transaction */ +/* Commit a transaction */ ssize_t jtrans_commit(struct jtrans *ts) { - int id, fd = -1; ssize_t rv; - uint32_t csum; - char *name; - unsigned char *buf_init, *bufp; struct joper *op; struct jlinger *linger; - off_t curpos = 0; + jop_t *jop; size_t written = 0; pthread_mutex_lock(&(ts->lock)); @@ -234,56 +168,6 @@ ssize_t jtrans_commit(struct jtrans *ts) if (ts->flags & J_RDONLY) goto exit; - name = (char *) malloc(PATH_MAX); - if (name == NULL) - goto exit; - - id = get_tid(ts->fs); - if (id == 0) - goto exit; - - /* open the transaction file */ - get_jtfile(ts->fs, id, name); - fd = open(name, O_RDWR | O_CREAT | O_TRUNC, 0600); - if (fd < 0) - goto exit; - - fiu_exit_on("jio/commit/created_tf"); - - /* and lock it */ - plockf(fd, F_LOCKW, 0, 0); - - ts->id = id; - ts->name = name; - - /* save the header */ - buf_init = malloc(J_DISKHEADSIZE); - if (buf_init == NULL) - goto unlink_exit; - - bufp = buf_init; - - memcpy(bufp, (void *) &(ts->id), 4); - bufp += 4; - - memcpy(bufp, (void *) &(ts->flags), 4); - bufp += 4; - - memcpy(bufp, (void *) &(ts->numops), 4); - bufp += 4; - - rv = spwrite(fd, buf_init, J_DISKHEADSIZE, 0); - if (rv != J_DISKHEADSIZE) { - free(buf_init); - goto unlink_exit; - } - - fiu_exit_on("jio/commit/tf_header"); - - free(buf_init); - - curpos = J_DISKHEADSIZE; - /* first of all lock all the regions we're going to work with; * otherwise there could be another transaction trying to write the * same spots and we could end up with interleaved writes, that could @@ -294,106 +178,18 @@ ssize_t jtrans_commit(struct jtrans *ts) lr = plockf(ts->fs->fd, F_LOCKW, op->offset, op->len); if (lr == -1) /* note it can fail with EDEADLK */ - goto unlink_exit; + goto unlock_exit; op->locked = 1; } } - /* save each transacion in the file */ - for (op = ts->op; op != NULL; op = op->next) { - /* read the current content only if the transaction is not - * marked as NOROLLBACK, and if the data is not there yet, - * which is the normal case, but for rollbacking we fill it - * ourselves */ - if (!(ts->flags & J_NOROLLBACK) && (op->pdata == NULL)) { - op->pdata = malloc(op->len); - if (op->pdata == NULL) - goto unlink_exit; - - op->plen = op->len; - - rv = spread(ts->fs->fd, op->pdata, op->len, - op->offset); - if (rv < 0) - goto unlink_exit; - if (rv < op->len) { - /* we are extending the file! */ - /* ftruncate(ts->fs->fd, op->offset + op->len); */ - op->plen = rv; - } - } - - /* save the operation's header */ - buf_init = malloc(J_DISKOPHEADSIZE); - if (buf_init == NULL) - goto unlink_exit; - - bufp = buf_init; - - memcpy(bufp, (void *) &(op->len), 4); - bufp += 4; - - memcpy(bufp, (void *) &(op->plen), 4); - bufp += 4; - - memcpy(bufp, (void *) &(op->offset), 8); - bufp += 8; - - rv = spwrite(fd, buf_init, J_DISKOPHEADSIZE, curpos); - if (rv != J_DISKOPHEADSIZE) { - free(buf_init); - goto unlink_exit; - } - - fiu_exit_on("jio/commit/tf_ophdr"); - - free(buf_init); - - curpos += J_DISKOPHEADSIZE; - - /* and save it to the disk */ - rv = spwrite(fd, op->buf, op->len, curpos); - if (rv != op->len) - goto unlink_exit; - - curpos += op->len; - - fiu_exit_on("jio/commit/tf_opdata"); - } - - fiu_exit_on("jio/commit/tf_data"); - - /* compute and save the checksum (curpos is always small, so there's - * no overflow possibility when we convert to size_t) */ - if (!checksum(fd, curpos, &csum)) - goto unlink_exit; + jop = journal_new(ts); + if (jop == NULL) + goto unlock_exit; - rv = spwrite(fd, &csum, sizeof(uint32_t), curpos); - if (rv != sizeof(uint32_t)) + rv = journal_save(jop); + if (rv < 0) goto unlink_exit; - curpos += sizeof(uint32_t); - - /* this is a simple but efficient optimization: instead of doing - * everything O_SYNC, we sync at this point only, this way we avoid - * doing a lot of very small writes; in case of a crash the - * transaction file is only useful if it's complete (ie. after this - * point) so we only flush here (both data and metadata) */ - if (fsync(fd) != 0) - goto unlink_exit; - if (fsync(ts->fs->jdirfd) != 0) { - /* it seems to be legal that fsync() on directories is not - * implemented, so if this fails with EINVAL or EBADF, just - * call a global sync(); which is awful (and might still - * return before metadata is done) but it seems to be the - * saner choice; otherwise we just fail */ - if (errno == EINVAL || errno == EBADF) { - sync(); - } else { - goto unlink_exit; - } - } - - fiu_exit_on("jio/commit/tf_sync"); /* now that we have a safe transaction file, let's apply it */ written = 0; @@ -403,6 +199,14 @@ ssize_t jtrans_commit(struct jtrans *ts) goto rollback_exit; written += rv; + + if (have_sync_range && !(ts->flags & J_LINGER)) { + rv = sync_range_submit(ts->fs->fd, op->len, + op->offset); + if (rv != 0) + goto rollback_exit; + } + fiu_exit_on("jio/commit/wrote_op"); } @@ -413,40 +217,54 @@ ssize_t jtrans_commit(struct jtrans *ts) if (linger == NULL) goto rollback_exit; - linger->id = id; - linger->name = strdup(name); + linger->jop = jop; pthread_mutex_lock(&(ts->fs->ltlock)); linger->next = ts->fs->ltrans; ts->fs->ltrans = linger; + ts->fs->ltrans_len += written; + autosync_check(ts->fs); pthread_mutex_unlock(&(ts->fs->ltlock)); } else { - if (fdatasync(ts->fs->fd) != 0) - goto rollback_exit; + if (have_sync_range) { + for (op = ts->op; op != NULL; op = op->next) { + rv = sync_range_wait(ts->fs->fd, op->len, + op->offset); + if (rv != 0) + goto rollback_exit; + } + } else { + if (fdatasync(ts->fs->fd) != 0) + goto rollback_exit; + } /* the transaction has been applied, so we cleanup and remove * it from the disk */ - unlink(name); - fiu_exit_on("jio/commit/pre_ok_free_tid"); - free_tid(ts->fs, ts->id); + rv = journal_free(jop); + if (rv != 0) + goto rollback_exit; } + jop = NULL; + /* mark the transaction as committed, _after_ it was removed */ ts->flags = ts->flags | J_COMMITTED; rollback_exit: - /* If the transaction failed we try to recover by rollbacking it + /* If the transaction failed we try to recover by rolling it back. + * * NOTE: on extreme conditions (ENOSPC/disk failure) this can fail * too! There's nothing much we can do in that case, the caller should * take care of it by itself. + * * The transaction file might be OK at this point, so the data could * be recovered by a posterior jfsck(); however, that's not what the * user expects (after all, if we return failure, new data should - * never appear), so we remove the transaction file. - * Transactions that were successfuly recovered by rollbacking them - * will have J_ROLLBACKED in their flags, so the caller can verify if - * the failure was recovered or not. */ + * never appear), so we remove the transaction file (see unlink_exit). + * + * Transactions that were successfuly recovered by rolling them back + * will have J_ROLLBACKED in their flags */ if (!(ts->flags & J_COMMITTED) && !(ts->flags & J_ROLLBACKING)) { rv = ts->flags; ts->flags = ts->flags | J_NOLOCK | J_ROLLBACKING; @@ -458,16 +276,13 @@ rollback_exit: } unlink_exit: - if (!(ts->flags & J_COMMITTED)) { - unlink(name); - free_tid(ts->fs, ts->id); - } - - close(fd); + if (jop) + journal_free(jop); +unlock_exit: /* always unlock everything at the end; otherwise we could have * half-overlapping transactions applying simultaneously, and if - * anything goes wrong it's possible to break consistency */ + * anything goes wrong it would be possible to break consistency */ if (!(ts->flags & J_NOLOCK)) { for (op = ts->op; op != NULL; op = op->next) { if (op->locked) { @@ -483,20 +298,22 @@ exit: /* return the length only if it was properly committed */ if (ts->flags & J_COMMITTED) return written; - else + else if (ts->flags & J_ROLLBACKED) return -1; + else + return -2; } -/* rollback a transaction */ +/* Rollback a transaction */ ssize_t jtrans_rollback(struct jtrans *ts) { ssize_t rv; - struct jtrans newts; + struct jtrans *newts; struct joper *op, *curop, *lop; - jtrans_init(ts->fs, &newts); - newts.flags = ts->flags; - newts.numops = ts->numops; + newts = jtrans_new(ts->fs); + newts->flags = ts->flags; + newts->numops = ts->numops; if (ts->op == NULL || ts->flags & J_NOROLLBACK) { rv = -1; @@ -537,12 +354,12 @@ ssize_t jtrans_rollback(struct jtrans *ts) curop->locked = 0; /* add the new transaction to the list */ - if (newts.op == NULL) { - newts.op = curop; + if (newts->op == NULL) { + newts->op = curop; curop->prev = NULL; curop->next = NULL; } else { - for (lop = newts.op; lop->next != NULL; lop = lop->next) + for (lop = newts->op; lop->next != NULL; lop = lop->next) ; lop->next = curop; curop->prev = lop; @@ -550,37 +367,44 @@ ssize_t jtrans_rollback(struct jtrans *ts) } } - rv = jtrans_commit(&newts); + rv = jtrans_commit(newts); exit: /* free the transaction */ - for (curop = newts.op; curop != NULL; curop = curop->next) { + for (curop = newts->op; curop != NULL; curop = curop->next) { curop->buf = NULL; curop->pdata = NULL; } - jtrans_free(&newts); + jtrans_free(newts); return rv; } + /* - * basic operations + * Basic operations */ -/* open a file */ -int jopen(struct jfs *fs, const char *name, int flags, int mode, int jflags) +/* Open a file */ +struct jfs *jopen(const char *name, int flags, int mode, int jflags) { int jfd, rv; unsigned int t; char jdir[PATH_MAX], jlockfile[PATH_MAX]; struct stat sinfo; pthread_mutexattr_t attr; + struct jfs *fs; + + fs = malloc(sizeof(struct jfs)); + if (fs == NULL) + return NULL; fs->fd = -1; fs->jfd = -1; fs->jdir = NULL; fs->jdirfd = -1; fs->jmap = MAP_FAILED; + fs->as_cfg = NULL; /* we provide either read-only or read-write access, because when we * commit a transaction we read the current contents before applying, @@ -597,7 +421,9 @@ int jopen(struct jfs *fs, const char *name, int flags, int mode, int jflags) fs->name = strdup(name); fs->flags = jflags; + fs->open_flags = flags; fs->ltrans = NULL; + fs->ltrans_len = 0; /* Note on fs->lock usage: this lock is used only to protect the file * pointer. This means that it must only be held while performing @@ -624,7 +450,7 @@ int jopen(struct jfs *fs, const char *name, int flags, int mode, int jflags) /* nothing else to do for read-only access */ if (jflags & J_RDONLY) { - return fs->fd; + return fs; } if (!get_jdir(name, jdir)) @@ -672,17 +498,17 @@ int jopen(struct jfs *fs, const char *name, int flags, int mode, int jflags) if (fs->jmap == MAP_FAILED) goto error_exit; - return fs->fd; + return fs; error_exit: /* if there was an error, clean up as much as possible so we don't * leak anything, and return failure; jclose just does this cleaning * for us */ jclose(fs); - return -1; + return NULL; } -/* sync a file (makes sense only if using lingering transactions) */ +/* Sync a file */ int jsync(struct jfs *fs) { int rv; @@ -691,28 +517,26 @@ int jsync(struct jfs *fs) if (fs->fd < 0) return -1; - rv = fsync(fs->fd); + rv = fdatasync(fs->fd); if (rv != 0) return rv; pthread_mutex_lock(&(fs->ltlock)); while (fs->ltrans != NULL) { - free_tid(fs, fs->ltrans->id); fiu_exit_on("jio/jsync/pre_unlink"); - unlink(fs->ltrans->name); - free(fs->ltrans->name); + journal_free(fs->ltrans->jop); ltmp = fs->ltrans->next; free(fs->ltrans); - fs->ltrans = ltmp; } + fs->ltrans_len = 0; pthread_mutex_unlock(&(fs->ltlock)); return 0; } -/* change the location of the journal directory */ +/* Change the location of the journal directory */ int jmove_journal(struct jfs *fs, const char *newpath) { int ret; @@ -774,13 +598,16 @@ exit: return ret; } -/* close a file */ +/* Close a file opened with jopen() */ int jclose(struct jfs *fs) { int ret; ret = 0; + if (jfs_autosync_stop(fs)) + ret = -1; + if (! (fs->flags & J_RDONLY)) { if (jsync(fs)) ret = -1; @@ -801,6 +628,8 @@ int jclose(struct jfs *fs) free(fs->jdir); pthread_mutex_destroy(&(fs->lock)); + free(fs); + return ret; } diff --git a/libjio/trans.h b/libjio/trans.h new file mode 100644 index 0000000..366585e --- /dev/null +++ b/libjio/trans.h @@ -0,0 +1,75 @@ + +#ifndef _TRANS_H +#define _TRANS_H + + +struct joper; + +/** A transaction */ +struct jtrans { + /** Journal file structure to operate on */ + struct jfs *fs; + + /** Transaction id */ + int id; + + /** Transaction flags */ + uint32_t flags; + + /** Number of operations in the list */ + unsigned int numops; + + /** Transaction's length */ + size_t len; + + /** Lock that protects the list of operations */ + pthread_mutex_t lock; + + /** List of operations */ + struct joper *op; +}; + +/* a single operation */ +struct joper { + int locked; /* is the region is locked? */ + off_t offset; /* operation's offset */ + size_t len; /* data length */ + void *buf; /* data */ + size_t plen; /* previous data length */ + void *pdata; /* previous data */ + struct joper *prev; + struct joper *next; +}; + +/* lingered transaction */ +struct journal_op; +struct jlinger { + struct journal_op *jop; + struct jlinger *next; +}; + + +/* on-disk structures */ + +/* header (fixed length, defined below) */ +struct disk_header { + uint32_t id; /* id */ + uint32_t flags; /* flags about this transaction */ + uint32_t numops; /* number of operations */ +}; + +/* operation */ +struct disk_operation { + uint32_t len; /* data length */ + uint32_t plen; /* previous data length */ + uint64_t offset; /* offset relative to the BOF */ + char *prevdata; /* previous data for rollback */ +}; + +/* disk constants */ +#define J_DISKHEADSIZE 12 /* length of disk_header */ +#define J_DISKOPHEADSIZE 16 /* length of disk_operation header */ + + +#endif + diff --git a/libjio/unix.c b/libjio/unix.c index a2d8e9e..1a67847 100644 --- a/libjio/unix.c +++ b/libjio/unix.c @@ -1,23 +1,24 @@ /* - * libjio - A library for Journaled I/O - * Alberto Bertogli (albertito@blitiri.com.ar) - * * UNIX API wrappers */ #include #include #include +#include #include #include "libjio.h" #include "common.h" +#include "trans.h" -/* read() family wrappers */ +/* + * read() family wrappers + */ -/* read wrapper */ +/* read() wrapper */ ssize_t jread(struct jfs *fs, void *buf, size_t count) { int rv; @@ -41,7 +42,7 @@ ssize_t jread(struct jfs *fs, void *buf, size_t count) return rv; } -/* pread wrapper */ +/* pread() wrapper */ ssize_t jpread(struct jfs *fs, void *buf, size_t count, off_t offset) { int rv; @@ -53,7 +54,7 @@ ssize_t jpread(struct jfs *fs, void *buf, size_t count, off_t offset) return rv; } -/* readv wrapper */ +/* readv() wrapper */ ssize_t jreadv(struct jfs *fs, const struct iovec *vector, int count) { int rv, i; @@ -75,88 +76,118 @@ ssize_t jreadv(struct jfs *fs, const struct iovec *vector, int count) } -/* write family wrappers */ +/* + * write() family wrappers + */ -/* write wrapper */ +/* write() wrapper */ ssize_t jwrite(struct jfs *fs, const void *buf, size_t count) { int rv; off_t pos; - struct jtrans ts; + struct jtrans *ts; + + ts = jtrans_new(fs); + if (ts == NULL) + return -1; pthread_mutex_lock(&(fs->lock)); - jtrans_init(fs, &ts); - pos = lseek(fs->fd, 0, SEEK_CUR); - jtrans_add(&ts, buf, count, pos); + if (fs->open_flags & O_APPEND) + pos = lseek(fs->fd, 0, SEEK_END); + else + pos = lseek(fs->fd, 0, SEEK_CUR); - rv = jtrans_commit(&ts); + rv = jtrans_add(ts, buf, count, pos); + if (rv < 0) + goto exit; + + rv = jtrans_commit(ts); if (rv > 0) { /* if success, advance the file pointer */ lseek(fs->fd, rv, SEEK_CUR); } +exit: + pthread_mutex_unlock(&(fs->lock)); - jtrans_free(&ts); + jtrans_free(ts); return rv; } -/* pwrite wrapper */ +/* pwrite() wrapper */ ssize_t jpwrite(struct jfs *fs, const void *buf, size_t count, off_t offset) { int rv; - struct jtrans ts; + struct jtrans *ts; + + ts = jtrans_new(fs); + if (ts == NULL) + return -1; - jtrans_init(fs, &ts); - jtrans_add(&ts, buf, count, offset); + rv = jtrans_add(ts, buf, count, offset); + if (rv < 0) + goto exit; - rv = jtrans_commit(&ts); + rv = jtrans_commit(ts); - jtrans_free(&ts); +exit: + jtrans_free(ts); return rv; } -/* writev wrapper */ +/* writev() wrapper */ ssize_t jwritev(struct jfs *fs, const struct iovec *vector, int count) { int rv, i; size_t sum; off_t ipos, t; - struct jtrans ts; + struct jtrans *ts; + + ts = jtrans_new(fs); + if (ts == NULL) + return -1; pthread_mutex_lock(&(fs->lock)); - jtrans_init(fs, &ts); - ipos = lseek(fs->fd, 0, SEEK_CUR); + if (fs->open_flags & O_APPEND) + ipos = lseek(fs->fd, 0, SEEK_END); + else + ipos = lseek(fs->fd, 0, SEEK_CUR); + t = ipos; sum = 0; for (i = 0; i < count; i++) { - jtrans_add(&ts, vector[i].iov_base, vector[i].iov_len, t); + rv = jtrans_add(ts, vector[i].iov_base, vector[i].iov_len, t); + if (rv < 0) + goto exit; + sum += vector[i].iov_len; t += vector[i].iov_len; } - rv = jtrans_commit(&ts); + rv = jtrans_commit(ts); if (rv > 0) { /* if success, advance the file pointer */ lseek(fs->fd, rv, SEEK_CUR); } +exit: pthread_mutex_unlock(&(fs->lock)); - jtrans_free(&ts); + jtrans_free(ts); return rv; } -/* truncate a file - be careful with this */ +/* Truncate a file. Be careful with this */ int jtruncate(struct jfs *fs, off_t length) { int rv; @@ -169,7 +200,7 @@ int jtruncate(struct jfs *fs, off_t length) return rv; } -/* lseek wrapper */ +/* lseek() wrapper */ off_t jlseek(struct jfs *fs, off_t offset, int whence) { off_t rv; diff --git a/samples/full.c b/samples/full.c index 83e069c..dd621b3 100644 --- a/samples/full.c +++ b/samples/full.c @@ -12,26 +12,25 @@ int main(void) { int r; - struct jfs file; - struct jtrans trans; + jfs_t *file; + struct jtrans *trans; struct jfsck_result result; /* check the file is OK */ - jfsck(FILENAME, NULL, &result); - jfsck_cleanup(FILENAME, NULL); + jfsck(FILENAME, NULL, &result, 0); /* and open it */ - r = jopen(&file, FILENAME, O_RDWR | O_CREAT | O_TRUNC, 0600, 0); - if (r < 0) { + file = jopen(FILENAME, O_RDWR | O_CREAT | O_TRUNC, 0600, 0); + if (file == NULL) { perror("jopen"); return 1; } /* write two "Hello world"s next to each other */ - jtrans_init(&file, &trans); - jtrans_add(&trans, TEXT, strlen(TEXT), 0); - jtrans_add(&trans, TEXT, strlen(TEXT), strlen(TEXT)); - r = jtrans_commit(&trans); + trans = jtrans_new(file); + jtrans_add(trans, TEXT, strlen(TEXT), 0); + jtrans_add(trans, TEXT, strlen(TEXT), strlen(TEXT)); + r = jtrans_commit(trans); if (r < 0) { perror("jtrans_commit"); return 1; @@ -40,7 +39,7 @@ int main(void) /* at this point the file has "Hello world!\nHello world!\n" */ /* now we rollback */ - r = jtrans_rollback(&trans); + r = jtrans_rollback(trans); if (r < 0) { perror("jtrans_rollback"); return 1; @@ -48,8 +47,8 @@ int main(void) /* and now the file is empty! */ - jtrans_free(&trans); - jclose(&file); + jtrans_free(trans); + jclose(file); return 0; } diff --git a/samples/jio1.c b/samples/jio1.c index 9fb4062..d45901f 100644 --- a/samples/jio1.c +++ b/samples/jio1.c @@ -12,17 +12,20 @@ static int jio(void) { - int fd, rv; - struct jfs fs; + int rv; + jfs_t *fs; - fd = jopen(&fs, "test1", O_RDWR | O_CREAT | O_TRUNC, 0660, 0); - if (fd < 0) + fs = jopen("test1", O_RDWR | O_CREAT | O_TRUNC, 0660, 0); + if (fs == NULL) perror("jopen()"); - rv = jwrite(&fs, STR, strlen(STR)); + rv = jwrite(fs, STR, strlen(STR)); if (rv != strlen(STR)) perror("jwrite()"); + if (jclose(fs)) + perror("jclose()"); + return 0; } diff --git a/samples/jio2.c b/samples/jio2.c index 3296dab..e720630 100644 --- a/samples/jio2.c +++ b/samples/jio2.c @@ -14,17 +14,20 @@ static int jio(void) { - int fd, rv; - struct jfs fs; + int rv; + jfs_t *fs; - fd = jopen(&fs, "test2", O_RDWR | O_CREAT | O_TRUNC, 0660, 0); - if (fd < 0) + fs = jopen("test2", O_RDWR | O_CREAT | O_TRUNC, 0660, 0); + if (fs == NULL) perror("jopen()"); - rv = jwrite(&fs, STR, strlen(STR)); + rv = jwrite(fs, STR, strlen(STR)); if (rv != strlen(STR)) perror("jwrite()"); + if (jclose(fs)) + perror("jclose()"); + return 0; } diff --git a/samples/jio3.c b/samples/jio3.c index d66ee8d..c71c727 100644 --- a/samples/jio3.c +++ b/samples/jio3.c @@ -11,35 +11,42 @@ int main(int argc, char **argv) { - int fd, rv; - struct jfs fs; - struct jtrans ts; + int rv; + jfs_t *fs; + jtrans_t *ts; - fd = jopen(&fs, "test3", O_RDWR | O_CREAT, 0660, 0); - if (fd < 0) + fs = jopen("test3", O_RDWR | O_CREAT, 0660, 0); + if (fs == NULL) perror("jopen()"); - jtrans_init(&fs, &ts); + ts = jtrans_new(fs); + if (ts == NULL) + perror("jtrans_new()"); #define str1 "1ROLLBACKTEST1!\n" - jtrans_add(&ts, str1, strlen(str1), 0); + jtrans_add(ts, str1, strlen(str1), 0); #define str2 "2ROLLBACKTEST2!\n" - jtrans_add(&ts, str2, strlen(str2), strlen(str1)); + jtrans_add(ts, str2, strlen(str2), strlen(str1)); #define str3 "3ROLLBACKTEST3!\n" - jtrans_add(&ts, str3, strlen(str3), strlen(str1) + strlen(str2)); + jtrans_add(ts, str3, strlen(str3), strlen(str1) + strlen(str2)); - rv = jtrans_commit(&ts); + rv = jtrans_commit(ts); if (rv != strlen(str1) + strlen(str2) + strlen(str3)) perror("jtrans_commit()"); printf("commit ok: %d\n", rv); - rv = jtrans_rollback(&ts); + rv = jtrans_rollback(ts); if (rv < 0) perror("jtrans_rollback()"); printf("rollback ok: %d\n", rv); + jtrans_free(ts); + + if (jclose(fs)) + perror("jclose()"); + return 0; } diff --git a/tests/behaviour/runtests b/tests/behaviour/runtests index 645a3bf..2860493 100755 --- a/tests/behaviour/runtests +++ b/tests/behaviour/runtests @@ -5,7 +5,8 @@ import tf possible_tests = ('normal', 'corruption', 'fi') if len(sys.argv) < 2 or sys.argv[1] not in possible_tests + ('all',): - print 'Usage: runtests', '|'.join(possible_tests + ('all',)) + print 'Usage: runtests', '|'.join(possible_tests + ('all',)), + print '[test_name]' sys.exit(1) if sys.argv[1] == 'all': @@ -13,7 +14,11 @@ if sys.argv[1] == 'all': else: mnames = ('t_' + sys.argv[1],) +specific_test = None +if len(sys.argv) >= 3: + specific_test = sys.argv[2] + for mn in mnames: print '--', mn - tf.autorun(__import__(mn)) + tf.autorun(__import__(mn), specific_test) diff --git a/tests/behaviour/tf.py b/tests/behaviour/tf.py index 016fd1e..786f24c 100644 --- a/tests/behaviour/tf.py +++ b/tests/behaviour/tf.py @@ -136,8 +136,6 @@ def fsck_verify(n, **kwargs): raise AssertionError, k + ' does not match: ' + \ str(res) - libjio.jfsck_cleanup(n) - def cleanup(path): """Unlinks the path and its temporary libjio directory. The libjio directory must only have the 'lock' file in it.""" @@ -216,7 +214,7 @@ def gen_ret_after(n, notyet, itstime): return newf -def autorun(module): +def autorun(module, specific_test = None): "Runs all the functions in the given module that begin with 'test'." for name in sorted(dir(module)): if not name.startswith('test'): @@ -227,6 +225,10 @@ def autorun(module): name = name[len('test'):] if name.startswith('_'): name = name[1:] + + if specific_test and name != specific_test: + continue + desc = '' if obj.__doc__: desc = obj.__doc__ diff --git a/tests/performance/Makefile b/tests/performance/Makefile index 11ddd33..8dae8a5 100644 --- a/tests/performance/Makefile +++ b/tests/performance/Makefile @@ -5,16 +5,20 @@ LIBS = -ljio default: all -all: performance +all: performance random performance: performance.o $(CC) $(LIBS) performance.o -o performance +random: random.o + $(CC) $(LIBS) random.o -o random + .c.o: $(CC) $(CFLAGS) -c $< -o $@ clean: rm -f performance.o performance + rm -f random.o random rm -f *.bb *.bbg *.da *.gcov gmon.out rm -f test_file rm -rf .test_file.jio diff --git a/tests/performance/performance.c b/tests/performance/performance.c index 849cd60..eb4f9ca 100644 --- a/tests/performance/performance.c +++ b/tests/performance/performance.c @@ -15,12 +15,13 @@ #include #include #include +#include #include #define FILENAME "test_file" /* These are shared among threads, to make the code simpler */ -static struct jfs fs; +static jfs_t *fs; static unsigned long mb; static ssize_t blocksize, towrite; @@ -53,13 +54,14 @@ static void *worker(void *tno) perror("malloc()"); return NULL; } + memset(buf, 5, blocksize); work_done = 0; gettimeofday(&tv1, NULL); while (work_done < towrite) { - rv = jpwrite(&fs, buf, blocksize, localoffset + work_done); + rv = jpwrite(fs, buf, blocksize, localoffset + work_done); if (rv != blocksize) { perror("jpwrite()"); break; @@ -83,12 +85,14 @@ static void *worker(void *tno) printf("%lu %zd %zd %f %f\n", tid, mb, blocksize, seconds, mb_per_sec); + free(buf); + return NULL; } int main(int argc, char **argv) { - int rv, nthreads; + int nthreads; unsigned long i; pthread_t *threads; struct jfsck_result ckres; @@ -109,13 +113,13 @@ int main(int argc, char **argv) return 1; } - rv = jopen(&fs, FILENAME, O_RDWR | O_CREAT | O_TRUNC, 0600, 0); - if (rv < 0) { + fs = jopen(FILENAME, O_RDWR | O_CREAT | O_TRUNC, 0600, 0); + if (fs == NULL) { perror("jopen()"); return 1; } - jtruncate(&fs, towrite * nthreads); + jtruncate(fs, towrite * nthreads); for (i = 0; i < nthreads; i++) { pthread_create(threads + i, NULL, &worker, (void *) i); @@ -125,17 +129,16 @@ int main(int argc, char **argv) pthread_join(*(threads + i), NULL); } - jclose(&fs); - jfsck(FILENAME, NULL, &ckres); + jclose(fs); + jfsck(FILENAME, NULL, &ckres, 0); if (ckres.total != 0) { fprintf(stderr, "There were %d errors during the test\n", ckres.total); - fprintf(stderr, "jfsck() was used to fix them, but that"); + fprintf(stderr, "jfsck() was used to fix them, but that "); fprintf(stderr, "shouldn't happen.\n"); return 1; } - jfsck_cleanup(FILENAME, NULL); return 0; } diff --git a/tests/performance/performance.c b/tests/performance/random.c similarity index 81% copy from tests/performance/performance.c copy to tests/performance/random.c index 849cd60..67c30cb 100644 --- a/tests/performance/performance.c +++ b/tests/performance/random.c @@ -1,6 +1,6 @@ /* - * performance.c - A program to test speed of parallel writes using libjio. + * random.c - A program to test speed of random writes using libjio. * Alberto Bertogli (albertito@blitiri.com.ar) * * It creates a big file, extends it using truncate, and forks N threads which @@ -15,12 +15,13 @@ #include #include #include +#include #include #define FILENAME "test_file" /* These are shared among threads, to make the code simpler */ -static struct jfs fs; +static jfs_t *fs; static unsigned long mb; static ssize_t blocksize, towrite; @@ -39,7 +40,7 @@ static void *worker(void *tno) void *buf; unsigned long tid; ssize_t work_done, rv; - off_t localoffset; + off_t localoffset, offset; long secs, usecs; double seconds, mb_per_sec; struct timeval tv1, tv2; @@ -53,13 +54,17 @@ static void *worker(void *tno) perror("malloc()"); return NULL; } + memset(buf, 5, blocksize); work_done = 0; + srandom(time(NULL)); + gettimeofday(&tv1, NULL); while (work_done < towrite) { - rv = jpwrite(&fs, buf, blocksize, localoffset + work_done); + offset = random() % (towrite - blocksize); + rv = jpwrite(fs, buf, blocksize, localoffset + offset); if (rv != blocksize) { perror("jpwrite()"); break; @@ -83,12 +88,14 @@ static void *worker(void *tno) printf("%lu %zd %zd %f %f\n", tid, mb, blocksize, seconds, mb_per_sec); + free(buf); + return NULL; } int main(int argc, char **argv) { - int rv, nthreads; + int nthreads; unsigned long i; pthread_t *threads; struct jfsck_result ckres; @@ -109,13 +116,13 @@ int main(int argc, char **argv) return 1; } - rv = jopen(&fs, FILENAME, O_RDWR | O_CREAT | O_TRUNC, 0600, 0); - if (rv < 0) { + fs = jopen(FILENAME, O_RDWR | O_CREAT | O_TRUNC, 0600, 0); + if (fs == NULL) { perror("jopen()"); return 1; } - jtruncate(&fs, towrite * nthreads); + jtruncate(fs, towrite * nthreads); for (i = 0; i < nthreads; i++) { pthread_create(threads + i, NULL, &worker, (void *) i); @@ -125,17 +132,16 @@ int main(int argc, char **argv) pthread_join(*(threads + i), NULL); } - jclose(&fs); - jfsck(FILENAME, NULL, &ckres); + jclose(fs); + jfsck(FILENAME, NULL, &ckres, 0); if (ckres.total != 0) { fprintf(stderr, "There were %d errors during the test\n", ckres.total); - fprintf(stderr, "jfsck() was used to fix them, but that"); + fprintf(stderr, "jfsck() was used to fix them, but that "); fprintf(stderr, "shouldn't happen.\n"); return 1; } - jfsck_cleanup(FILENAME, NULL); return 0; } diff --git a/tests/stress/jiostress b/tests/stress/jiostress index ca4a83a..76c3d6b 100755 --- a/tests/stress/jiostress +++ b/tests/stress/jiostress @@ -26,19 +26,11 @@ except ImportError: # Auxiliary stuff # -randsrc = open('/dev/urandom', mode = 'rb') -def randbytes(n): - "Returns n random bytes" - if n < 64: - return randsrc.read(n) - - # to avoid reading too much from urandom (which needlessly stresses - # the kernel), just get a small random stream and repeat it - s = randsrc.read(64) - while len(s) < n: - s += s + s[::-1] - - return s[:n] +gbcount = 0 +def getbytes(n): + global gbcount + gbcount = (gbcount + 1) % 10 + return bytes(str(gbcount) * n, 'ascii') def randfrange(maxend, maxsize): start = random.randint(0, maxend - 1) @@ -54,18 +46,29 @@ class ConsistencyError (Exception): # class Stresser: - def __init__(self, fname, fsize, nops): + def __init__(self, fname, fsize, nops, use_fi, use_as): self.fname = fname self.fsize = fsize self.nops = nops + self.use_fi = use_fi + self.use_as = use_as - self.maxoplen = min(int(self.fsize / 4), - 5 * 1024 * 1024) + self.maxoplen = min(int(self.fsize / 256), + 64 * 1024) - self.jf = libjio.open(fname, - libjio.O_RDWR | libjio.O_CREAT, 0o600) + jflags = 0 + if use_as: + jflags = libjio.J_LINGER + + self.jf = libjio.open(fname, libjio.O_RDWR | libjio.O_CREAT, + 0o600, jflags) self.f = open(fname, mode = 'rb') + self.jf.truncate(fsize) + + if use_as: + self.jf.autosync_start(5, 2 * 1024 * 1024) + # data used for consistency checks self.current_range = (0, 0) self.prev_data = b"" @@ -79,24 +82,27 @@ class Stresser: return r def randwrite(self): + start, end = randfrange(self.fsize, self.maxoplen) + + # read an extended range so we can check we + # only wrote what we were supposed to + estart = max(0, start - 32) + eend = min(self.fsize, end + 32) + self.current_range = (estart, eend) + self.prev_data = self.pread(estart, eend) + + nd = getbytes(end - start) + self.new_data = self.prev_data[:start - estart] \ + + nd + self.prev_data[- (eend - end):] + self.jf.pwrite(nd, start) + return True + + def randwrite_fork(self): pid = os.fork() if pid == 0: # child try: - start, end = randfrange(self.fsize, - self.maxoplen) - - # read an extended range so we can check we - # only wrote what we were supposed to - estart = max(0, start - 32) - eend = min(self.fsize, end + 32) - self.current_range = (estart, eend) - self.prev_data = self.pread(estart, eend) - - nd = randbytes(end - start) - self.new_data = self.prev_data[estart:start] \ - + nd + self.prev_data[end:eend] - self.jf.pwrite(nd, start) + self.randwrite() except IOError: sys.exit(1) except: @@ -124,17 +130,18 @@ class Stresser: self.jf = None r = libjio.jfsck(self.fname) self.verify() - libjio.jfsck_cleanup(self.fname) self.jf = libjio.open(self.fname, libjio.O_RDWR | libjio.O_CREAT, 0o600) return r def fiu_enable(self): - fiu.enable_random('jio/*', probability = 0.02) + if self.use_fi: + fiu.enable_random('jio/*', probability = 0.02) def fiu_disable(self): - fiu.disable('jio/*') + if self.use_fi: + fiu.disable('jio/*') def run(self): self.fiu_enable() @@ -148,7 +155,12 @@ class Stresser: sys.stdout.write(" %d\n" % i) sys.stdout.write(" ") sys.stdout.flush() - if not self.randwrite(): + + if self.use_fi: + r = self.randwrite_fork() + else: + r = self.randwrite() + if not r: nfailures += 1 self.fiu_disable() r = self.reopen() @@ -168,9 +180,17 @@ class Stresser: def usage(): print(""" Use: jiostress [] + [--fi] [--as] If the number of operations is not provided, the default (1000) will be -used.""") +used. + +If the "--fi" option is passed, the test will perform fault injection. This +option conflicts with "--as". + +If the "--as" option is passed, lingering transactions will be used, along +with the automatic syncing thread. This option conflicts with "--fi". +""") def main(): @@ -178,22 +198,34 @@ def main(): fname = sys.argv[1] fsize = int(sys.argv[2]) * 1024 * 1024 nops = 1000 - if len(sys.argv) >= 4: + if len(sys.argv) >= 4 and sys.argv[3].isnumeric(): nops = int(sys.argv[3]) + + use_fi = False + if '--fi' in sys.argv: + use_fi = True + + use_as = False + if '--as' in sys.argv: + use_as = True except: usage() sys.exit(1) - s = Stresser(fname, fsize, nops) + if use_fi and use_as: + print("Error: --fi and --as cannot be used together") + sys.exit(1) + + s = Stresser(fname, fsize, nops, use_fi, use_as) print("Running stress test") nfailures = s.run() + del s print("Stress test completed") print(" %d operations" % nops) print(" %d simulated failures" % nfailures) r = libjio.jfsck(fname) assert r['total'] == 0 - libjio.jfsck_cleanup(fname) print("Final check completed") #os.unlink(fname)