.gitignore | 13 +- INSTALL | 3 +- LICENSE | 5 +- Makefile | 32 ++- README | 16 +- UPGRADING | 7 + bindings/bigloo/nmdb.scm | 9 +- bindings/d/nmdb.d | 22 +- bindings/d/nmdb_ll.d | 4 +- bindings/d/testt.d | 8 +- bindings/haskell/Nmdb.hs | 19 +- bindings/haskell/nmdb.cabal | 2 +- bindings/newlisp/nmdb.lsp | 9 +- bindings/python/nmdb.py | 28 +- bindings/python/nmdb_ll.c | 16 +- bindings/python/setup.py | 4 +- bindings/python3/LICENSE | 34 ++ bindings/{python => python3}/nmdb.py | 47 ++-- bindings/{python => python3}/nmdb_ll.c | 80 +++--- bindings/{python => python3}/setup.py | 4 +- bindings/ruby/nmdb.rb | 4 +- bindings/ruby/nmdb_ll.c | 11 +- doc/design.rst | 60 ++--- doc/guide.rst | 62 ++-- doc/network.rst | 83 +++-- libnmdb/Makefile | 27 ++- libnmdb/internal.h | 2 +- libnmdb/libnmdb.3 | 18 +- libnmdb/libnmdb.c | 457 +++++++++++++++------------- libnmdb/libnmdb.skel.pc | 12 + libnmdb/net-const.h | 62 +---- libnmdb/netutils.c | 1 + libnmdb/netutils.h | 1 + libnmdb/nmdb.skel.h | 15 +- libnmdb/sctp.c | 8 +- libnmdb/sctp.h | 3 +- libnmdb/tcp.c | 8 +- libnmdb/tcp.h | 3 +- libnmdb/tipc.c | 2 + libnmdb/udp.c | 8 +- libnmdb/udp.h | 3 +- nmdb/Makefile | 43 ++- nmdb/be-tc.c | 52 ++++ nmdb/be.h | 7 + nmdb/cache.c | 69 ++--- nmdb/cache.h | 2 +- nmdb/common.h | 7 +- nmdb/dbloop.c | 25 +- nmdb/log.c | 19 +- nmdb/log.h | 1 + nmdb/main.c | 23 +- nmdb/net-const.h | 22 +- nmdb/net.c | 12 +- nmdb/netutils.c | 53 ++++ nmdb/netutils.h | 11 + nmdb/nmdb.1 | 15 +- nmdb/parse.c | 340 +++++++++++---------- nmdb/queue.c | 7 + nmdb/queue.h | 21 +- nmdb/req.h | 9 +- nmdb/sctp.c | 16 +- nmdb/sparse.h | 21 ++ nmdb/stats.c | 31 ++ nmdb/stats.h | 41 +++ nmdb/tcp.c | 15 +- nmdb/tipc.c | 16 +- nmdb/udp.c | 16 +- tests/c/incr.c | 8 +- tests/c/make.sh | 5 +- tests/c/timer.h | 4 +- tests/perf/ag.sh | 126 ++++++++ tests/perf/perf.sh | 211 +++++++++++++ tests/python/random1-cache.py | 15 +- tests/python/random1.py | 15 +- tests/python3/README | 4 + tests/{python => python3}/random1-cache.py | 37 ++- tests/{python => python3}/random1.py | 51 ++-- utils/Makefile | 52 ++++ utils/nmdb-stats.1 | 33 ++ utils/nmdb-stats.c | 150 +++++++++ 80 files changed, 1917 insertions(+), 900 deletions(-) diff --git a/.gitignore b/.gitignore index dc7cf5e..7cad653 100644 --- a/.gitignore +++ b/.gitignore @@ -9,8 +9,15 @@ *.gcda # *~ /nmdb/nmdb -tests/c/*-*-cache -tests/c/*-*-normal -tests/c/*-*-sync +/utils/nmdb-stats +/tests/c/*-*-cache +/tests/c/*-*-normal +/tests/c/*-*-sync /libnmdb/nmdb.h +/libnmdb/libnmdb.pc /tags +/bindings/python/build +/bindings/python3/build +/tests/perf/out +/tests/perf/ag-data +/tests/perf/graph diff --git a/INSTALL b/INSTALL index 687baa8..64f1615 100644 --- a/INSTALL +++ b/INSTALL @@ -60,7 +60,8 @@ Bindings To compile the Python bindings, you need to have the library already installed. Use "make python_install" at the top level directory to build and -install the modules. The module will be named "nmdb". +install the modules. The module will be named "nmdb". The same goes for Python +3, use "make python3_install". The other bindings do not have a properly defined install procedure, and you'll need knowledge of the language to install them. diff --git a/LICENSE b/LICENSE index bde222e..e1e5faa 100644 --- a/LICENSE +++ b/LICENSE @@ -9,10 +9,11 @@ includes subdirectories). As a brief resume, here's how each sub-project is licensed: * nmdb: OSL 3.0 * libnmdb: BOLA (Public domain) - * bindings/python: BOLA (Public domain) + * bindings/bigloo: BOLA (Public domain) * bindings/d: BOLA (Public domain) + * bindings/haskell: BOLA (Public domain) * bindings/newlisp: BOLA (Public domain) + * bindings/python: BOLA (Public domain) * bindings/ruby: BOLA (Public domain) - * bindings/bigloo: BOLA (Public domain) diff --git a/Makefile b/Makefile index 3fcab55..5ef8582 100644 --- a/Makefile +++ b/Makefile @@ -1,21 +1,26 @@ all: default -default: nmdb libnmdb +default: nmdb libnmdb utils nmdb: - make -C nmdb + $(MAKE) -C nmdb libnmdb: - make -C libnmdb + $(MAKE) -C libnmdb + +utils: + $(MAKE) -C utils install: - make -C nmdb install - make -C libnmdb install + $(MAKE) -C nmdb install + $(MAKE) -C libnmdb install + $(MAKE) -C utils install clean: - make -C nmdb clean - make -C libnmdb clean + $(MAKE) -C nmdb clean + $(MAKE) -C libnmdb clean + $(MAKE) -C utils clean python: @@ -27,7 +32,18 @@ python_install: python_clean: cd bindings/python && rm -rf build/ +python3: + cd bindings/python3 && python3 setup.py build + +python3_install: + cd bindings/python3 && python3 setup.py install + +python3_clean: + cd bindings/python3 && rm -rf build/ + -.PHONY: default all clean nmdb libnmdb python python_install python_clean +.PHONY: default all clean nmdb libnmdb utils \ + python python_install python_clean \ + python3 python3_install python3_clean diff --git a/README b/README index 9702b34..36e3411 100644 --- a/README +++ b/README @@ -1,6 +1,6 @@ nmdb - A multiprotocol network database manager -Alberto Bertogli (albertito@gmail.com) +Alberto Bertogli (albertito@blitiri.com.ar) --------------------------------------------------- nmdb is a network database that can use different protocols to communicate @@ -13,9 +13,9 @@ Both work combined, but the use of the persistent backend is optional, so you can use the server only for cache queries, pretty much like memcached. This source distribution is composed of several parts: the server called -"nmdb", the library and bindings for Python, D, NewLISP, Ruby, Bigloo Scheme -and Haskell. Each one has a separate directory, and is licensed individually. -See the LICENSE file for more information. +"nmdb", the library and bindings for Python (2 and 3), D, NewLISP, Ruby, +Bigloo Scheme and Haskell. Each one has a separate directory, and is licensed +individually. See the LICENSE file for more information. Documentation @@ -26,15 +26,13 @@ directory. It is recommended that you read the User Guide (doc/guide.rst, or the online version) to learn how to setup and use nmdb. For additional documentation and resources, go to the project's website at -http://auriga.wearlab.de/~alb/nmdb. +http://blitiri.com.ar/p/nmdb. Where to report bugs -------------------- -If you want to report bugs, suggestions, issues, comments, or just get in -touch with nmdb developers and users, join the mailing list at -http://lists.auriga.wearlab.de/cgi-bin/mailman/listinfo/nmdb-devel or just -send an email to nmdb-devel@lists.auriga.wearlab.de. +If you want to report bugs, or have any questions or comments, just let me +know at albertito@blitiri.com.ar. diff --git a/UPGRADING b/UPGRADING index bd4c765..bd906cd 100644 --- a/UPGRADING +++ b/UPGRADING @@ -11,3 +11,10 @@ upgrading. bindings have been upgraded, so you only need to worry if you used the C library. +0.21 -> 0.22 + * The protocol has been changed in an incompatible way. Do not use the + 0.21 library with the new server. + * nmdb_incr() and nmdb_cache_incr() now have a new "newval" parameter. If the + increment was successful, newval will be set to the resulted value. + All bindings have been upgraded to reflect this change. + diff --git a/bindings/bigloo/nmdb.scm b/bindings/bigloo/nmdb.scm index 014dbdb..3b086bb 100644 --- a/bindings/bigloo/nmdb.scm +++ b/bindings/bigloo/nmdb.scm @@ -6,6 +6,7 @@ ;; C functions (extern (type _nmdb_t (pointer void) "void *") + (type long* (pointer long) "int64_t *") (macro _nmdb_init::_nmdb_t () "nmdb_init") (macro _nmdb_free::int (::_nmdb_t) "nmdb_free") @@ -54,10 +55,10 @@ "nmdb_cache_cas") (macro _nmdb_incr::int - (::_nmdb_t ::string ::uint ::long) + (::_nmdb_t ::string ::uint ::long ::long*) "nmdb_incr") (macro _nmdb_cache_incr::int - (::_nmdb_t ::string ::uint ::long) + (::_nmdb_t ::string ::uint ::long ::long*) "nmdb_cache_incr") ) @@ -147,7 +148,9 @@ ;; incr functions (define (nmdb-generic-incr func db key increment) - (func db key (string-length key) increment ) ) + (define newval (make-long* 1)) + (define res (func db key (string-length key) increment newval) ) + (list res (long*-ref newval 0) ) ) (define (nmdb-incr db key increment) (nmdb-generic-incr _nmdb_incr db key increment)) (define (nmdb-cache-incr db key increment) diff --git a/bindings/d/nmdb.d b/bindings/d/nmdb.d index 9c96062..2eb9d58 100644 --- a/bindings/d/nmdb.d +++ b/bindings/d/nmdb.d @@ -1,7 +1,7 @@ /* * Digital Mars D programming language bindings for the nmdb C library. - * Alberto Bertogli (albertito@gmail.com) + * Alberto Bertogli (albertito@blitiri.com.ar) */ module nmdb; @@ -165,15 +165,17 @@ class DB return res; } - private int do_incr(char[] key, long increment, int mode) + private int do_incr(char[] key, long increment, long *newval, + int mode) { ubyte* k = cast(ubyte *) key.ptr; int res = 0; if (mode == MODE_NORMAL || mode == MODE_SYNC) { - res = nmdb_incr(db, k, key.length, increment); + res = nmdb_incr(db, k, key.length, increment, newval); } else if (mode == MODE_CACHE) { - res = nmdb_cache_incr(db, k, key.length, increment); + res = nmdb_cache_incr(db, k, key.length, increment, + newval); } else { throw new Exception("Invalid mode"); } @@ -255,19 +257,19 @@ class DB } - int incr(char[] key, long increment) + int incr(char[] key, long increment, long *newval) { - return do_incr(key, increment, mode); + return do_incr(key, increment, newval, mode); } - int incr_normal(char[] key, long increment) + int incr_normal(char[] key, long increment, long *newval) { - return do_incr(key, increment, MODE_NORMAL); + return do_incr(key, increment, newval, MODE_NORMAL); } - int cache_incr(char[] key, long increment) + int cache_incr(char[] key, long increment, long *newval) { - return do_incr(key, increment, MODE_CACHE); + return do_incr(key, increment, newval, MODE_CACHE); } diff --git a/bindings/d/nmdb_ll.d b/bindings/d/nmdb_ll.d index beb28d3..03c2426 100644 --- a/bindings/d/nmdb_ll.d +++ b/bindings/d/nmdb_ll.d @@ -65,7 +65,7 @@ extern (C) int nmdb_cache_cas(nmdb_t *db, ubyte *key, size_t ksize, ubyte *oldval, size_t ovsize, ubyte *newval, size_t nvsize); extern (C) int nmdb_incr(nmdb_t *db, ubyte *key, size_t ksize, - long increment); + long increment, long *newval); extern (C) int nmdb_cache_incr(nmdb_t *db, ubyte *key, size_t ksize, - long increment); + long increment, long *newval); diff --git a/bindings/d/testt.d b/bindings/d/testt.d index e2b820c..a70441d 100644 --- a/bindings/d/testt.d +++ b/bindings/d/testt.d @@ -7,16 +7,20 @@ import std.stream; int main() { char[] val1; + long newval, ret; nmdb.DB db = new nmdb.DB(); db.add_tipc_server(); db.mode = MODE_CACHE; - db["1"] = "D"; - val1 = db["1"]; + db["D"] = "1\0"; + val1 = db["D"]; writefln(val1); + ret = db.incr("D", 4, &newval); + writefln(newval, " ", db["D"]); + return 0; } diff --git a/bindings/haskell/Nmdb.hs b/bindings/haskell/Nmdb.hs index 735385a..06fee1f 100644 --- a/bindings/haskell/Nmdb.hs +++ b/bindings/haskell/Nmdb.hs @@ -1,6 +1,6 @@ -- Haskell bindings for the nmdb C library --- Alberto Bertogli (albertito@gmail.com) +-- Alberto Bertogli (albertito@blitiri.com.ar) module Nmdb ( NmdbStruct, @@ -14,7 +14,9 @@ module Nmdb ( nmdbIncr, nmdbCacheIncr, ) where +import Foreign import Foreign.Ptr +import Foreign.Storable import Foreign.C.Types import Foreign.C.String import Foreign.Marshal.Alloc @@ -158,15 +160,22 @@ nmdbCacheCAS = nmdbGenericCAS llNmdbCacheCAS -- Incr functions foreign import ccall "nmdb.h nmdb_incr" llNmdbIncr :: - NmdbPtr -> CString -> Int -> Int -> IO Int + NmdbPtr -> CString -> Int -> Int64 -> Ptr Int64 -> IO Int foreign import ccall "nmdb.h nmdb_cache_incr" llNmdbCacheIncr :: - NmdbPtr -> CString -> Int -> Int -> IO Int + NmdbPtr -> CString -> Int -> Int64 -> Ptr Int64 -> IO Int nmdbGenericIncr llfunc db key increment = do kl <- newCStringLen key - r <- llfunc db (fst kl) (snd kl) increment + newval <- malloc + r <- llfunc db (fst kl) (snd kl) increment newval + v <- (peek newval) free (fst kl) - return r + free newval + if r == 2 + then do + return $ Just v + else + return Nothing nmdbIncr = nmdbGenericIncr llNmdbIncr nmdbCacheIncr = nmdbGenericIncr llNmdbCacheIncr diff --git a/bindings/haskell/nmdb.cabal b/bindings/haskell/nmdb.cabal index 5f1371d..cb7da4f 100644 --- a/bindings/haskell/nmdb.cabal +++ b/bindings/haskell/nmdb.cabal @@ -3,7 +3,7 @@ Version: 0.1 License: PublicDomain License-file: LICENSE Author: Alberto Bertogli -Homepage: http://auriga.wearlab.de/~alb/nmdb/ +Homepage: http://blitiri.com.ar/p/nmdb/ Category: Database Build-Depends: base Exposed-modules: Nmdb diff --git a/bindings/newlisp/nmdb.lsp b/bindings/newlisp/nmdb.lsp index 640bfe7..4f4641b 100644 --- a/bindings/newlisp/nmdb.lsp +++ b/bindings/newlisp/nmdb.lsp @@ -1,7 +1,7 @@ ; ; newlisp (http://www.newlisp.org/) bindings for nmdb -; Alberto Bertogli (albertito@gmail.com) +; Alberto Bertogli (albertito@blitiri.com.ar) ; ; Functions: ; @@ -142,8 +142,11 @@ ; *-incr functions (define (priv-incr func key increment) - (letn ( (keylen (length key)) ) - (func NMDB key keylen increment) ) ) + (letn ( (keylen (length key)) + (newval 0) + ) + (set 'ret (func NMDB key keylen increment (address newval)) ) + (list ret newval) ) ) (define (db-incr key increment) (priv-incr nmdb_incr key increment)) (define (cache-incr key increment) (priv-incr nmdb_cache_incr key increment)) diff --git a/bindings/python/nmdb.py b/bindings/python/nmdb.py index 38f7366..e185d55 100644 --- a/bindings/python/nmdb.py +++ b/bindings/python/nmdb.py @@ -40,7 +40,11 @@ Traceback (most recent call last): KeyError """ -import cPickle +try: + import cPickle as pickle +except ImportError: + import pickle + import nmdb_ll @@ -78,7 +82,7 @@ class GenericDB (object): def generic_get(self, getf, key): "d[k] Returns the value associated with the key k." if self.autopickle: - key = str(hash(key)) + key = pickle.dumps(key, protocol = -1) try: r = getf(key) except: @@ -88,7 +92,7 @@ class GenericDB (object): # so we know it's a miss. raise KeyError if self.autopickle: - r = cPickle.loads(r) + r = pickle.loads(r) return r def cache_get(self, key): @@ -101,8 +105,8 @@ class GenericDB (object): def generic_set(self, setf, key, val): "d[k] = v Associates the value v to the key k." if self.autopickle: - key = str(hash(key)) - val = cPickle.dumps(val, protocol = -1) + key = pickle.dumps(key, protocol = -1) + val = pickle.dumps(val, protocol = -1) r = setf(key, val) if r <= 0: raise NetworkError @@ -121,7 +125,7 @@ class GenericDB (object): def generic_delete(self, delf, key): "del d[k] Deletes the key k." if self.autopickle: - key = str(hash(key)) + key = pickle.dumps(key, protocol = -1) r = delf(key) if r < 0: raise NetworkError @@ -142,9 +146,9 @@ class GenericDB (object): def generic_cas(self, casf, key, oldval, newval): "Perform a compare-and-swap." if self.autopickle: - key = str(hash(key)) - oldval = cPickle.dumps(oldval, protocol = -1) - newval = cPickle.dumps(newval, protocol = -1) + key = pickle.dumps(key, protocol = -1) + oldval = pickle.dumps(oldval, protocol = -1) + newval = pickle.dumps(newval, protocol = -1) r = casf(key, oldval, newval) if r == 2: # success @@ -171,11 +175,11 @@ class GenericDB (object): """Atomically increment the value associated with the given key by the given increment.""" if self.autopickle: - key = str(hash(key)) - r = incrf(key, increment) + key = pickle.dumps(key, protocol = -1) + r, v = incrf(key, increment) if r == 2: # success - return 2 + return v elif r == 1: # no match, because the value didn't have the right # format diff --git a/bindings/python/nmdb_ll.c b/bindings/python/nmdb_ll.c index 2374118..c4bdd91 100644 --- a/bindings/python/nmdb_ll.c +++ b/bindings/python/nmdb_ll.c @@ -1,7 +1,7 @@ /* * Python bindings for libnmdb - * Alberto Bertogli (albertito@gmail.com) + * Alberto Bertogli (albertito@blitiri.com.ar) * * This is the low-level module, used by the python one to construct * friendlier objects. @@ -136,7 +136,7 @@ static PyObject *db_cache_get(nmdbobject *db, PyObject *args) /* Miss, handled in the high-level module. */ r = PyLong_FromLong(-1); } else { - r = PyString_FromStringAndSize(val, rv); + r = PyString_FromStringAndSize((char *) val, rv); } free(val); @@ -189,6 +189,7 @@ static PyObject *db_cache_incr(nmdbobject *db, PyObject *args) int ksize; int rv; long long int increment; + int64_t newval; if (!PyArg_ParseTuple(args, "s#L:cache_incr", &key, &ksize, &increment)) { @@ -196,10 +197,10 @@ static PyObject *db_cache_incr(nmdbobject *db, PyObject *args) } Py_BEGIN_ALLOW_THREADS - rv = nmdb_cache_incr(db->db, key, ksize, increment); + rv = nmdb_cache_incr(db->db, key, ksize, increment, &newval); Py_END_ALLOW_THREADS - return PyLong_FromLong(rv); + return Py_BuildValue("LL", rv, newval); } @@ -251,7 +252,7 @@ static PyObject *db_get(nmdbobject *db, PyObject *args) /* Miss, handled in the high-level module. */ r = PyLong_FromLong(-1); } else { - r = PyString_FromStringAndSize(val, rv); + r = PyString_FromStringAndSize((char *) val, rv); } free(val); @@ -303,16 +304,17 @@ static PyObject *db_incr(nmdbobject *db, PyObject *args) int ksize; int rv; long long int increment; + int64_t newval; if (!PyArg_ParseTuple(args, "s#L:incr", &key, &ksize, &increment)) { return NULL; } Py_BEGIN_ALLOW_THREADS - rv = nmdb_incr(db->db, key, ksize, increment); + rv = nmdb_incr(db->db, key, ksize, increment, &newval); Py_END_ALLOW_THREADS - return PyLong_FromLong(rv); + return Py_BuildValue("LL", rv, newval); } diff --git a/bindings/python/setup.py b/bindings/python/setup.py index 0ba115e..54c1bcd 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -9,8 +9,8 @@ setup( name = 'nmdb', description = "libnmdb bindings", author = "Alberto Bertogli", - author_email = "albertito@gmail.com", - url = "http://auriga.wearlab.de/~alb/nmdb", + author_email = "albertito@blitiri.com.ar", + url = "http://blitiri.com.ar/p/nmdb", py_modules = ['nmdb'], ext_modules = [nmdb_ll] ) diff --git a/bindings/python3/LICENSE b/bindings/python3/LICENSE new file mode 100644 index 0000000..f3a9498 --- /dev/null +++ b/bindings/python3/LICENSE @@ -0,0 +1,34 @@ + +I don't like licenses, because I don't like having to worry about all this +legal stuff just for a simple piece of software I don't really mind anyone +using. But I also believe that it's important that people share and give back; +so I'm placing this library under the following license, so you feel guilty if +you don't ;) + + +BOLA - Buena Onda License Agreement +----------------------------------- + +This work is provided 'as-is', without any express or implied warranty. In no +event will the authors be held liable for any damages arising from the use of +this work. + +To all effects and purposes, this work is to be considered Public Domain. + + +However, if you want to be "Buena onda", you should: + +1. Not take credit for it, and give proper recognition to the authors. +2. Share your modifications, so everybody benefits from them. +4. Do something nice for the authors. +5. Help someone who needs it: sign up for some volunteer work or help your + neighbour paint the house. +6. Don't waste. Anything, but specially energy that comes from natural + non-renewable resources. Extra points if you discover or invent something + to replace them. +7. Be tolerant. Everything that's good in nature comes from cooperation. + +The order is important, and the further you go the more "Buena onda" you are. +Make the world a better place: be "Buena onda". + + diff --git a/bindings/python/nmdb.py b/bindings/python3/nmdb.py similarity index 85% copy from bindings/python/nmdb.py copy to bindings/python3/nmdb.py index 38f7366..2c0b88c 100644 --- a/bindings/python/nmdb.py +++ b/bindings/python3/nmdb.py @@ -1,6 +1,6 @@ """ -libnmdb python wrapper +libnmdb python 3 wrapper This module is a wrapper for the libnmdb, the C library used to implement clients to the nmdb server. @@ -26,21 +26,26 @@ Here is an example using the DB class: >>> db = nmdb.DB() >>> db.add_tipc_server() >>> db[1] = { 'english': 'one', 'castellano': 'uno', 'quechua': 'huk' } ->>> print db[1] +>>> print(db[1]) {'english': 'one', 'castellano': 'uno', 'quechua': 'huk'} >>> db[(1, 2)] = (True, False) ->>> print db[(1, 2)] +>>> print(db[(1, 2)]) (True, False) >>> del db[(1, 2)] ->>> print db[(1, 2)] +>>> print(db[(1, 2)]) Traceback (most recent call last): File "", line 1, in - File "/usr/lib64/python2.5/site-packages/nmdb.py", line 74, in __getitem__ + File "/usr/local/lib/python3.0/dist-packages/nmdb.py", line 206, in __getitem__ + return self.get(key) + File "/usr/local/lib/python3.0/dist-packages/nmdb.py", line 102, in normal_get + return self.generic_get(self._db.get, key) + File "/usr/local/lib/python3.0/dist-packages/nmdb.py", line 93, in generic_get raise KeyError KeyError +>>> """ -import cPickle +import pickle import nmdb_ll @@ -48,9 +53,9 @@ class NetworkError (Exception): pass -class GenericDB (object): +class GenericDB: def __init__(self): - self._db = nmdb_ll.new() + self._db = nmdb_ll.nmdb() self.autopickle = True def add_tipc_server(self, port = -1): @@ -78,7 +83,7 @@ class GenericDB (object): def generic_get(self, getf, key): "d[k] Returns the value associated with the key k." if self.autopickle: - key = str(hash(key)) + key = pickle.dumps(key, protocol = -1) try: r = getf(key) except: @@ -88,7 +93,7 @@ class GenericDB (object): # so we know it's a miss. raise KeyError if self.autopickle: - r = cPickle.loads(r) + r = pickle.loads(r) return r def cache_get(self, key): @@ -101,8 +106,8 @@ class GenericDB (object): def generic_set(self, setf, key, val): "d[k] = v Associates the value v to the key k." if self.autopickle: - key = str(hash(key)) - val = cPickle.dumps(val, protocol = -1) + key = pickle.dumps(key, protocol = -1) + val = pickle.dumps(val, protocol = -1) r = setf(key, val) if r <= 0: raise NetworkError @@ -121,7 +126,7 @@ class GenericDB (object): def generic_delete(self, delf, key): "del d[k] Deletes the key k." if self.autopickle: - key = str(hash(key)) + key = pickle.dumps(key, protocol = -1) r = delf(key) if r < 0: raise NetworkError @@ -142,9 +147,9 @@ class GenericDB (object): def generic_cas(self, casf, key, oldval, newval): "Perform a compare-and-swap." if self.autopickle: - key = str(hash(key)) - oldval = cPickle.dumps(oldval, protocol = -1) - newval = cPickle.dumps(newval, protocol = -1) + key = pickle.dumps(key, protocol = -1) + oldval = pickle.dumps(oldval, protocol = -1) + newval = pickle.dumps(newval, protocol = -1) r = casf(key, oldval, newval) if r == 2: # success @@ -171,16 +176,16 @@ class GenericDB (object): """Atomically increment the value associated with the given key by the given increment.""" if self.autopickle: - key = str(hash(key)) - r = incrf(key, increment) + key = pickle.dumps(key, protocol = -1) + r, v = incrf(key, increment) if r == 2: # success - return 2 + return v elif r == 1: # no match, because the value didn't have the right # format - raise TypeError, \ - "The value must be a NULL-terminated string" + raise TypeError("The value must be a " + \ + "NULL-terminated string") elif r == 0: # not in raise KeyError diff --git a/bindings/python/nmdb_ll.c b/bindings/python3/nmdb_ll.c similarity index 88% copy from bindings/python/nmdb_ll.c copy to bindings/python3/nmdb_ll.c index 2374118..a6a4ab8 100644 --- a/bindings/python/nmdb_ll.c +++ b/bindings/python3/nmdb_ll.c @@ -1,7 +1,7 @@ /* - * Python bindings for libnmdb - * Alberto Bertogli (albertito@gmail.com) + * Python 3 bindings for libnmdb + * Alberto Bertogli (albertito@blitiri.com.ar) * * This is the low-level module, used by the python one to construct * friendlier objects. @@ -136,7 +136,7 @@ static PyObject *db_cache_get(nmdbobject *db, PyObject *args) /* Miss, handled in the high-level module. */ r = PyLong_FromLong(-1); } else { - r = PyString_FromStringAndSize(val, rv); + r = PyBytes_FromStringAndSize((char *) val, rv); } free(val); @@ -189,6 +189,7 @@ static PyObject *db_cache_incr(nmdbobject *db, PyObject *args) int ksize; int rv; long long int increment; + int64_t newval; if (!PyArg_ParseTuple(args, "s#L:cache_incr", &key, &ksize, &increment)) { @@ -196,10 +197,10 @@ static PyObject *db_cache_incr(nmdbobject *db, PyObject *args) } Py_BEGIN_ALLOW_THREADS - rv = nmdb_cache_incr(db->db, key, ksize, increment); + rv = nmdb_cache_incr(db->db, key, ksize, increment, &newval); Py_END_ALLOW_THREADS - return PyLong_FromLong(rv); + return Py_BuildValue("LL", rv, newval); } @@ -251,7 +252,7 @@ static PyObject *db_get(nmdbobject *db, PyObject *args) /* Miss, handled in the high-level module. */ r = PyLong_FromLong(-1); } else { - r = PyString_FromStringAndSize(val, rv); + r = PyBytes_FromStringAndSize((char *) val, rv); } free(val); @@ -303,16 +304,17 @@ static PyObject *db_incr(nmdbobject *db, PyObject *args) int ksize; int rv; long long int increment; + int64_t newval; if (!PyArg_ParseTuple(args, "s#L:incr", &key, &ksize, &increment)) { return NULL; } Py_BEGIN_ALLOW_THREADS - rv = nmdb_incr(db->db, key, ksize, increment); + rv = nmdb_incr(db->db, key, ksize, increment, &newval); Py_END_ALLOW_THREADS - return PyLong_FromLong(rv); + return Py_BuildValue("LL", rv, newval); } @@ -380,33 +382,12 @@ static PyMethodDef nmdb_methods[] = { { NULL } }; -static PyObject *db_getattr(nmdbobject *db, char *name) -{ - return Py_FindMethod(nmdb_methods, (PyObject *)db, name); -} - -static PyTypeObject nmdbType = { - PyObject_HEAD_INIT(NULL) - 0, - "nmdb_ll.nmdb", - sizeof(nmdbobject), - 0, - (destructor) db_dealloc, - 0, - (getattrfunc) db_getattr, -}; - - -/* - * The module - */ - /* new, returns an nmdb object */ -static PyObject *db_new(PyObject *self, PyObject *args) +static PyObject *db_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { nmdbobject *db; - db = PyObject_New(nmdbobject, &nmdbType); + db = (nmdbobject *) type->tp_alloc(type, 0); if (db == NULL) return NULL; @@ -428,18 +409,43 @@ static PyObject *db_new(PyObject *self, PyObject *args) return (PyObject *) db; } -static PyMethodDef nmdb_functions[] = { - { "new", (PyCFunction) db_new, METH_VARARGS, NULL }, - { NULL } + +static PyTypeObject nmdbType = { + PyObject_HEAD_INIT(NULL) + .tp_name = "nmdb_ll.nmdb", + .tp_itemsize = sizeof(nmdbobject), + .tp_dealloc = (destructor) db_dealloc, + .tp_methods = nmdb_methods, + .tp_new = db_new, }; -PyMODINIT_FUNC initnmdb_ll(void) + + +/* + * The module + */ + +static PyModuleDef nmdb_module = { + PyModuleDef_HEAD_INIT, + .m_name = "nmdb_ll", + .m_doc = NULL, + .m_size = -1, +}; + + +PyMODINIT_FUNC PyInit_nmdb_ll(void) { PyObject *m; - nmdbType.ob_type = &PyType_Type; + if (PyType_Ready(&nmdbType) < 0) + return NULL; + + m = PyModule_Create(&nmdb_module); + + Py_INCREF(&nmdbType); + PyModule_AddObject(m, "nmdb", (PyObject *) &nmdbType); - m = Py_InitModule("nmdb_ll", nmdb_functions); + return m; } diff --git a/bindings/python/setup.py b/bindings/python3/setup.py similarity index 76% copy from bindings/python/setup.py copy to bindings/python3/setup.py index 0ba115e..54c1bcd 100644 --- a/bindings/python/setup.py +++ b/bindings/python3/setup.py @@ -9,8 +9,8 @@ setup( name = 'nmdb', description = "libnmdb bindings", author = "Alberto Bertogli", - author_email = "albertito@gmail.com", - url = "http://auriga.wearlab.de/~alb/nmdb", + author_email = "albertito@blitiri.com.ar", + url = "http://blitiri.com.ar/p/nmdb", py_modules = ['nmdb'], ext_modules = [nmdb_ll] ) diff --git a/bindings/ruby/nmdb.rb b/bindings/ruby/nmdb.rb index 09672af..4d38d82 100644 --- a/bindings/ruby/nmdb.rb +++ b/bindings/ruby/nmdb.rb @@ -142,14 +142,14 @@ class GenericDB if @automarshal then key = Marshal.dump(key) end - r = gfunc.call(key, increment) + r, v = gfunc.call(key, increment) if r == 0 then # key not in the database return nil elsif r == 1 or r < 0 then raise NetworkException else - return r + return v end end diff --git a/bindings/ruby/nmdb_ll.c b/bindings/ruby/nmdb_ll.c index 1ef13cc..a07e274 100644 --- a/bindings/ruby/nmdb_ll.c +++ b/bindings/ruby/nmdb_ll.c @@ -202,23 +202,24 @@ static VALUE m_cache_cas(VALUE self, VALUE key, VALUE oldval, VALUE newval) { } -/* Del functions */ +/* Increment functions */ typedef int (*incrf_t) (nmdb_t *db, const unsigned char *k, size_t ks, - int64_t increment); + int64_t increment, int64_t *newval); VALUE generic_incr(VALUE self, VALUE key, VALUE increment, incrf_t incr_func) { ssize_t rv; unsigned char *k; size_t ksize; - int64_t cincr; + int64_t cincr, newval; nmdb_t *db; Data_Get_Struct(self, nmdb_t, db); k = rb_str2cstr(key, &ksize); cincr = rb_num2ll(increment); - rv = incr_func(db, k, ksize, cincr); - return INT2NUM(rv); + rv = incr_func(db, k, ksize, cincr, &newval); + + return rb_ary_new3(2, INT2NUM(rv), rb_ll2inum(newval)); } VALUE m_incr(VALUE self, VALUE key, VALUE increment) { diff --git a/doc/design.rst b/doc/design.rst index ddb60b9..4470fec 100644 --- a/doc/design.rst +++ b/doc/design.rst @@ -2,7 +2,7 @@ ================================================= nmdb - A multiprotocol network database manager ================================================= -:Author: Alberto Bertogli (albertito@gmail.com) +:Author: Alberto Bertogli (albertito@blitiri.com.ar) Introduction @@ -41,6 +41,12 @@ get *key* Retrieves the value for the given key. If the key is in the cache, it returns immediately. If not, it performs a query in the database. +set *key* *value* + Stores the *(key, value)* pair in the database. + +del *key* + Removes the key and its associated value from the database. + cas *key* *oldvalue* *newvalue* Do a compare-and-swap, using *oldvalue* to compare with the value stored in the database, and replacing it with *newvalue* if they match. @@ -48,44 +54,21 @@ cas *key* *oldvalue* *newvalue* incr *key* *increment* Increments the value associated to the given key by the given increment. -set_async *key* *value* - Stores the *(key, value)* pair in the database. It does the set in the cache, - queues the operation for the database, and returns. - -del_async *key* - Removes the key and it's associated value from the database. It does the del - in the cache, queues the operation for the database, and returns. - -set_sync *key* *value* - Like *set*, but return only after the database has completed the operation. - -del_sync *key* - Like *del*, but return only after the database has completed the operation. - -cache_get *key* - Like *get*, but only affects the cache and not the database. If the key is - not in the cache, returns a special value indicating "miss". - -cache_set *key* *value* - Like *set*, but only affects the cache and not the database. - -cache_del *key* - Like *del*, but only affects the cache and not the database. - -cache_cas *key* *oldvalue* *newvalue* - Like *cas*, but only affects the cache and not the database. - -cache_incr *key* *increment* - Like *incr*, but only affects the cache and not the database. +Request can have flags that affect their behaviour. The *cache-only* flag +makes the operation affect only the cache but not the database; and the +*sync* flag makes the server wait until the request has been performed to +reply, instead of replying as soon as the request is queued for processing. +Not all the flags make sense for all the operations, consult the library +documentation for details. As you can see, it's possible to operate exclusively with the cache, ignoring the database completely. This is very similar to what memcached_ does. Note that the downside is that it's possible to mess with the cache, and leave it -out of sync with the database. You can only do this if you mix *cache_set* -with *set* or *set_sync*, which is hard to miss, so it's unlikely you will do -this. +out of sync with the database. You can only do this if you mix a *cache-only +set* with a normal *set*, which is hard to miss, so it's unlikely you will do +this by mistake. -The server assumes you have a brain, and you will not make a mess out of it. +The server assumes you have a brain, and that you will use it. Request processing @@ -140,9 +123,8 @@ operation. A specific solution could have been used, and the database backend code is isolated enough to allow this to happen in the future if necessity arises. -QDBM_ was chosen for the backend, because its speed, although most DBMs would -have been equally useful regarding features, because the server is not at all -demanding. +Several backends are supported (at the moment QDBM_, BDB_, tokyocabinet_ and a +null backend); the selection is done at build time. The processing is performed by taking requests from the aforementioned queue, and acting upon the database accordingly, which involves calling the backend's @@ -216,9 +198,11 @@ Nonetheless, it's advisable to use a large cache size, specially if the usage pattern involves handling lots of different keys. -.. _nmdb: http://auriga.wearlab.de/~alb/nmdb/ +.. _nmdb: http://blitiri.com.ar/p/nmdb/ .. _libevent: http://www.monkey.org/~provos/libevent/ .. _TIPC: http://tipc.sf.net .. _memcached: http://www.danga.com/memcached/ .. _QDBM: http://qdbm.sf.net +.. _BDB: http://www.oracle.com/technology/products/berkeley-db/db/ +.. _tokyocabinet: http://tokyocabinet.sf.net/index.html diff --git a/doc/guide.rst b/doc/guide.rst index ed8fe17..3d94cfd 100644 --- a/doc/guide.rst +++ b/doc/guide.rst @@ -2,7 +2,7 @@ ================ nmdb User Guide ================ -:Author: Alberto Bertogli (albertito@gmail.com) +:Author: Alberto Bertogli (albertito@blitiri.com.ar) Introduction @@ -31,7 +31,7 @@ Prerequisites Before you install nmdb, you will need the following software: - libevent_, a library for fast event handling. -- QDBM_, for the database backend. +- Either QDBM_, BDB_ or tokyocabinet_ for the database backend. And, if you're going to use TIPC_: @@ -42,10 +42,12 @@ Compiling and installing ------------------------ There are three components of the nmdb tarball: the server in the *nmdb/* -directory, the C library in *libnmdb/*, and the Python module in *python/*. +directory, the C library in *libnmdb/*, and the Python module in +*bindings/python/*. To install the server and the C library, run ``make install; ldconfig``. To -install the Python module, run ``make python_install``. +install the Python module, run ``make python_install`` after installing the C +library. If you want to disable support for some protocol (i.e. TIPC), you can do so by running ``make ENABLE_TIPC=0 install``. @@ -56,8 +58,7 @@ Quick start For a very quick start, using a single host, you can do the following:: - # dpmgr create /tmp/nmdb-db # create the backend database - # nmdb -d /tmp/nmdb-db # start the server + # nmdb -d /tmp/nmdb-db # start the server, use the given database At this point you have created a database and started the server. An easy and simple way to test it is to use the python module, like this:: @@ -72,7 +73,7 @@ simple way to test it is to use the python module, like this:: >>> db['x'] = 1 # store some data >>> db[(1, 2)] = (2, 6) >>> print db['x'], db[(1, 2)] # retreive the values - 1 (3, 5) + 1 (2, 6) >>> del db['x'] # delete from the database Everything should have worked as shown, and you are now ready to use some @@ -137,18 +138,17 @@ Cache size important issue if you have performance requirements. It is only possible to limit the cache size by the maximum number of objects - in the cache. + in it, and not by byte size. Backend database - You will need to create a backend database using QDBM_'s utilities. This is - quite simple, just run ``dpmgr create /path/to/the/database`` and you're - done. + The backend database engine can be selected at build time; QDBM_ is the + default. If for some reason (hardware failure, for instance) the database becomes - corrupt, you should use QDBM's utilities to fix it. It shouldn't happen, so - it's a good idea to report it if it does. + corrupt, you should use your database utilities to fix it. It shouldn't + happen, so it's a good idea to report it if it does. - QDBM databases are not meant to be shared among processes, so avoid having + Most databases are not meant to be shared among processes, so avoid having other processes using them. Database redundancy @@ -181,9 +181,8 @@ TIPC Port numbers everything will act weird. -Now that you know all that, starting a server should be quite simple: first -create the database as explained above, and then run the daemon with -``nmdb -d /path/to/the/database``. +Now that you know all that, starting a server should be quite simple: just run +the daemon with ``nmdb -d /path/to/the/database``. There are several options you can change at start time. Of course you won't remember all that (I know I don't), so check out ``nmdb -h`` to see a complete @@ -200,9 +199,9 @@ Example Following the previous example, if you want to start three servers you can do it like this:: - box1# ndbm -d /var/lib/nmdb/db-1 -l 11 - box2# ndbm -d /var/lib/nmdb/db-2 -l 12 - box3# ndbm -d /var/lib/nmdb/db-3 -l 13 + box1# nmdb -d /var/lib/nmdb/db-1 -l 11 + box2# nmdb -d /var/lib/nmdb/db-2 -l 12 + box3# nmdb -d /var/lib/nmdb/db-3 -l 13 Writing clients @@ -222,16 +221,17 @@ Thread safety that needs it. Available operations - You can request the server to do four operations: *set* a value to a key, + You can request the server to do five operations: *set* a value to a key, *get* the value associated with the given key, *delete* a given key (with - its associated value), and perform a *compare-and-swap* of the values - associated with the given key. + its associated value), perform a *compare-and-swap* of the values associated + with the given key, and (atomically) *increment* the value associated with + the given key. Request modes For each operation, you will have three different modes available: - A *normal mode*, which makes the operation impact on the database - asynchronously (ie. the functions return right after the operation was + asynchronously (i.e. the functions return right after the operation was queued, there is no completion notification). - A *synchronous mode* similar to the previous one, but when the functions return, the operation has hit the disk. @@ -310,7 +310,7 @@ as their associated values. Applying this technique is commonly known as We can use a local dictionary to cache the data, but that would mean we would have to write some cache management code to avoid using too much memory, and, -worse of all, each instance of the code running in the network would have its +worst of all, each instance of the code running in the network would have its own private cache and can't reuse calculations performed by other instances. Instead, we can use nmdb to make a cache that is shared among the network. @@ -409,18 +409,18 @@ simple clients to play with. If you are in doubt about something, you can consult the manpages or the documentation inside the *doc/* directory. -To get in touch with nmdb developers and users, you can join the `mailing -list`_, or just send an email to nmdb-devel@lists.auriga.wearlab.de. +If you want to report bugs, or have any questions or comments, just let me +know at albertito@blitiri.com.ar. -.. _nmdb: http://auriga.wearlab.de/~alb/nmdb/ +.. _nmdb: http://blitiri.com.ar/p/nmdb/ .. _libevent: http://www.monkey.org/~provos/libevent/ .. _TIPC: http://tipc.sf.net .. _memcached: http://www.danga.com/memcached/ -.. _QDBM: http://qdbm.sf.net .. _`Linux kernel`: http://kernel.org .. _tetrations: http://en.wikipedia.org/wiki/Tetration -.. _`mailing list`: - http://lists.auriga.wearlab.de/cgi-bin/mailman/listinfo/nmdb-devel +.. _QDBM: http://qdbm.sf.net +.. _BDB: http://www.oracle.com/technology/products/berkeley-db/db/ +.. _tokyocabinet: http://tokyocabinet.sf.net/index.html diff --git a/doc/network.rst b/doc/network.rst index c1e9909..13f0d36 100644 --- a/doc/network.rst +++ b/doc/network.rst @@ -2,7 +2,7 @@ ====================== nmdb_ Network Protocol ====================== -:Author: Alberto Bertogli (albertito@gmail.com) +:Author: Alberto Bertogli (albertito@blitiri.com.ar) **NOTE:** All integers are in network byte order. @@ -11,7 +11,8 @@ protocol. It can be used on top of TIPC, UDP, TCP (with a thin messaging layer) or SCTP. This document describes the protocol in a transport-independent way, assuming the transport protocol can send and receive messages reliably and preserve message boundaries. No ordering -guarantees are required. +guarantee is required for the request-reply part of the protocol, but it is +highly desirable to avoid reordering of requests. Requests @@ -20,23 +21,32 @@ Requests All requests begin with a common header, and then have a request-specific payload. They look like this:: - +-----+------------+------------------+--- - - - ---+ - | Ver | Request ID | Request code | Payload | - +-----+------------+------------------+--- - - - ---+ + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | Version | Request ID | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | Request code | Flags | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + : Payload : + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + Where the fields are: -Ver - Version of the protocol. 4 bits. Must be 1. -Request ID +Ver (4 bits) + Version of the protocol. Must be 1. +Request ID (28 bits) A number identifying the request. A request is uniquely represented by the *(ID, sender)* pair, where *sender* is the sender host information. IDs can - be reused once a matching response has arrived to the sender. 28 bits. -Request code + be reused once a matching response has arrived to the sender. +Request code (16 bits) The code of the operation to be performed. They're defined in the server - source code. 32 bits. -Payload - The payload is specific to the request code. Some requests can carry no + source code. +Flags (16 bits) + Flags that affect the request code. +Payload (variable, can be absent) + The payload is specific to the request code. Some requests carry no associated payload. @@ -50,38 +60,45 @@ completeness. ============== ====== Name Code ============== ====== -REQ_CACHE_GET 0x101 -REQ_CACHE_SET 0x102 -REQ_CACHE_DEL 0x103 -REQ_GET 0x104 -REQ_SET_SYNC 0x105 -REQ_DEL_SYNC 0x106 -REQ_SET_ASYNC 0x107 -REQ_DEL_ASYNC 0x108 -REQ_CACHE_CAS 0x109 -REQ_CAS 0x110 -REQ_CACHE_INCR 0x111 -REQ_INCR 0x112 +REQ_GET 0x101 +REQ_SET 0x102 +REQ_DEL 0x103 +REQ_CAS 0x104 +REQ_INCR 0x105 ============== ====== +Flags +----- + +Note that not all requests accept all the flags. Flags that are not relevant +for a given request will be ignored. + +================= ====== ============================================= + Name Code Relevant to +================= ====== ============================================= +FLAGS_CACHE_ONLY 1 REQ_GET, REQ_SET, REQ_DEL, REQ_CAS, REQ_INCR +FLAGS_SYNC 2 REQ_SET, REQ_DEL +================= ====== ============================================= + + Request payload formats ----------------------- -REQ_GET and REQ_CACHE_GET +REQ_GET These requests have the same payload format, and only differ on the code. First the key size (32 bits), and then the key. -REQ_SET_* and REQ_CACHE_SET +REQ_SET Like the previous requests, they share the payload format. First the key size (32 bits), then the value size (32 bits), then the key, and then the value. -REQ_DEL_* and REQ_CACHE_DEL +REQ_DEL You guessed it, they share the payload format too: first the key size (32 bits), and then the key. -REQ_CAS and REQ_CACHE_CAS +REQ_CAS First the key size, then the old value size, then the new value size, and then the key, the old value and the new value. -REQ_INCR and REQ_CACHE_INCR +REQ_INCR First the key size (32 bits), then the key, and then the increment as a signed network byte order 64 bit integer. @@ -140,7 +157,9 @@ REP_CACHE_HIT REP_OK Depending on the request, this reply does or doesn't have an associated value. For *REQ_SET**, *REQ_DEL** and *REQ_CAS** there is no payload. But - for *REQ_GET* the first 32 bits are the value size, and then the value. + for *REQ_GET* the first 32 bits are the value size, and then the value; and + for *REQ_INCR* the first 32 bits are the payload size, and then the + post-increment value as a signed 64-bit integer in network byte order. Reply error codes @@ -158,5 +177,5 @@ ERR_DB 0x106 Database error ============ ====== ========================= -.. _nmdb: http://auriga.wearlab.de/~alb/nmdb/ +.. _nmdb: http://blitiri.com.ar/p/nmdb/ diff --git a/libnmdb/Makefile b/libnmdb/Makefile index e0e0cfe..831cc3f 100644 --- a/libnmdb/Makefile +++ b/libnmdb/Makefile @@ -24,12 +24,19 @@ endif PREFIX=/usr/local -OBJS = libnmdb.o tcp.o tipc.o udp.o sctp.o +OBJS = libnmdb.o netutils.o tcp.o tipc.o udp.o sctp.o + + +ifneq ($(V), 1) + NICE_CC = @echo " CC $@"; $(CC) +else + NICE_CC = $(CC) +endif default: all -all: libs +all: libs libnmdb.pc nmdb.h: nmdb.skel.h @echo "generating nmdb.h" @@ -40,21 +47,29 @@ nmdb.h: nmdb.skel.h sed 's/++CONFIG_ENABLE_SCTP++/$(ENABLE_SCTP)/g' \ > nmdb.h +libnmdb.pc: libnmdb.skel.pc + @echo "generating libnmdb.pc" + @cat libnmdb.skel.pc | \ + sed 's@++PREFIX++@$(PREFIX)@g' \ + > libnmdb.pc + libs: libnmdb.so libnmdb.a libnmdb.so: nmdb.h $(OBJS) - $(CC) $(ALL_CFLAGS) -shared -fPIC $(OBJS) -o libnmdb.so + $(NICE_CC) $(ALL_CFLAGS) -shared -fPIC $(OBJS) -o libnmdb.so libnmdb.a: nmdb.h $(OBJS) $(AR) cr libnmdb.a $(OBJS) -install-lib: libs +install-lib: libs libnmdb.pc install -d $(PREFIX)/lib install -m 0755 libnmdb.so $(PREFIX)/lib install -m 0755 libnmdb.a $(PREFIX)/lib install -d $(PREFIX)/include install -m 0644 nmdb.h $(PREFIX)/include + install -d $(PREFIX)/lib/pkgconfig + install -m 644 libnmdb.pc $(PREFIX)/lib/pkgconfig @echo @echo "Please run ldconfig to update your library cache" @echo @@ -67,10 +82,10 @@ install: install-lib install-man .c.o: - $(CC) $(ALL_CFLAGS) -c $< -o $@ + $(NICE_CC) $(ALL_CFLAGS) -c $< -o $@ clean: - rm -f nmdb.h $(OBJS) libnmdb.so libnmdb.a + rm -f nmdb.h libnmdb.pc $(OBJS) libnmdb.so libnmdb.a rm -f *.bb *.bbg *.da *.gcov *.gcda *.gcno gmon.out .PHONY: default all libs install-lib install-man install clean diff --git a/libnmdb/internal.h b/libnmdb/internal.h index ce52cee..d80424c 100644 --- a/libnmdb/internal.h +++ b/libnmdb/internal.h @@ -10,7 +10,7 @@ #define SCTP_CONN 4 /* The ID code for requests is hardcoded for now, until asynchronous requests - * * are implemented. */ + * are implemented. */ #define ID_CODE 1 /* For a given buffer, how much into it should the generic library code write diff --git a/libnmdb/libnmdb.3 b/libnmdb/libnmdb.3 index 6d23c50..226255f 100644 --- a/libnmdb/libnmdb.3 +++ b/libnmdb/libnmdb.3 @@ -47,15 +47,15 @@ libnmdb - Library for interacting with a nmdb server .sp .BI "int nmdb_incr(nmdb_t *" db "," .BI " const unsigned char *" key " , size_t " ksize "," -.BI " int64_t " increment ");" +.BI " int64_t " increment ", int64_t *" newval ");" .BI "int nmdb_cache_incr(nmdb_t *" db "," .BI " const unsigned char *" key " , size_t " ksize "," -.BI " int64_t " increment ");" +.BI " int64_t " increment ", int64_t *" newval ");" .fi .SH DESCRIPTION libnmdb is a library for interacting with a nmdb(1) server. For more -information about it, see nmdb(1) or http://auriga.wearlab.de/~alb/nmdb. +information about it, see nmdb(1) or http://blitiri.com.ar/p/nmdb. The first step to access a server is to call .BR nmdb_init () @@ -157,7 +157,10 @@ bit number to add to the value (it can be negative), and returns 2 if the increment was performed, 1 if the value was not in the appropriate format, 0 if the key was not in the database/cache, or < 0 on failure. The value MUST be a NULL-terminated string with only a number in base 10 in it -(i.e. it must be parseable by strtoll(3)). +(i.e. it must be parseable by strtoll(3)). If the number was incremented +correctly and the +.I newval +parameter is not NULL, it will be set to the resulted value. .SH SEE ALSO @@ -165,11 +168,10 @@ The value MUST be a NULL-terminated string with only a number in base 10 in it .B TIPC (http://tipc.sf.net). .SH AUTHORS -Created by Alberto Bertogli (albertito@gmail.com). +Created by Alberto Bertogli (albertito@blitiri.com.ar). .SH CONTACT -To get in touch with developers and users, join the mailing list at -http://lists.auriga.wearlab.de/cgi-bin/mailman/listinfo/nmdb-devel or just -send an email to nmdb-devel@lists.auriga.wearlab.de. +If you want to report bugs, or have any questions or comments, just let me +know at albertito@blitiri.com.ar. diff --git a/libnmdb/libnmdb.c b/libnmdb/libnmdb.c index eb2a1d7..6cb612d 100644 --- a/libnmdb/libnmdb.c +++ b/libnmdb/libnmdb.c @@ -14,9 +14,10 @@ #include "udp.h" #include "sctp.h" #include "internal.h" +#include "netutils.h" -/* Compare two servers by their connection identifiers. It is used internally +/* Compares two servers by their connection identifiers. It is used internally * to keep the server array sorted with qsort(). */ int compare_servers(const void *s1, const void *s2) { @@ -72,8 +73,8 @@ int compare_servers(const void *s1, const void *s2) } -/* Like recv() but either fails, or returns a complete read; if we return less - * than count is because EOF was reached */ +/* Like recv(), but either fails, or returns a complete read; if we return + * less than count is because EOF was reached. */ ssize_t srecv(int fd, unsigned char *buf, size_t count, int flags) { ssize_t rv, c; @@ -95,7 +96,7 @@ ssize_t srecv(int fd, unsigned char *buf, size_t count, int flags) return count; } -/* Like srecv() but for send() */ +/* Like srecv(), but for send(). */ ssize_t ssend(int fd, const unsigned char *buf, size_t count, int flags) { ssize_t rv, c; @@ -117,9 +118,8 @@ ssize_t ssend(int fd, const unsigned char *buf, size_t count, int flags) return count; } -/* Create a nmdb_t and set the first server to port. If port is < 0, the - * standard port is used. */ -nmdb_t *nmdb_init() +/* Creates a nmdb_t. */ +nmdb_t *nmdb_init(void) { nmdb_t *db; @@ -147,10 +147,9 @@ int nmdb_free(nmdb_t *db) return 1; } -/* Used internally to send a buffer to the given server. Calls the appropriate - * sender according to the server protocol. */ +/* Sends a buffer to the given server. */ static int srv_send(struct nmdb_srv *srv, - const unsigned char *buf, size_t bsize) + unsigned char *buf, size_t bsize) { if (srv == NULL) return 0; @@ -169,6 +168,7 @@ static int srv_send(struct nmdb_srv *srv, } } +/* Gets a reply from the given server. */ static uint32_t get_rep(struct nmdb_srv *srv, unsigned char *buf, size_t bsize, unsigned char **payload, size_t *psize) @@ -190,6 +190,9 @@ static uint32_t get_rep(struct nmdb_srv *srv, } } +/* When a packet arrives, the message it contains begins on a + * protocol-dependant offset. This functions returns the offset to use when + * sending/receiving messages for the given server. */ static int srv_get_msg_offset(struct nmdb_srv *srv) { if (srv == NULL) @@ -229,7 +232,7 @@ static uint32_t checksum(const unsigned char *buf, size_t bsize) return ~sum; } -/* Used internally to select which server to use for the given key. */ +/* Selects which server to use for the given key. */ static struct nmdb_srv *select_srv(nmdb_t *db, const unsigned char *key, size_t ksize) { @@ -242,56 +245,103 @@ static struct nmdb_srv *select_srv(nmdb_t *db, return &(db->servers[n]); } +/* Creates a new buffer for packets. */ +static unsigned char *new_packet(struct nmdb_srv *srv, unsigned int request, + unsigned short flags, size_t *bufsize, size_t *payload_offset, + ssize_t payload_size) +{ + unsigned char *buf, *p; + unsigned int moff = srv_get_msg_offset(srv); + + if (payload_size == -1) { + /* Because our callers will reuse the buffer to get the reply, + * and we don't know how big it will be, we just alloc a bit + * over the max packet (64kb) */ + *bufsize = 68 * 1024; + } else { + /* 8 is the size of the common header */ + *bufsize = moff + 8 + payload_size; + } + buf = malloc(*bufsize); + if (buf == NULL) + return NULL; + + p = buf + moff; + + * (uint32_t *) p = htonl( (PROTO_VER << 28) | ID_CODE ); + * ((uint16_t *) p + 2) = htons(request); + * ((uint16_t *) p + 3) = htons(flags); + + if (payload_offset != NULL) + *payload_offset = moff + 8; + + return buf; +} + +/* Functions to append different numbers of (value, len) to the given buffer; + * it's not worth the trouble of making this generic because we never go past + * three and they're quite trivial. */ +static size_t append_1v(unsigned char *buf, + const unsigned char *datum, size_t dsize) +{ + * ((uint32_t *) buf) = htonl(dsize); + memcpy(buf + 4, datum, dsize); + return 4 + dsize; +} + +static size_t append_2v(unsigned char *buf, + const unsigned char *datum1, size_t dsize1, + const unsigned char *datum2, size_t dsize2) +{ + * ((uint32_t *) buf) = htonl(dsize1); + * ((uint32_t *) buf + 1) = htonl(dsize2); + memcpy(buf + 8, datum1, dsize1); + memcpy(buf + 8 + dsize1, datum2, dsize2); + return 8 + dsize1 + dsize2; +} + +static size_t append_3v(unsigned char *buf, + const unsigned char *datum1, size_t dsize1, + const unsigned char *datum2, size_t dsize2, + const unsigned char *datum3, size_t dsize3) +{ + * ((uint32_t *) buf) = htonl(dsize1); + * ((uint32_t *) buf + 1) = htonl(dsize2); + * ((uint32_t *) buf + 2) = htonl(dsize3); + memcpy(buf + 12, datum1, dsize1); + memcpy(buf + 12 + dsize1, datum2, dsize2); + memcpy(buf + 12 + dsize1 + dsize2, datum3, dsize3); + return 12 + dsize1 + dsize2 + dsize3; +} + +/* Functions to perform a get. */ static ssize_t do_get(nmdb_t *db, const unsigned char *key, size_t ksize, - unsigned char *val, size_t vsize, int impact_db) + unsigned char *val, size_t vsize, unsigned short flags) { - int moff; ssize_t rv, t; unsigned char *buf, *p; - size_t bsize, reqsize, psize = 0; - uint32_t request, reply; + size_t bufsize, reqsize, payload_offset, psize = 0; + uint32_t reply; struct nmdb_srv *srv; - if (impact_db) { - request = REQ_GET; - } else { - request = REQ_CACHE_GET; - } + flags = flags & NMDB_CACHE_ONLY; srv = select_srv(db, key, ksize); - moff = srv_get_msg_offset(srv); - - /* Use the same buffer for the request and the reply. - * Request: 4 bytes ver+id, 4 bytes request code, 4 bytes ksize, - * ksize bytes key. - * Reply: 4 bytes id, 4 bytes reply code, 4 bytes vsize, - * vsize bytes key. - * - * We don't know vsize beforehand, but we do know our max packet size - * is 64kb. We malloc 68kb just in case. - */ - bsize = 68 * 1024; - buf = malloc(bsize); + buf = new_packet(srv, REQ_GET, flags, &bufsize, &payload_offset, -1); if (buf == NULL) return -1; + reqsize = payload_offset; + reqsize += append_1v(buf + payload_offset, key, ksize); - p = buf + moff; - - * (uint32_t *) p = htonl( (PROTO_VER << 28) | ID_CODE ); - * ((uint32_t *) p + 1) = htonl(request); - * ((uint32_t *) p + 2) = htonl(ksize); - memcpy(p + 3 * 4, key, ksize); - reqsize = 3 * 4 + ksize; - - t = srv_send(srv, buf, moff + reqsize); + t = srv_send(srv, buf, reqsize); if (t <= 0) { rv = -2; goto exit; } - reply = get_rep(srv, buf, bsize, &p, &psize); + reply = get_rep(srv, buf, bufsize, &p, &psize); if (reply == REP_CACHE_MISS || reply == REP_NOTIN) { rv = -1; @@ -324,66 +374,45 @@ exit: ssize_t nmdb_get(nmdb_t *db, const unsigned char *key, size_t ksize, unsigned char *val, size_t vsize) { - return do_get(db, key, ksize, val, vsize, 1); + return do_get(db, key, ksize, val, vsize, 0); } ssize_t nmdb_cache_get(nmdb_t *db, const unsigned char *key, size_t ksize, unsigned char *val, size_t vsize) { - return do_get(db, key, ksize, val, vsize, 0); + return do_get(db, key, ksize, val, vsize, NMDB_CACHE_ONLY); } - +/* Functions to perform a set. */ static int do_set(nmdb_t *db, const unsigned char *key, size_t ksize, const unsigned char *val, size_t vsize, - int impact_db, int async) + unsigned short flags) { - int moff; ssize_t rv, t; - unsigned char *buf, *p; - size_t bsize; - uint32_t request, reply; + unsigned char *buf; + size_t bufsize, payload_offset, reqsize; + uint32_t reply; struct nmdb_srv *srv; - if (impact_db) { - if (async) - request = REQ_SET_ASYNC; - else - request = REQ_SET_SYNC; - } else { - request = REQ_CACHE_SET; - } + flags = flags & (NMDB_CACHE_ONLY | NMDB_SYNC); srv = select_srv(db, key, ksize); - moff = srv_get_msg_offset(srv); - - /* Use the same buffer for the request and the reply. - * Request: 4 bytes ver+id, 4 bytes request code, 4 bytes ksize, 4 - * bytes vsize, ksize bytes key, vsize bytes val. - * Reply: 4 bytes id, 4 bytes reply code. - */ - bsize = moff + 4 + 4 + 4 + 4 + ksize + vsize; - buf = malloc(bsize); + + buf = new_packet(srv, REQ_SET, flags, &bufsize, &payload_offset, + 4 * 2 + ksize + vsize); if (buf == NULL) return -1; + reqsize = payload_offset; + reqsize += append_2v(buf + payload_offset, key, ksize, val, vsize); - p = buf + moff; - - * (uint32_t *) p = htonl( (PROTO_VER << 28) | ID_CODE ); - * ((uint32_t *) p + 1) = htonl(request); - * ((uint32_t *) p + 2) = htonl(ksize); - * ((uint32_t *) p + 3) = htonl(vsize); - memcpy(p + 4 * 4, key, ksize); - memcpy(p + 4 * 4 + ksize, val, vsize); - - t = srv_send(srv, buf, bsize); + t = srv_send(srv, buf, reqsize); if (t <= 0) { rv = -1; goto exit; } - reply = get_rep(srv, buf, bsize, NULL, NULL); + reply = get_rep(srv, buf, bufsize, NULL, NULL); if (reply == REP_OK) { rv = 1; @@ -402,69 +431,50 @@ exit: int nmdb_set(nmdb_t *db, const unsigned char *key, size_t ksize, const unsigned char *val, size_t vsize) { - return do_set(db, key, ksize, val, vsize, 1, 1); + return do_set(db, key, ksize, val, vsize, 0); } int nmdb_set_sync(nmdb_t *db, const unsigned char *key, size_t ksize, const unsigned char *val, size_t vsize) { - return do_set(db, key, ksize, val, vsize, 1, 0); + return do_set(db, key, ksize, val, vsize, NMDB_SYNC); } int nmdb_cache_set(nmdb_t *db, const unsigned char *key, size_t ksize, const unsigned char *val, size_t vsize) { - return do_set(db, key, ksize, val, vsize, 0, 0); + return do_set(db, key, ksize, val, vsize, NMDB_CACHE_ONLY); } - +/* Functions to perform a del. */ static int do_del(nmdb_t *db, const unsigned char *key, size_t ksize, - int impact_db, int async) + unsigned short flags) { - int moff; ssize_t rv, t; - unsigned char *buf, *p; - size_t bsize; - uint32_t request, reply; + unsigned char *buf; + size_t bufsize, payload_offset, reqsize; + uint32_t reply; struct nmdb_srv *srv; - if (impact_db) { - if (async) - request = REQ_DEL_ASYNC; - else - request = REQ_DEL_SYNC; - } else { - request = REQ_CACHE_DEL; - } + flags = flags & (NMDB_CACHE_ONLY | NMDB_SYNC); srv = select_srv(db, key, ksize); - moff = srv_get_msg_offset(srv); - - /* Use the same buffer for the request and the reply. - * Request: 4 bytes ver+id, 4 bytes request code, 4 bytes ksize, - * ksize bytes key. - * Reply: 4 bytes id, 4 bytes reply code. - */ - bsize = moff + 8 + 4 + ksize; - buf = malloc(bsize); + + buf = new_packet(srv, REQ_DEL, flags, &bufsize, &payload_offset, + 4 + ksize); if (buf == NULL) return -1; + reqsize = payload_offset; + reqsize += append_1v(buf + payload_offset, key, ksize); - p = buf + moff; - - * (uint32_t *) p = htonl( (PROTO_VER << 28) | ID_CODE ); - * ((uint32_t *) p + 1) = htonl(request); - * ((uint32_t *) p + 2) = htonl(ksize); - memcpy(p + 3 * 4, key, ksize); - - t = srv_send(srv, buf, bsize); + t = srv_send(srv, buf, reqsize); if (t <= 0) { rv = -1; goto exit; } - reply = get_rep(srv, buf, bsize, NULL, NULL); + reply = get_rep(srv, buf, bufsize, NULL, NULL); if (reply == REP_OK) { rv = 1; @@ -485,72 +495,51 @@ exit: int nmdb_del(nmdb_t *db, const unsigned char *key, size_t ksize) { - return do_del(db, key, ksize, 1, 1); + return do_del(db, key, ksize, 0); } int nmdb_del_sync(nmdb_t *db, const unsigned char *key, size_t ksize) { - return do_del(db, key, ksize, 1, 0); + return do_del(db, key, ksize, NMDB_SYNC); } int nmdb_cache_del(nmdb_t *db, const unsigned char *key, size_t ksize) { - return do_del(db, key, ksize, 0, 0); + return do_del(db, key, ksize, NMDB_CACHE_ONLY); } +/* Functions to perform a CAS. */ static int do_cas(nmdb_t *db, const unsigned char *key, size_t ksize, const unsigned char *oldval, size_t ovsize, const unsigned char *newval, size_t nvsize, - int impact_db) + unsigned short flags) { - int moff; ssize_t rv, t; - unsigned char *buf, *p, *q; - size_t bsize; - uint32_t request, reply; + unsigned char *buf; + size_t bufsize, payload_offset, reqsize; + uint32_t reply; struct nmdb_srv *srv; - request = REQ_CACHE_CAS; - if (impact_db) - request = REQ_CAS; + flags = flags & NMDB_CACHE_ONLY; srv = select_srv(db, key, ksize); - moff = srv_get_msg_offset(srv); - - /* Use the same buffer for the request and the reply. - * Request: 4 bytes ver+id, 4 bytes request code, 4 bytes ksize, 4 - * bytes ovsize, 4 bytes nvsize, ksize bytes key, - * ovsize bytes oldval, nvsize bytes newval. - * Reply: 4 bytes id, 4 bytes reply code. - */ - bsize = moff + 4 + 4 + 4 + 4 + 4 + ksize + ovsize + nvsize; - buf = malloc(bsize); + + buf = new_packet(srv, REQ_CAS, flags, &bufsize, &payload_offset, + 4 * 3 + ksize + ovsize + nvsize); if (buf == NULL) return -1; + reqsize = payload_offset; + reqsize += append_3v(buf + payload_offset, key, ksize, oldval, ovsize, + newval, nvsize); - p = buf + moff; - - * (uint32_t *) p = htonl( (PROTO_VER << 28) | ID_CODE ); - * ((uint32_t *) p + 1) = htonl(request); - * ((uint32_t *) p + 2) = htonl(ksize); - * ((uint32_t *) p + 3) = htonl(ovsize); - * ((uint32_t *) p + 4) = htonl(nvsize); - q = p + 5 * 4; - memcpy(q, key, ksize); - q += ksize; - memcpy(q, oldval, ovsize); - q += ovsize; - memcpy(q, newval, nvsize); - - srv = select_srv(db, key, ksize); - t = srv_send(srv, buf, bsize); + t = srv_send(srv, buf, reqsize); if (t <= 0) { rv = -1; goto exit; } - reply = get_rep(srv, buf, bsize, NULL, NULL); + reply = get_rep(srv, buf, bufsize, NULL, NULL); if (reply == REP_OK) { rv = 2; @@ -576,92 +565,60 @@ int nmdb_cas(nmdb_t *db, const unsigned char *key, size_t ksize, const unsigned char *oldval, size_t ovsize, const unsigned char *newval, size_t nvsize) { - return do_cas(db, key, ksize, oldval, ovsize, newval, nvsize, 1); + return do_cas(db, key, ksize, oldval, ovsize, newval, nvsize, 0); } int nmdb_cache_cas(nmdb_t *db, const unsigned char *key, size_t ksize, const unsigned char *oldval, size_t ovsize, const unsigned char *newval, size_t nvsize) { - return do_cas(db, key, ksize, oldval, ovsize, newval, nvsize, 0); + return do_cas(db, key, ksize, oldval, ovsize, newval, nvsize, + NMDB_CACHE_ONLY); } -/* ntohll() is not standard, so we define it using an UGLY trick because there - * is no standard way to check for endianness at runtime! (same problem as the - * one in nmdb/parse.c) */ -static uint64_t htonll(uint64_t x) -{ - static int endianness = 0; - - /* determine the endianness by checking how htonl() behaves; use -1 - * for little endian and 1 for big endian */ - if (endianness == 0) { - if (htonl(1) == 1) - endianness = 1; - else - endianness = -1; - } - - if (endianness == 1) { - /* big endian */ - return x; - } - - /* little endian */ - return ( htonl( (x >> 32) & 0xFFFFFFFF ) | \ - ( (uint64_t) htonl(x & 0xFFFFFFFF) ) << 32 ); -} - +/* Functions to perform an atomic increment. */ static int do_incr(nmdb_t *db, const unsigned char *key, size_t ksize, - int64_t increment, int impact_db) + int64_t increment, int64_t *newval, unsigned short flags) { - int moff; ssize_t rv, t; - unsigned char *buf, *p; - size_t bsize; - uint32_t request, reply; + unsigned char *buf, *payload; + size_t bufsize, payload_offset, reqsize, psize; + uint32_t reply; struct nmdb_srv *srv; - if (impact_db) - request = REQ_INCR; - else - request = REQ_CACHE_INCR; + flags = flags & NMDB_CACHE_ONLY; srv = select_srv(db, key, ksize); - moff = srv_get_msg_offset(srv); - - /* Use the same buffer for the request and the reply. - * Request: 4 bytes ver+id, 4 bytes request code, 4 bytes ksize, - * ksize bytes key, 8 bytes increment. - * Reply: 4 bytes id, 4 bytes reply code. - */ - bsize = moff + 4 + 4 + 4 + ksize + 8; - buf = malloc(bsize); - if (buf == NULL) - return -1; increment = htonll(increment); - p = buf + moff; - - * (uint32_t *) p = htonl( (PROTO_VER << 28) | ID_CODE ); - * ((uint32_t *) p + 1) = htonl(request); - * ((uint32_t *) p + 2) = htonl(ksize); - memcpy(p + 3 * 4, key, ksize); - memcpy(p + 3 * 4 + ksize, &increment, sizeof(int64_t)); + buf = new_packet(srv, REQ_INCR, flags, &bufsize, &payload_offset, + 4 + ksize + sizeof(int64_t)); + if (buf == NULL) + return -1; + reqsize = payload_offset; + reqsize += append_1v(buf + payload_offset, key, ksize); + memcpy(buf + reqsize, &increment, sizeof(int64_t)); + reqsize += sizeof(int64_t); - t = srv_send(srv, buf, bsize); + t = srv_send(srv, buf, reqsize); if (t <= 0) { rv = -1; goto exit; } - reply = get_rep(srv, buf, bsize, NULL, NULL); + psize = sizeof(int64_t); + reply = get_rep(srv, buf, bufsize, &payload, &psize); switch (reply) { case REP_OK: rv = 2; + if (newval != NULL && psize == sizeof(int64_t) + 4) { + /* skip the 4 bytes of length */ + *newval = *((int64_t *) (payload + 4)); + *newval = ntohll(*newval); + } break; case REP_NOMATCH: rv = 1; @@ -681,15 +638,89 @@ exit: } int nmdb_incr(nmdb_t *db, const unsigned char *key, size_t ksize, - int64_t increment) + int64_t increment, int64_t *newval) { - return do_incr(db, key, ksize, increment, 1); + return do_incr(db, key, ksize, increment, newval, 0); } int nmdb_cache_incr(nmdb_t *db, const unsigned char *key, size_t ksize, - int64_t increment) + int64_t increment, int64_t *newval) +{ + return do_incr(db, key, ksize, increment, newval, NMDB_CACHE_ONLY); +} + + +/* Request servers' statistics, return the aggregated results in buf, with the + * number of servers in nservers and the number of stats per server in nstats. + * Used in the "nmdb-stats" utility, matches the server version. + * + * Return: + * 1 if success + * -1 if there was an error in the server + * -2 if there was a network error + * -3 if the buffer was too small + * -4 if the server replies were of different size (indicates different + * server versions, not supported at the time) + * + * TODO: The API could be improved by having nstats be provided by the caller, + * and making sure its used by all servers. Also buf should be an uint64_t + * *buf to make the typing more explicit. */ +int nmdb_stats(nmdb_t *db, unsigned char *buf, size_t bsize, + unsigned int *nservers, unsigned int *nstats) { - return do_incr(db, key, ksize, increment, 0); + int i; + size_t reqsize; + ssize_t t, reply; + unsigned char *request; + struct nmdb_srv *srv; + + /* This buffer is used for a single reply, must be big enough to + * hold STATS_REPLY_SIZE. 32 elements is enough to allow future + * improvements. */ + unsigned char tmpbuf[32 * sizeof(uint64_t)]; + size_t tmpbufsize = 32 * sizeof(uint64_t); + + unsigned char *payload; + size_t psize; + + *nstats = 0; + + for (i = 0; i < db->nservers; i++) { + srv = db->servers + i; + request = new_packet(srv, REQ_STATS, 0, &reqsize, NULL, 0); + + t = srv_send(srv, request, reqsize); + free(request); + if (t <= 0) + return -2; + + reply = get_rep(srv, tmpbuf, tmpbufsize, &payload, &psize); + if (reply != REP_OK) + return -1; + + /* Check if there is enough room for the copy */ + if (bsize < psize) + return -3; + + /* Now copy the reply in the buffer given to us, skipping the + * 4 bytes of length */ + memcpy(buf, payload + 4, psize); + buf += psize; + bsize -= psize; + + /* nstats should be the same for all the servers; if not, + * return error because the caller can't cope with that + * situation */ + if (*nstats == 0) { + *nstats = psize / sizeof(uint64_t); + } else if (*nstats != psize / sizeof(uint64_t)) { + return -4; + } + } + + *nservers = db->nservers; + + return 1; } diff --git a/libnmdb/libnmdb.skel.pc b/libnmdb/libnmdb.skel.pc new file mode 100644 index 0000000..f0f3c01 --- /dev/null +++ b/libnmdb/libnmdb.skel.pc @@ -0,0 +1,12 @@ + +prefix=++PREFIX++ +libdir=${prefix}/lib +includedir=${prefix}/include + +Name: libnmdb +Description: Library for interacting with a nmdb server +URL: http://blitiri.com.ar/p/nmdb/ +Version: 0.22 +Libs: -L${libdir} -lnmdb +Cflags: -I${includedir} + diff --git a/libnmdb/net-const.h b/libnmdb/net-const.h deleted file mode 100644 index b9e875a..0000000 --- a/libnmdb/net-const.h +++ /dev/null @@ -1,61 +0,0 @@ - -#ifndef _NET_CONST_H -#define _NET_CONST_H - -/* - * Local network constants. - * Isolated so it's shared between the server and the library code. - */ - -/* TIPC server type (hardcoded) and default instance. */ -#define TIPC_SERVER_TYPE 26001 -#define TIPC_SERVER_INST 10 - -/* TCP default listen address and port. */ -#define TCP_SERVER_ADDR "0.0.0.0" -#define TCP_SERVER_PORT 26010 - -/* UDP default listen address and port. */ -#define UDP_SERVER_ADDR "0.0.0.0" -#define UDP_SERVER_PORT 26010 - -/* SCTP default listen address and port. */ -#define SCTP_SERVER_ADDR "0.0.0.0" -#define SCTP_SERVER_PORT 26010 - -/* Protocol version, for checking in the network header. */ -#define PROTO_VER 1 - -/* Network requests */ -#define REQ_CACHE_GET 0x101 -#define REQ_CACHE_SET 0x102 -#define REQ_CACHE_DEL 0x103 -#define REQ_GET 0x104 -#define REQ_SET_SYNC 0x105 -#define REQ_DEL_SYNC 0x106 -#define REQ_SET_ASYNC 0x107 -#define REQ_DEL_ASYNC 0x108 -#define REQ_CACHE_CAS 0x109 -#define REQ_CAS 0x110 -#define REQ_CACHE_INCR 0x111 -#define REQ_INCR 0x112 - -/* Network replies (different namespace from requests) */ -#define REP_ERR 0x800 -#define REP_CACHE_HIT 0x801 -#define REP_CACHE_MISS 0x802 -#define REP_OK 0x803 -#define REP_NOTIN 0x804 -#define REP_NOMATCH 0x805 - -/* Network error replies */ -#define ERR_VER 0x101 /* Version mismatch */ -#define ERR_SEND 0x102 /* Error sending data */ -#define ERR_BROKEN 0x103 /* Broken request */ -#define ERR_UNKREQ 0x104 /* Unknown request */ -#define ERR_MEM 0x105 /* Memory allocation error */ -#define ERR_DB 0x106 /* Database error */ - - -#endif - diff --git a/libnmdb/net-const.h b/libnmdb/net-const.h new file mode 120000 index 0000000..e088a86 --- /dev/null +++ b/libnmdb/net-const.h @@ -0,0 +1 @@ +../nmdb/net-const.h \ No newline at end of file diff --git a/libnmdb/netutils.c b/libnmdb/netutils.c new file mode 120000 index 0000000..73cb507 --- /dev/null +++ b/libnmdb/netutils.c @@ -0,0 +1 @@ +../nmdb/netutils.c \ No newline at end of file diff --git a/libnmdb/netutils.h b/libnmdb/netutils.h new file mode 120000 index 0000000..24df4a2 --- /dev/null +++ b/libnmdb/netutils.h @@ -0,0 +1 @@ +../nmdb/netutils.h \ No newline at end of file diff --git a/libnmdb/nmdb.skel.h b/libnmdb/nmdb.skel.h index 4c8b915..448fb15 100644 --- a/libnmdb/nmdb.skel.h +++ b/libnmdb/nmdb.skel.h @@ -51,6 +51,14 @@ typedef struct nmdb_t { struct nmdb_srv *servers; } nmdb_t; + +/* Possible flags, notice it may make no sense to mix them, consult the + * documentation before doing weird things. Values should be kept in sync with + * the internal net-const.h */ +#define NMDB_CACHE_ONLY 1 +#define NMDB_SYNC 2 + + nmdb_t *nmdb_init(); int nmdb_add_tipc_server(nmdb_t *db, int port); int nmdb_add_tcp_server(nmdb_t *db, const char *addr, int port); @@ -82,9 +90,12 @@ int nmdb_cache_cas(nmdb_t *db, const unsigned char *key, size_t ksize, const unsigned char *newval, size_t nvsize); int nmdb_incr(nmdb_t *db, const unsigned char *key, size_t ksize, - int64_t increment); + int64_t increment, int64_t *newval); int nmdb_cache_incr(nmdb_t *db, const unsigned char *key, size_t ksize, - int64_t increment); + int64_t increment, int64_t *newval); + +int nmdb_stats(nmdb_t *db, unsigned char *buf, size_t bsize, + unsigned int *nservers, unsigned int *nstats); #endif diff --git a/libnmdb/sctp.c b/libnmdb/sctp.c index 1f1c1ab..427c03b 100644 --- a/libnmdb/sctp.c +++ b/libnmdb/sctp.c @@ -15,6 +15,7 @@ #include "nmdb.h" #include "net-const.h" #include "internal.h" +#include "sctp.h" /* Used internally to really add the server once we have an IP address. */ @@ -85,8 +86,7 @@ int nmdb_add_sctp_server(nmdb_t *db, const char *addr, int port) return add_sctp_server_addr(db, &(ia.s_addr), port); } -int sctp_srv_send(struct nmdb_srv *srv, - const unsigned char *buf, size_t bsize) +int sctp_srv_send(struct nmdb_srv *srv, unsigned char *buf, size_t bsize) { ssize_t rv; rv = sendto(srv->fd, buf, bsize, 0, @@ -131,14 +131,14 @@ uint32_t sctp_get_rep(struct nmdb_srv *srv, #include #include "nmdb.h" +#include "sctp.h" int nmdb_add_sctp_server(nmdb_t *db, const char *addr, int port) { return 0; } -int sctp_srv_send(struct nmdb_srv *srv, - const unsigned char *buf, size_t bsize) +int sctp_srv_send(struct nmdb_srv *srv, unsigned char *buf, size_t bsize) { return 0; } diff --git a/libnmdb/sctp.h b/libnmdb/sctp.h index 56b0add..21eb074 100644 --- a/libnmdb/sctp.h +++ b/libnmdb/sctp.h @@ -2,8 +2,7 @@ #ifndef _SCTP_H #define _SCTP_H -int sctp_srv_send(struct nmdb_srv *srv, - const unsigned char *buf, size_t bsize); +int sctp_srv_send(struct nmdb_srv *srv, unsigned char *buf, size_t bsize); uint32_t sctp_get_rep(struct nmdb_srv *srv, unsigned char *buf, size_t bsize, unsigned char **payload, size_t *psize); diff --git a/libnmdb/tcp.c b/libnmdb/tcp.c index 0bee671..22d5d83 100644 --- a/libnmdb/tcp.c +++ b/libnmdb/tcp.c @@ -15,6 +15,7 @@ #include "nmdb.h" #include "net-const.h" #include "internal.h" +#include "tcp.h" /* Used internally to really add the server once we have an IP address. */ @@ -102,8 +103,7 @@ int nmdb_add_tcp_server(nmdb_t *db, const char *addr, int port) return add_tcp_server_addr(db, &(ia.s_addr), port); } -int tcp_srv_send(struct nmdb_srv *srv, - unsigned char *buf, size_t bsize) +int tcp_srv_send(struct nmdb_srv *srv, unsigned char *buf, size_t bsize) { ssize_t rv; uint32_t len; @@ -187,14 +187,14 @@ uint32_t tcp_get_rep(struct nmdb_srv *srv, #include #include "nmdb.h" +#include "tcp.h" int nmdb_add_tcp_server(nmdb_t *db, const char *addr, int port) { return 0; } -int tcp_srv_send(struct nmdb_srv *srv, - const unsigned char *buf, size_t bsize) +int tcp_srv_send(struct nmdb_srv *srv, unsigned char *buf, size_t bsize) { return 0; } diff --git a/libnmdb/tcp.h b/libnmdb/tcp.h index 4252353..1459d42 100644 --- a/libnmdb/tcp.h +++ b/libnmdb/tcp.h @@ -2,8 +2,7 @@ #ifndef _TCP_H #define _TCP_H -int tcp_srv_send(struct nmdb_srv *srv, - const unsigned char *buf, size_t bsize); +int tcp_srv_send(struct nmdb_srv *srv, unsigned char *buf, size_t bsize); uint32_t tcp_get_rep(struct nmdb_srv *srv, unsigned char *buf, size_t bsize, unsigned char **payload, size_t *psize); diff --git a/libnmdb/tipc.c b/libnmdb/tipc.c index 6482352..21b6eb2 100644 --- a/libnmdb/tipc.c +++ b/libnmdb/tipc.c @@ -14,6 +14,7 @@ #include "nmdb.h" #include "net-const.h" #include "internal.h" +#include "tipc.h" /* Add a TIPC server to the db connection. Requests will select which server @@ -107,6 +108,7 @@ uint32_t tipc_get_rep(struct nmdb_srv *srv, #include #include "nmdb.h" +#include "tipc.h" int nmdb_add_tipc_server(nmdb_t *db, int port) { diff --git a/libnmdb/udp.c b/libnmdb/udp.c index 55902d2..7e690fe 100644 --- a/libnmdb/udp.c +++ b/libnmdb/udp.c @@ -15,6 +15,7 @@ #include "nmdb.h" #include "net-const.h" #include "internal.h" +#include "udp.h" /* Used internally to really add the server once we have an IP address. */ @@ -77,8 +78,7 @@ int nmdb_add_udp_server(nmdb_t *db, const char *addr, int port) return add_udp_server_addr(db, &(ia.s_addr), port); } -int udp_srv_send(struct nmdb_srv *srv, - const unsigned char *buf, size_t bsize) +int udp_srv_send(struct nmdb_srv *srv, unsigned char *buf, size_t bsize) { ssize_t rv; rv = sendto(srv->fd, buf, bsize, 0, @@ -123,14 +123,14 @@ uint32_t udp_get_rep(struct nmdb_srv *srv, #include #include "nmdb.h" +#include "udp.h" int nmdb_add_udp_server(nmdb_t *db, const char *addr, int port) { return 0; } -int udp_srv_send(struct nmdb_srv *srv, - const unsigned char *buf, size_t bsize) +int udp_srv_send(struct nmdb_srv *srv, unsigned char *buf, size_t bsize) { return 0; } diff --git a/libnmdb/udp.h b/libnmdb/udp.h index 7515941..c989a83 100644 --- a/libnmdb/udp.h +++ b/libnmdb/udp.h @@ -2,8 +2,7 @@ #ifndef _UDP_H #define _UDP_H -int udp_srv_send(struct nmdb_srv *srv, - const unsigned char *buf, size_t bsize); +int udp_srv_send(struct nmdb_srv *srv, unsigned char *buf, size_t bsize); uint32_t udp_get_rep(struct nmdb_srv *srv, unsigned char *buf, size_t bsize, unsigned char **payload, size_t *psize); diff --git a/nmdb/Makefile b/nmdb/Makefile index c3e103a..66a9f14 100644 --- a/nmdb/Makefile +++ b/nmdb/Makefile @@ -5,11 +5,11 @@ ENABLE_TCP = 1 ENABLE_UDP = 1 ENABLE_SCTP = 1 -# Backend to use, can be qdbm, bdb, or null +# Backend to use, can be qdbm, bdb, tc, or null BACKEND = qdbm CFLAGS += -std=c99 -pedantic -Wall -O3 -ALL_CFLAGS = -D_XOPEN_SOURCE=500 $(CFLAGS) +ALL_CFLAGS = -D_XOPEN_SOURCE=600 $(CFLAGS) ALL_CFLAGS += -DENABLE_TIPC=$(ENABLE_TIPC) \ -DENABLE_TCP=$(ENABLE_TCP) \ -DENABLE_UDP=$(ENABLE_UDP) \ @@ -20,7 +20,11 @@ ALL_CFLAGS += -g endif ifdef PROFILE -ALL_CFLAGS += -g -pg -fprofile-arcs -ftest-coverage +ALL_CFLAGS += -g -pg -ftest-coverage -fprofile-generate +endif + +ifdef PROFILE_USE +ALL_CFLAGS += -fprofile-use endif @@ -28,7 +32,7 @@ endif PREFIX=/usr/local -OBJS = cache.o dbloop.o queue.o log.o net.o parse.o main.o +OBJS = cache.o dbloop.o queue.o log.o net.o netutils.o parse.o stats.o main.o LIBS = -levent -lpthread -lrt @@ -61,29 +65,41 @@ endif # gmake 3.81, which is too new. ifeq ($(BACKEND), qdbm) OBJS += be-qdbm.o - ALL_CFLAGS += -DBACKEND_QDBM - LIBS += -lqdbm + ALL_CFLAGS += `pkg-config qdbm --cflags` -DBACKEND_QDBM + LIBS += `pkg-config qdbm --libs-only-L` -lqdbm endif ifeq ($(BACKEND), bdb) OBJS += be-bdb.o ALL_CFLAGS += -DBACKEND_BDB LIBS += -ldb endif +ifeq ($(BACKEND), tc) + OBJS += be-tc.o + ALL_CFLAGS += `pkg-config tokyocabinet --cflags` -DBACKEND_TC + LIBS += `pkg-config tokyocabinet --libs` +endif ifeq ($(BACKEND), null) OBJS += be-null.o ALL_CFLAGS += -DBACKEND_NULL endif +ifneq ($(V), 1) + NICE_CC = @echo " CC $@"; $(CC) +else + NICE_CC = $(CC) +endif + + default: all all: nmdb nmdb: $(OBJS) - $(CC) $(ALL_CFLAGS) $(OBJS) $(LIBS) -o nmdb + $(NICE_CC) $(ALL_CFLAGS) $(OBJS) $(LIBS) -o nmdb .c.o: - $(CC) $(ALL_CFLAGS) -c $< -o $@ + $(NICE_CC) $(ALL_CFLAGS) -c $< -o $@ install-bin: nmdb install -d $(PREFIX)/bin @@ -95,10 +111,15 @@ install-man: install: install-bin install-man -clean: +clean: clean-build clean-prof + +clean-build: rm -f $(OBJS) nmdb - rm -f *.bb *.bbg *.da *.gcov *.gcda *.gcno gmon.out -.PHONY: default all clean install-bin install-man install +clean-prof: + rm -f *.bb *.bbg *.da *.gcov *.gcda *.gcno gmon.out +.PHONY: default all \ + install-bin install-man install \ + clean-build clean-prof clean diff --git a/nmdb/be-tc.c b/nmdb/be-tc.c new file mode 100644 index 0000000..11f5267 --- /dev/null +++ b/nmdb/be-tc.c @@ -0,0 +1,52 @@ + +#include /* Tokyo Cabinet's hash API */ +#include + +#include "be.h" + + +db_t *db_open(const char *name, int flags) +{ + db_t *db = tchdbnew(); + + if (!tchdbopen(db, name, HDBOWRITER | HDBOCREAT)) + return NULL; + + return db; +} + + +int db_close(db_t *db) +{ + int r = tchdbclose(db); + tchdbdel(db); + return r; +} + + +int db_set(db_t *db, const unsigned char *key, size_t ksize, + unsigned char *val, size_t vsize) +{ + return tchdbput(db, key, ksize, val, vsize); +} + + +int db_get(db_t *db, const unsigned char *key, size_t ksize, + unsigned char *val, size_t *vsize) +{ + int rv; + + rv = tchdbget3(db, key, ksize, val, *vsize); + if (rv >= 0) { + *vsize = rv; + return 1; + } else { + return 0; + } +} + +int db_del(db_t *db, const unsigned char *key, size_t ksize) +{ + return tchdbout(db, key, ksize); +} + diff --git a/nmdb/be.h b/nmdb/be.h index fd8e384..d333b59 100644 --- a/nmdb/be.h +++ b/nmdb/be.h @@ -7,14 +7,21 @@ #if defined BACKEND_QDBM #include typedef DEPOT db_t; + #elif defined BACKEND_BDB /* typedefs to work around db.h include bug */ typedef unsigned int u_int; typedef unsigned long u_long; #include typedef DB db_t; + +#elif defined BACKEND_TC + #include + typedef TCHDB db_t; + #elif defined BACKEND_NULL typedef int db_t; + #else #error "Unknown backend" /* Define it anyway, so this is the only warning/error the user sees. */ diff --git a/nmdb/cache.c b/nmdb/cache.c index 5f8ae14..71f2f93 100644 --- a/nmdb/cache.c +++ b/nmdb/cache.c @@ -106,48 +106,51 @@ static uint32_t hash(const unsigned char *key, const size_t ksize) } -/* Looks given key up in the chain. Returns NULL if not found, or a pointer to - * the cache entry if it's found. The chain can be empty. Used in cache_get() - * and cache_set(). */ +/* Looks up the given key in the chain. Returns NULL if not found, or a + * pointer to the cache entry if it is. The chain can be empty. */ static struct cache_entry *find_in_chain(struct cache_chain *c, const unsigned char *key, size_t ksize) { - int found = 0; struct cache_entry *e; - e = c->first; - while (e != NULL) { + for (e = c->first; e != NULL; e = e->next) { if (ksize != e->ksize) { - e = e->next; continue; } if (memcmp(key, e->key, ksize) == 0) { - found = 1; break; } - - e = e->next; } - if (found) - return e; - return NULL; + /* e will be either the found chain or NULL */ + return e; +} + + +/* Looks up the given key in the cache. Returns NULL if not found, or a + * pointer to the cache entry if it is. Useful to avoid doing the calculation + * in the open when the cache chain will not be needed. */ +static struct cache_entry *find_in_cache(struct cache *cd, + const unsigned char *key, size_t ksize) +{ + uint32_t h; + struct cache_chain *c; + h = hash(key, ksize) % cd->hashlen; + c = cd->table + h; + + return find_in_chain(c, key, ksize); } + /* Gets the matching value for the given key. Returns 0 if no match was * found, or 1 otherwise. */ int cache_get(struct cache *cd, const unsigned char *key, size_t ksize, unsigned char **val, size_t *vsize) { - uint32_t h = 0; - struct cache_chain *c; struct cache_entry *e; - h = hash(key, ksize) % cd->hashlen; - c = cd->table + h; - - e = find_in_chain(c, key, ksize); + e = find_in_cache(cd, key, ksize); if (e == NULL) { *val = NULL; @@ -314,15 +317,10 @@ int cache_cas(struct cache *cd, const unsigned char *key, size_t ksize, const unsigned char *newval, size_t nvsize) { int rv = 1; - uint32_t h = 0; - struct cache_chain *c; struct cache_entry *e; unsigned char *buf; - h = hash(key, ksize) % cd->hashlen; - c = cd->table + h; - - e = find_in_chain(c, key, ksize); + e = find_in_cache(cd, key, ksize); if (e == NULL) { rv = -1; @@ -363,21 +361,19 @@ exit: * -1 if the value was not in the cache. * -2 if the value was not null terminated. * -3 if there was a memory error. + * + * The new value will be set in the newval parameter if the increment was + * successful. */ int cache_incr(struct cache *cd, const unsigned char *key, size_t ksize, - int64_t increment) + int64_t increment, int64_t *newval) { - uint32_t h = 0; - unsigned char *val, *newval; + unsigned char *val; int64_t intval; size_t vsize; - struct cache_chain *c; struct cache_entry *e; - h = hash(key, ksize) % cd->hashlen; - c = cd->table + h; - - e = find_in_chain(c, key, ksize); + e = find_in_cache(cd, key, ksize); if (e == NULL) return -1; @@ -397,15 +393,16 @@ int cache_incr(struct cache *cd, const unsigned char *key, size_t ksize, * and strlen('18446744073709551615') = 20, so if the value is smaller * than 24 (just in case) we create a new buffer. */ if (vsize < 24) { - newval = malloc(24); - if (newval == NULL) + unsigned char *nv = malloc(24); + if (nv == NULL) return -3; free(val); - e->val = val = newval; + e->val = val = nv; e->vsize = vsize = 24; } snprintf((char *) val, vsize, "%23lld", (long long int) intval); + *newval = intval; return 1; } diff --git a/nmdb/cache.h b/nmdb/cache.h index c4d3d7f..039dd91 100644 --- a/nmdb/cache.h +++ b/nmdb/cache.h @@ -49,7 +49,7 @@ int cache_cas(struct cache *cd, const unsigned char *key, size_t ksize, const unsigned char *oldval, size_t ovsize, const unsigned char *newval, size_t nvsize); int cache_incr(struct cache *cd, const unsigned char *key, size_t ksize, - int64_t increment); + int64_t increment, int64_t *newval); #endif diff --git a/nmdb/common.h b/nmdb/common.h index 1a872c7..a1b129e 100644 --- a/nmdb/common.h +++ b/nmdb/common.h @@ -31,11 +31,8 @@ struct settings { extern struct settings settings; /* Statistics */ -struct stats { - unsigned long net_version_mismatch; - unsigned long net_broken_req; - unsigned long net_unk_req; -}; +#include "stats.h" extern struct stats stats; + #endif diff --git a/nmdb/dbloop.c b/nmdb/dbloop.c index 998720d..bb20504 100644 --- a/nmdb/dbloop.c +++ b/nmdb/dbloop.c @@ -13,6 +13,8 @@ #include "net-const.h" #include "req.h" #include "log.h" +#include "netutils.h" +#include "sparse.h" static void *db_loop(void *arg); @@ -73,6 +75,9 @@ static void *db_loop(void *arg) if (rv != 0 && rv != ETIMEDOUT) { errlog("Error in queue_timedwait()"); + /* When the timedwait fails the lock is released, so + * we need to properly annotate this case. */ + __release(op_queue->lock); continue; } @@ -100,17 +105,17 @@ static void *db_loop(void *arg) static void process_op(db_t *db, struct queue_entry *e) { int rv; - if (e->operation == REQ_SET_SYNC) { + if (e->operation == REQ_SET) { rv = db_set(db, e->key, e->ksize, e->val, e->vsize); + if (!(e->req->flags & FLAGS_SYNC)) + return; + if (!rv) { e->req->reply_err(e->req, ERR_DB); return; } e->req->reply_mini(e->req, REP_OK); - } else if (e->operation == REQ_SET_ASYNC) { - db_set(db, e->key, e->ksize, e->val, e->vsize); - } else if (e->operation == REQ_GET) { unsigned char *val; size_t vsize = 64 * 1024; @@ -129,17 +134,17 @@ static void process_op(db_t *db, struct queue_entry *e) e->req->reply_long(e->req, REP_OK, val, vsize); free(val); - } else if (e->operation == REQ_DEL_SYNC) { + } else if (e->operation == REQ_DEL) { rv = db_del(db, e->key, e->ksize); + if (!(e->req->flags & FLAGS_SYNC)) + return; + if (rv == 0) { e->req->reply_mini(e->req, REP_NOTIN); return; } e->req->reply_mini(e->req, REP_OK); - } else if (e->operation == REQ_DEL_ASYNC) { - db_del(db, e->key, e->ksize); - } else if (e->operation == REQ_CAS) { unsigned char *dbval; size_t dbvsize = 64 * 1024; @@ -216,7 +221,9 @@ static void process_op(db_t *db, struct queue_entry *e) return; } - e->req->reply_mini(e->req, REP_OK); + intval = htonll(intval); + e->req->reply_long(e->req, REP_OK, + (unsigned char *) &intval, sizeof(intval)); free(dbval); diff --git a/nmdb/log.c b/nmdb/log.c index 89d107e..0a485b9 100644 --- a/nmdb/log.c +++ b/nmdb/log.c @@ -7,13 +7,14 @@ #include /* write() */ #include /* strcmp(), strerror() */ #include /* errno */ +#include /* time() and friends */ #include "log.h" #include "common.h" /* Logging file descriptor, -1 if logging is disabled */ -int logfd = -1; +static int logfd = -1; int log_init(void) @@ -25,15 +26,23 @@ int log_init(void) if (strcmp(settings.logfname, "-") == 0) { logfd = 1; - } else { - logfd = open(settings.logfname, O_WRONLY | O_APPEND | O_CREAT); - if (logfd < 0) - return 0; + return 1; } + logfd = open(settings.logfname, O_WRONLY | O_APPEND | O_CREAT, 0660); + if (logfd < 0) + return 0; + return 1; } +int log_reopen(void) +{ + /* Just call log_init(), it will do just fine as we don't need any + * special considerations for reopens */ + return log_init(); +} + void wlog(const char *fmt, ...) { int r, tr; diff --git a/nmdb/log.h b/nmdb/log.h index b187cc0..3e5672b 100644 --- a/nmdb/log.h +++ b/nmdb/log.h @@ -6,6 +6,7 @@ #define MAX_LOG_STR 512 int log_init(void); +int log_reopen(void); /* Normal logging, printf()-alike */ void wlog(const char *fmt, ...); diff --git a/nmdb/main.c b/nmdb/main.c index 5bb3cad..79d92b4 100644 --- a/nmdb/main.c +++ b/nmdb/main.c @@ -12,6 +12,7 @@ #include "common.h" #include "net-const.h" #include "log.h" +#include "stats.h" #define DEFDBNAME "database" @@ -26,7 +27,8 @@ struct queue *op_queue; static void help(void) { char h[] = \ "nmdb [options]\n" - " -d dbpath database path ('database', must be created with dpmgr)\n" + "\n" + " -d dbpath database path ('database' by default)\n" " -l lower TIPC lower port number (10)\n" " -L upper TIPC upper port number (= lower)\n" " -t port TCP listening port (26010)\n" @@ -36,12 +38,12 @@ static void help(void) { " -s port SCTP listening port (26010)\n" " -S addr SCTP listening address (all local addresses)\n" " -c nobj max. number of objects to be cached, in thousands (128)\n" - " -o fname log to the file 'fname'.\n" + " -o fname log to the given file (stdout).\n" " -f don't fork and stay in the foreground\n" " -p enable passive mode, for redundancy purposes (read docs.)\n" " -h show this help\n" "\n" - "Please report bugs to Alberto Bertogli (albertito@gmail.com)\n" + "Please report bugs to Alberto Bertogli (albertito@blitiri.com.ar)\n" "\n"; printf("%s", h); } @@ -67,7 +69,7 @@ static int load_settings(int argc, char **argv) settings.dbname = malloc(strlen(DEFDBNAME) + 1); strcpy(settings.dbname, DEFDBNAME); - while ((c = getopt(argc, argv, "d:l:L:t:T:u:U:c:o:fph?")) != -1) { + while ((c = getopt(argc, argv, "d:l:L:t:T:u:U:s:S:c:o:fph?")) != -1) { switch(c) { case 'd': free(settings.dbname); @@ -152,15 +154,6 @@ static int load_settings(int argc, char **argv) } -static void init_stats(void) -{ - stats.net_version_mismatch = 0; - stats.net_broken_req = 0; - stats.net_unk_req = 0; - return; -} - - int main(int argc, char **argv) { struct cache *cd; @@ -177,7 +170,7 @@ int main(int argc, char **argv) return 1; } - init_stats(); + stats_init(&stats); cd = cache_create(settings.numobjs, 0); if (cd == NULL) { @@ -213,6 +206,8 @@ int main(int argc, char **argv) setsid(); } + wlog("Starting nmdb\n"); + dbthread = db_loop_start(db); net_loop(); diff --git a/nmdb/net-const.h b/nmdb/net-const.h index b9e875a..290ab90 100644 --- a/nmdb/net-const.h +++ b/nmdb/net-const.h @@ -27,18 +27,16 @@ #define PROTO_VER 1 /* Network requests */ -#define REQ_CACHE_GET 0x101 -#define REQ_CACHE_SET 0x102 -#define REQ_CACHE_DEL 0x103 -#define REQ_GET 0x104 -#define REQ_SET_SYNC 0x105 -#define REQ_DEL_SYNC 0x106 -#define REQ_SET_ASYNC 0x107 -#define REQ_DEL_ASYNC 0x108 -#define REQ_CACHE_CAS 0x109 -#define REQ_CAS 0x110 -#define REQ_CACHE_INCR 0x111 -#define REQ_INCR 0x112 +#define REQ_GET 0x101 +#define REQ_SET 0x102 +#define REQ_DEL 0x103 +#define REQ_CAS 0x104 +#define REQ_INCR 0x105 +#define REQ_STATS 0x106 + +/* Possible request flags (which can be applied to the documented requests) */ +#define FLAGS_CACHE_ONLY 1 /* get, set, del, cas, incr */ +#define FLAGS_SYNC 2 /* set, del */ /* Network replies (different namespace from requests) */ #define REP_ERR 0x800 diff --git a/nmdb/net.c b/nmdb/net.c index fc01220..e1cc3b9 100644 --- a/nmdb/net.c +++ b/nmdb/net.c @@ -29,6 +29,12 @@ static void passive_to_active_sighandler(int fd, short event, void *arg) settings.passive = !settings.passive; } +static void logfd_reopen_sighandler(int fd, short event, void *arg) +{ + if (log_reopen()) + wlog("Log reopened\n"); +} + void net_loop(void) { int tipc_fd = -1; @@ -36,7 +42,7 @@ void net_loop(void) int udp_fd = -1; int sctp_fd = -1; struct event tipc_evt, tcp_evt, udp_evt, sctp_evt, - sigterm_evt, sigint_evt, sigusr2_evt; + sigterm_evt, sigint_evt, sigusr1_evt, sigusr2_evt; event_init(); @@ -95,6 +101,9 @@ void net_loop(void) signal_add(&sigterm_evt, NULL); signal_set(&sigint_evt, SIGINT, exit_sighandler, &sigint_evt); signal_add(&sigint_evt, NULL); + signal_set(&sigusr1_evt, SIGUSR1, logfd_reopen_sighandler, + &sigusr1_evt); + signal_add(&sigusr1_evt, NULL); signal_set(&sigusr2_evt, SIGUSR2, passive_to_active_sighandler, &sigusr2_evt); signal_add(&sigusr2_evt, NULL); @@ -112,6 +121,7 @@ void net_loop(void) signal_del(&sigterm_evt); signal_del(&sigint_evt); + signal_del(&sigusr1_evt); signal_del(&sigusr2_evt); tipc_close(tipc_fd); diff --git a/nmdb/netutils.c b/nmdb/netutils.c new file mode 100644 index 0000000..663b3c0 --- /dev/null +++ b/nmdb/netutils.c @@ -0,0 +1,53 @@ + +#include /* htonl() and friends */ +#include "netutils.h" + + +/* ntohll() and htonll() are not standard, so we define it using an UGLY trick + * because there is no standard way to check for endianness at runtime! */ +uint64_t ntohll(uint64_t x) +{ + static int endianness = 0; + + /* determine the endianness by checking how htonl() behaves; use -1 + * for little endian and 1 for big endian */ + if (endianness == 0) { + if (htonl(1) == 1) + endianness = 1; + else + endianness = -1; + } + + if (endianness == 1) { + /* big endian */ + return x; + } + + /* little endian */ + return ( ntohl( (x >> 32) & 0xFFFFFFFF ) | \ + ( (uint64_t) ntohl(x & 0xFFFFFFFF) ) << 32 ); +} + +uint64_t htonll(uint64_t x) +{ + static int endianness = 0; + + /* determine the endianness by checking how htonl() behaves; use -1 + * for little endian and 1 for big endian */ + if (endianness == 0) { + if (htonl(1) == 1) + endianness = 1; + else + endianness = -1; + } + + if (endianness == 1) { + /* big endian */ + return x; + } + + /* little endian */ + return ( htonl( (x >> 32) & 0xFFFFFFFF ) | \ + ( (uint64_t) htonl(x & 0xFFFFFFFF) ) << 32 ); +} + diff --git a/nmdb/netutils.h b/nmdb/netutils.h new file mode 100644 index 0000000..301617c --- /dev/null +++ b/nmdb/netutils.h @@ -0,0 +1,11 @@ + +#ifndef _NETUTILS_H +#define _NETUTILS_H + +#include + +uint64_t ntohll(uint64_t x); +uint64_t htonll(uint64_t x); + +#endif + diff --git a/nmdb/nmdb.1 b/nmdb/nmdb.1 index 49c5a77..b7fffb2 100644 --- a/nmdb/nmdb.1 +++ b/nmdb/nmdb.1 @@ -6,7 +6,7 @@ nmdb [-d dbpath] [-l lower] [-L upper] [-t tcpport] [-T tcpaddr] [-u udpport] [-U udpaddr] [-s sctpport] [-S sctpaddr] - [-c nobj] [-f] [-p] [-h] + [-c nobj] [-o fname] [-f] [-p] [-h] .SH DESCRIPTION @@ -23,7 +23,7 @@ library. Consult its manual page for programming information. Python bindings are also available. For additional documentation, go to the project's website at -.IR http://auriga.wearlab.de/~alb/nmdb . +.IR http://blitiri.com.ar/p/nmdb . .SH OPTIONS .TP @@ -64,6 +64,10 @@ that the size of the memory used by the cache layer depends on the size of the object exclusively. It defaults to 128, so the default cache size has space to hold 128 thousand objects. .TP +.B "-o fname" +Enable logging into the given file name. By default, output the debugging +information to stdout. +.TP .B "-f" Stay in the foreground, don't fork. Useful for debugging. The default is to fork. @@ -101,11 +105,10 @@ can't bind a port twice. .BR qdbm (3). .SH AUTHORS -Created by Alberto Bertogli (albertito@gmail.com). +Created by Alberto Bertogli (albertito@blitiri.com.ar). .SH CONTACT -To get in touch with developers and users, join the mailing list at -http://lists.auriga.wearlab.de/cgi-bin/mailman/listinfo/nmdb-devel or just -send an email to nmdb-devel@lists.auriga.wearlab.de. +If you want to report bugs, or have any questions or comments, just let me +know at albertito@blitiri.com.ar. diff --git a/nmdb/parse.c b/nmdb/parse.c index 368e67e..1df7e1d 100644 --- a/nmdb/parse.c +++ b/nmdb/parse.c @@ -4,25 +4,26 @@ #include /* memcpy() */ #include /* htonl() and friends */ - #include "parse.h" #include "req.h" #include "queue.h" #include "net-const.h" #include "common.h" +#include "netutils.h" -static void parse_get(struct req_info *req, int impact_db); -static void parse_set(struct req_info *req, int impact_db, int async); -static void parse_del(struct req_info *req, int impact_db, int async); -static void parse_cas(struct req_info *req, int impact_db); -static void parse_incr(struct req_info *req, int impact_db); +static void parse_get(const struct req_info *req); +static void parse_set(struct req_info *req); +static void parse_del(struct req_info *req); +static void parse_cas(struct req_info *req); +static void parse_incr(struct req_info *req); +static void parse_stats(struct req_info *req); /* Create a queue entry structure based on the parameters passed. Memory * allocated here will be free()'d in queue_entry_free(). It's not the * cleanest way, but the alternatives are even messier. */ -static struct queue_entry *make_queue_long_entry(struct req_info *req, +static struct queue_entry *make_queue_long_entry(const struct req_info *req, uint32_t operation, const unsigned char *key, size_t ksize, const unsigned char *val, size_t vsize, const unsigned char *newval, size_t nvsize) @@ -101,13 +102,46 @@ static struct queue_entry *make_queue_long_entry(struct req_info *req, return e; } -/* Like make_queue_long_entry() but with few parameters because most actions - * do not need newval. */ -static struct queue_entry *make_queue_entry(struct req_info *req, - uint32_t operation, const unsigned char *key, size_t ksize, + +/* Creates a new queue entry and puts it into the queue. Returns 1 if success, + * 0 if memory error. */ +static int put_in_queue_long(const struct req_info *req, + uint32_t operation, int sync, + const unsigned char *key, size_t ksize, + const unsigned char *val, size_t vsize, + const unsigned char *newval, size_t nvsize) +{ + struct queue_entry *e; + + e = make_queue_long_entry(req, operation, key, ksize, val, vsize, + newval, nvsize); + if (e == NULL) { + return 0; + } + queue_lock(op_queue); + queue_put(op_queue, e); + queue_unlock(op_queue); + + if (sync) { + /* Signal the DB thread it has work only if it's a + * synchronous operation, asynchronous don't mind + * waiting. It does have a measurable impact on + * performance (2083847usec vs 2804973usec for sets on + * "test2d 100000 10 10"). */ + queue_signal(op_queue); + } + + return 1; +} + +/* Like put_in_queue_long() but with few parameters because most actions do + * not need newval. */ +static int put_in_queue(const struct req_info *req, + uint32_t operation, int sync, + const unsigned char *key, size_t ksize, const unsigned char *val, size_t vsize) { - return make_queue_long_entry(req, operation, key, ksize, val, vsize, + return put_in_queue_long(req, operation, sync, key, ksize, val, vsize, NULL, 0); } @@ -119,13 +153,15 @@ static struct queue_entry *make_queue_entry(struct req_info *req, int parse_message(struct req_info *req, const unsigned char *buf, size_t len) { - uint32_t hdr, ver, id, cmd; + uint32_t hdr, ver, id; + uint16_t cmd, flags; const unsigned char *payload; size_t psize; /* The header is: * 4 bytes Version + ID - * 4 bytes Command + * 2 bytes Command + * 2 bytes Flags * Variable Payload */ @@ -137,7 +173,8 @@ int parse_message(struct req_info *req, id = hdr & 0x0FFFFFFF; req->id = id; - cmd = ntohl(* ((uint32_t *) buf + 1)); + cmd = ntohs(* ((uint16_t *) buf + 2)); + flags = ntohs(* ((uint16_t *) buf + 3)); if (ver != PROTO_VER) { stats.net_version_mismatch++; @@ -156,34 +193,23 @@ int parse_message(struct req_info *req, * to calculate it at send time. */ req->id = htonl(id); req->cmd = cmd; + req->flags = flags; req->payload = payload; req->psize = psize; - if (cmd == REQ_CACHE_GET) - parse_get(req, 0); - else if (cmd == REQ_CACHE_SET) - parse_set(req, 0, 0); - else if (cmd == REQ_CACHE_DEL) - parse_del(req, 0, 0); - else if (cmd == REQ_GET) - parse_get(req, 1); - else if (cmd == REQ_SET_SYNC) - parse_set(req, 1, 0); - else if (cmd == REQ_DEL_SYNC) - parse_del(req, 1, 0); - else if (cmd == REQ_SET_ASYNC) - parse_set(req, 1, 1); - else if (cmd == REQ_DEL_ASYNC) - parse_del(req, 1, 1); - else if (cmd == REQ_CACHE_CAS) - parse_cas(req, 0); - else if (cmd == REQ_CAS) - parse_cas(req, 1); - else if (cmd == REQ_CACHE_INCR) - parse_incr(req, 0); - else if (cmd == REQ_INCR) - parse_incr(req, 1); - else { + if (cmd == REQ_GET) { + parse_get(req); + } else if (cmd == REQ_SET) { + parse_set(req); + } else if (cmd == REQ_DEL) { + parse_del(req); + } else if (cmd == REQ_CAS) { + parse_cas(req); + } else if (cmd == REQ_INCR) { + parse_incr(req); + } else if (cmd == REQ_STATS) { + parse_stats(req); + } else { stats.net_unk_req++; req->reply_err(req, ERR_UNKREQ); } @@ -192,9 +218,23 @@ int parse_message(struct req_info *req, } -static void parse_get(struct req_info *req, int impact_db) +/* Small macros used to handle flags in the parse_*() functions */ +#define FILL_CACHE_FLAG(OP) \ + do { \ + cache_only = req->flags & FLAGS_CACHE_ONLY; \ + if (cache_only) stats.cache_##OP++; \ + else stats.db_##OP++; \ + } while (0) + +#define FILL_SYNC_FLAG() \ + do { \ + sync = req->flags & FLAGS_SYNC; \ + } while(0) + + +static void parse_get(const struct req_info *req) { - int hit; + int hit, cache_only, rv; const unsigned char *key; uint32_t ksize; unsigned char *val = NULL; @@ -208,35 +248,33 @@ static void parse_get(struct req_info *req, int impact_db) return; } + FILL_CACHE_FLAG(get); + key = req->payload + sizeof(uint32_t); hit = cache_get(cache_table, key, ksize, &val, &vsize); - if (!hit && !impact_db) { + if (cache_only && !hit) { + stats.cache_misses++; req->reply_mini(req, REP_CACHE_MISS); return; - } else if (!hit && impact_db) { - struct queue_entry *e; - e = make_queue_entry(req, REQ_GET, key, ksize, NULL, 0); - if (e == NULL) { + } else if (!cache_only && !hit) { + rv = put_in_queue(req, REQ_GET, 1, key, ksize, NULL, 0); + if (!rv) { req->reply_err(req, ERR_MEM); return; } - queue_lock(op_queue); - queue_put(op_queue, e); - queue_unlock(op_queue); - queue_signal(op_queue); return; } else { + stats.cache_hits++; req->reply_long(req, REP_CACHE_HIT, val, vsize); return; } } - -static void parse_set(struct req_info *req, int impact_db, int async) +static void parse_set(struct req_info *req) { - int rv; + int rv, cache_only, sync; const unsigned char *key, *val; uint32_t ksize, vsize; const int max = 65536; @@ -265,6 +303,9 @@ static void parse_set(struct req_info *req, int impact_db, int async) return; } + FILL_CACHE_FLAG(set); + FILL_SYNC_FLAG(); + key = req->payload + sizeof(uint32_t) * 2; val = key + ksize; @@ -274,33 +315,17 @@ static void parse_set(struct req_info *req, int impact_db, int async) return; } - if (impact_db) { - struct queue_entry *e; - uint32_t request; - - request = REQ_SET_SYNC; - if (async) - request = REQ_SET_ASYNC; - - e = make_queue_entry(req, request, key, ksize, val, vsize); - if (e == NULL) { + if (!cache_only) { + rv = put_in_queue(req, REQ_SET, sync, key, ksize, val, vsize); + if (!rv) { req->reply_err(req, ERR_MEM); return; } - queue_lock(op_queue); - queue_put(op_queue, e); - queue_unlock(op_queue); - if (async) { + if (!sync) { req->reply_mini(req, REP_OK); - } else { - /* Signal the DB thread it has work only if it's a - * synchronous operation, asynchronous don't mind - * waiting. It does have a measurable impact on - * performance (2083847usec vs 2804973usec for sets on - * "test2d 100000 10 10"). */ - queue_signal(op_queue); } + return; } else { req->reply_mini(req, REP_OK); @@ -310,9 +335,9 @@ static void parse_set(struct req_info *req, int impact_db, int async) } -static void parse_del(struct req_info *req, int impact_db, int async) +static void parse_del(struct req_info *req) { - int hit; + int hit, cache_only, sync, rv; const unsigned char *key; uint32_t ksize; @@ -324,36 +349,26 @@ static void parse_del(struct req_info *req, int impact_db, int async) return; } + FILL_CACHE_FLAG(del); + FILL_SYNC_FLAG(); + key = req->payload + sizeof(uint32_t); hit = cache_del(cache_table, key, ksize); - if (!impact_db && hit) { + if (cache_only && hit) { req->reply_mini(req, REP_OK); - } else if (!impact_db && !hit) { + } else if (cache_only && !hit) { req->reply_mini(req, REP_NOTIN); - } else if (impact_db) { - struct queue_entry *e; - uint32_t request; - - request = REQ_DEL_SYNC; - if (async) - request = REQ_DEL_ASYNC; - - e = make_queue_entry(req, request, key, ksize, NULL, 0); - if (e == NULL) { + } else if (!cache_only) { + rv = put_in_queue(req, REQ_DEL, sync, key, ksize, NULL, 0); + if (!rv) { req->reply_err(req, ERR_MEM); return; } - queue_lock(op_queue); - queue_put(op_queue, e); - queue_unlock(op_queue); - if (async) { + if (!sync) { req->reply_mini(req, REP_OK); - } else { - /* See comment on parse_set(). */ - queue_signal(op_queue); } return; @@ -362,9 +377,9 @@ static void parse_del(struct req_info *req, int impact_db, int async) return; } -static void parse_cas(struct req_info *req, int impact_db) +static void parse_cas(struct req_info *req) { - int rv; + int rv, cache_only; const unsigned char *key, *oldval, *newval; uint32_t ksize, ovsize, nvsize; const int max = 65536; @@ -399,6 +414,8 @@ static void parse_cas(struct req_info *req, int impact_db) return; } + FILL_CACHE_FLAG(cas); + key = req->payload + sizeof(uint32_t) * 3; oldval = key + ksize; newval = oldval + ovsize; @@ -412,7 +429,7 @@ static void parse_cas(struct req_info *req, int impact_db) return; } - if (!impact_db) { + if (cache_only) { if (rv == -1) { req->reply_mini(req, REP_NOTIN); return; @@ -421,59 +438,25 @@ static void parse_cas(struct req_info *req, int impact_db) return; } } else { - /* impact_db = 1 and the key is either not in the cache, or + /* !cache_only and the key is either not in the cache, or * cache_cas() was successful. We now need to queue the CAS in * the database. */ - struct queue_entry *e; - - e = make_queue_long_entry(req, REQ_CAS, key, ksize, + rv = put_in_queue_long(req, REQ_CAS, 1, key, ksize, oldval, ovsize, newval, nvsize); - if (e == NULL) { + if (!rv) { req->reply_err(req, ERR_MEM); return; } - - queue_lock(op_queue); - queue_put(op_queue, e); - queue_unlock(op_queue); - queue_signal(op_queue); } return; } - -/* ntohll() is not standard, so we define it using an UGLY trick because there - * is no standard way to check for endianness at runtime! */ -static uint64_t ntohll(uint64_t x) -{ - static int endianness = 0; - - /* determine the endianness by checking how htonl() behaves; use -1 - * for little endian and 1 for big endian */ - if (endianness == 0) { - if (htonl(1) == 1) - endianness = 1; - else - endianness = -1; - } - - if (endianness == 1) { - /* big endian */ - return x; - } - - /* little endian */ - return ( ntohl( (x >> 32) & 0xFFFFFFFF ) | \ - ( (uint64_t) ntohl(x & 0xFFFFFFFF) ) << 32 ); -} - - -static void parse_incr(struct req_info *req, int impact_db) +static void parse_incr(struct req_info *req) { - int cres; + int cres, cache_only, rv; const unsigned char *key; uint32_t ksize; - int64_t increment; + int64_t increment, newval; const int max = 65536; /* Request format: @@ -494,10 +477,12 @@ static void parse_incr(struct req_info *req, int impact_db) return; } + FILL_CACHE_FLAG(incr); + key = req->payload + sizeof(uint32_t); increment = ntohll( * (int64_t *) (key + ksize) ); - cres = cache_incr(cache_table, key, ksize, increment); + cres = cache_incr(cache_table, key, ksize, increment, &newval); if (cres == -3) { req->reply_err(req, ERR_MEM); return; @@ -507,32 +492,79 @@ static void parse_incr(struct req_info *req, int impact_db) return; } - if (impact_db) { - struct queue_entry *e; - + if (!cache_only) { /* at this point, the cache_incr() was either successful or a * miss, but we don't really care */ - - e = make_queue_entry(req, REQ_INCR, key, ksize, + rv = put_in_queue(req, REQ_INCR, 1, key, ksize, (unsigned char *) &increment, sizeof(increment)); - if (e == NULL) { + if (!rv) { req->reply_err(req, ERR_MEM); return; } - queue_lock(op_queue); - queue_put(op_queue, e); - queue_unlock(op_queue); - - queue_signal(op_queue); } else { - if (cres == -1) + if (cres == -1) { req->reply_mini(req, REP_NOTIN); - else - req->reply_mini(req, REP_OK); + } else { + newval = htonll(newval); + req->reply_long(req, REP_OK, + (unsigned char *) &newval, + sizeof(newval)); + } } return; } +static void parse_stats(struct req_info *req) +{ + int i; + uint64_t response[STATS_REPLY_SIZE]; + + /* The packet is just the request, there's no payload. We need to + * reply with the stats structure. + * The response structure is just several uint64_t packed together, + * each one corresponds to a single value of the stats structure. */ + + /* We define a macro to do the assignment easily; it's not nice, but + * it's more portable than using a packed struct */ + i = 0; + #define fcpy(field) \ + do { response[i] = htonll(stats.field); i++; } while(0) + + + fcpy(cache_get); + fcpy(cache_set); + fcpy(cache_del); + fcpy(cache_cas); + fcpy(cache_incr); + + fcpy(db_get); + fcpy(db_set); + fcpy(db_del); + fcpy(db_cas); + fcpy(db_incr); + + fcpy(cache_hits); + fcpy(cache_misses); + + fcpy(db_hits); + fcpy(db_misses); + + fcpy(msg_tipc); + fcpy(msg_tcp); + fcpy(msg_udp); + fcpy(msg_sctp); + + fcpy(net_version_mismatch); + fcpy(net_broken_req); + fcpy(net_unk_req); + + req->reply_long(req, REP_OK, (unsigned char *) response, + sizeof(response)); + + return; +} + + diff --git a/nmdb/queue.c b/nmdb/queue.c index eb56b09..0629f82 100644 --- a/nmdb/queue.c +++ b/nmdb/queue.c @@ -32,11 +32,18 @@ void queue_free(struct queue *q) { struct queue_entry *e; + /* We know when we're called there is no other possible queue user; + * however, we don't have any sane way to tell sparse about this, so + * fake the acquisition of the lock to comply with the operations + * performed inside. Obviously, it would be completely safe to do the + * queue_lock()/unlock(), but it'd be misleading to the reader */ + __acquire(q->lock); e = queue_get(q); while (e != NULL) { queue_entry_free(e); e = queue_get(q); } + __release(q->lock); pthread_mutex_destroy(&(q->lock)); diff --git a/nmdb/queue.h b/nmdb/queue.h index f50abe5..17a4ec0 100644 --- a/nmdb/queue.h +++ b/nmdb/queue.h @@ -5,6 +5,7 @@ #include /* for mutexes */ #include /* for uint32_t */ #include "req.h" /* for req_info */ +#include "sparse.h" struct queue { pthread_mutex_t lock; @@ -38,14 +39,20 @@ void queue_free(struct queue *q); struct queue_entry *queue_entry_create(); void queue_entry_free(struct queue_entry *e); -void queue_lock(struct queue *q); -void queue_unlock(struct queue *q); +void queue_lock(struct queue *q) + __acquires(q->lock); +void queue_unlock(struct queue *q) + __releases(q->lock); void queue_signal(struct queue *q); -int queue_timedwait(struct queue *q, struct timespec *ts); - -void queue_put(struct queue *q, struct queue_entry *e); -struct queue_entry *queue_get(struct queue *q); -int queue_isempty(struct queue *q); +int queue_timedwait(struct queue *q, struct timespec *ts) + __with_lock_acquired(q->lock); + +void queue_put(struct queue *q, struct queue_entry *e) + __with_lock_acquired(q->lock); +struct queue_entry *queue_get(struct queue *q) + __with_lock_acquired(q->lock); +int queue_isempty(struct queue *q) + __with_lock_acquired(q->lock); #endif diff --git a/nmdb/req.h b/nmdb/req.h index 4637307..3660d86 100644 --- a/nmdb/req.h +++ b/nmdb/req.h @@ -24,14 +24,15 @@ struct req_info { /* operation information */ uint32_t id; - uint32_t cmd; + uint16_t cmd; + uint16_t flags; const unsigned char *payload; size_t psize; /* operations */ - void (*reply_mini)(struct req_info *req, uint32_t reply); - void (*reply_err)(struct req_info *req, uint32_t reply); - void (*reply_long)(struct req_info *req, uint32_t reply, + void (*reply_mini)(const struct req_info *req, uint32_t reply); + void (*reply_err)(const struct req_info *req, uint32_t reply); + void (*reply_long)(const struct req_info *req, uint32_t reply, unsigned char *val, size_t vsize); }; diff --git a/nmdb/sctp.c b/nmdb/sctp.c index 42fdb4c..a0e3adf 100644 --- a/nmdb/sctp.c +++ b/nmdb/sctp.c @@ -17,12 +17,6 @@ #include "log.h" -static void sctp_reply_mini(struct req_info *req, uint32_t reply); -static void sctp_reply_err(struct req_info *req, uint32_t reply); -static void sctp_reply_long(struct req_info *req, uint32_t reply, - unsigned char *val, size_t vsize); - - /* * Miscelaneous helper functions */ @@ -69,7 +63,7 @@ static int rep_send(const struct req_info *req, const unsigned char *buf, /* Send small replies, consisting in only a value. */ -static void sctp_reply_mini(struct req_info *req, uint32_t reply) +static void sctp_reply_mini(const struct req_info *req, uint32_t reply) { /* We use a mini buffer to speedup the small replies, to avoid the * malloc() overhead. */ @@ -89,12 +83,12 @@ static void sctp_reply_mini(struct req_info *req, uint32_t reply) /* The sctp_reply_* functions are used by the db code to send the network * replies. */ -void sctp_reply_err(struct req_info *req, uint32_t reply) +static void sctp_reply_err(const struct req_info *req, uint32_t reply) { rep_send_error(req, reply); } -void sctp_reply_long(struct req_info *req, uint32_t reply, +static void sctp_reply_long(const struct req_info *req, uint32_t reply, unsigned char *val, size_t vsize) { if (val == NULL) { @@ -212,11 +206,13 @@ void sctp_recv(int fd, short event, void *arg) goto exit; } - if (rv < 2) { + if (rv < 8) { stats.net_broken_req++; goto exit; } + stats.msg_sctp++; + req.fd = fd; req.type = REQTYPE_SCTP; req.clisa = (struct sockaddr *) &clisa; diff --git a/nmdb/sparse.h b/nmdb/sparse.h new file mode 100644 index 0000000..20adafc --- /dev/null +++ b/nmdb/sparse.h @@ -0,0 +1,21 @@ + +/* Useful defines to use with sparse */ + +#ifndef _SPARSE_H +#define _SPARSE_H + +#ifdef __CHECKER__ +# define __acquires(x) __attribute__((exact_context(x,0,1))) +# define __releases(x) __attribute__((exact_context(x,1,0))) +# define __with_lock_acquired(x) __attribute__((exact_context(x,1,1))) +# define __acquire(x) __context__(x,1,0) +# define __release(x) __context__(x,-1,1) +#else +# define __acquires(x) +# define __releases(x) +# define __with_lock_acquired(x) +# define __acquire(x) (void)0 +# define __release(x) (void)0 +#endif + +#endif diff --git a/nmdb/stats.c b/nmdb/stats.c new file mode 100644 index 0000000..8cd85ee --- /dev/null +++ b/nmdb/stats.c @@ -0,0 +1,31 @@ + +#include "stats.h" + +void stats_init(struct stats *s) +{ + s->cache_get = 0; + s->cache_set = 0; + s->cache_del = 0; + s->cache_cas = 0; + s->cache_incr = 0; + + s->db_get = 0; + s->db_set = 0; + s->db_del = 0; + s->db_cas = 0; + s->db_incr = 0; + + s->cache_misses = 0; + s->cache_hits = 0; + + s->msg_tipc = 0; + s->msg_tcp = 0; + s->msg_udp = 0; + s->msg_sctp = 0; + + s->net_version_mismatch = 0; + s->net_broken_req = 0; + s->net_unk_req = 0; +} + + diff --git a/nmdb/stats.h b/nmdb/stats.h new file mode 100644 index 0000000..8d85ff6 --- /dev/null +++ b/nmdb/stats.h @@ -0,0 +1,41 @@ + +#ifndef _STATS_H +#define _STATS_H + +/* Statistics structure */ +struct stats { + unsigned long cache_get; + unsigned long cache_set; + unsigned long cache_del; + unsigned long cache_cas; + unsigned long cache_incr; /* 5 */ + + unsigned long db_get; + unsigned long db_set; + unsigned long db_del; + unsigned long db_cas; + unsigned long db_incr; /* 10 */ + + unsigned long cache_hits; + unsigned long cache_misses; + + unsigned long db_hits; + unsigned long db_misses; + + + unsigned long msg_tipc; /* 15 */ + unsigned long msg_tcp; + unsigned long msg_udp; + unsigned long msg_sctp; + + unsigned long net_version_mismatch; + unsigned long net_broken_req; /* 20 */ + unsigned long net_unk_req; +}; + +#define STATS_REPLY_SIZE 21 + +void stats_init(struct stats *s); + +#endif + diff --git a/nmdb/tcp.c b/nmdb/tcp.c index dc98c26..b435756 100644 --- a/nmdb/tcp.c +++ b/nmdb/tcp.c @@ -44,9 +44,9 @@ static void tcp_recv(int fd, short event, void *arg); static void process_buf(struct tcp_socket *tcpsock, unsigned char *buf, size_t len); -static void tcp_reply_mini(struct req_info *req, uint32_t reply); -static void tcp_reply_err(struct req_info *req, uint32_t reply); -static void tcp_reply_long(struct req_info *req, uint32_t reply, +static void tcp_reply_mini(const struct req_info *req, uint32_t reply); +static void tcp_reply_err(const struct req_info *req, uint32_t reply); +static void tcp_reply_long(const struct req_info *req, uint32_t reply, unsigned char *val, size_t vsize); @@ -144,7 +144,7 @@ static int rep_send(const struct req_info *req, const unsigned char *buf, /* Send small replies, consisting in only a value. */ -void tcp_reply_mini(struct req_info *req, uint32_t reply) +static void tcp_reply_mini(const struct req_info *req, uint32_t reply) { /* We use a mini buffer to speedup the small replies, to avoid the * malloc() overhead. */ @@ -164,12 +164,12 @@ void tcp_reply_mini(struct req_info *req, uint32_t reply) } -void tcp_reply_err(struct req_info *req, uint32_t reply) +static void tcp_reply_err(const struct req_info *req, uint32_t reply) { rep_send_error(req, reply); } -void tcp_reply_long(struct req_info *req, uint32_t reply, +static void tcp_reply_long(const struct req_info *req, uint32_t reply, unsigned char *val, size_t vsize) { if (val == NULL) { @@ -379,7 +379,7 @@ static void process_buf(struct tcp_socket *tcpsock, if (len >= 4) { totaltoget = * (uint32_t *) buf; totaltoget = ntohl(totaltoget); - if (totaltoget > (64 * 1024) || totaltoget <= 12) { + if (totaltoget > (64 * 1024) || totaltoget <= 8) { /* Message too big or too small, close the connection. */ goto error_exit; } @@ -421,6 +421,7 @@ static void process_buf(struct tcp_socket *tcpsock, } /* The buffer is complete, parse it as usual. */ + stats.msg_tcp++; if (parse_message(&(tcpsock->req), buf + 4, len - 4)) { goto exit; } else { diff --git a/nmdb/tipc.c b/nmdb/tipc.c index 658a056..2a21351 100644 --- a/nmdb/tipc.c +++ b/nmdb/tipc.c @@ -16,12 +16,6 @@ #include "log.h" -static void tipc_reply_mini(struct req_info *req, uint32_t reply); -static void tipc_reply_err(struct req_info *req, uint32_t reply); -static void tipc_reply_long(struct req_info *req, uint32_t reply, - unsigned char *val, size_t vsize); - - /* * Miscelaneous helper functions */ @@ -68,7 +62,7 @@ static int rep_send(const struct req_info *req, const unsigned char *buf, /* Send small replies, consisting in only a value. */ -static void tipc_reply_mini(struct req_info *req, uint32_t reply) +static void tipc_reply_mini(const struct req_info *req, uint32_t reply) { /* We use a mini buffer to speedup the small replies, to avoid the * malloc() overhead. */ @@ -88,12 +82,12 @@ static void tipc_reply_mini(struct req_info *req, uint32_t reply) /* The tipc_reply_* functions are used by the db code to send the network * replies. */ -void tipc_reply_err(struct req_info *req, uint32_t reply) +static void tipc_reply_err(const struct req_info *req, uint32_t reply) { rep_send_error(req, reply); } -void tipc_reply_long(struct req_info *req, uint32_t reply, +static void tipc_reply_long(const struct req_info *req, uint32_t reply, unsigned char *val, size_t vsize) { if (val == NULL) { @@ -196,11 +190,13 @@ void tipc_recv(int fd, short event, void *arg) goto exit; } - if (rv < 2) { + if (rv < 8) { stats.net_broken_req++; goto exit; } + stats.msg_tipc++; + req.fd = fd; req.type = REQTYPE_TIPC; req.clisa = (struct sockaddr *) &clisa; diff --git a/nmdb/udp.c b/nmdb/udp.c index c9c51ea..62fcbcb 100644 --- a/nmdb/udp.c +++ b/nmdb/udp.c @@ -17,12 +17,6 @@ #include "log.h" -static void udp_reply_mini(struct req_info *req, uint32_t reply); -static void udp_reply_err(struct req_info *req, uint32_t reply); -static void udp_reply_long(struct req_info *req, uint32_t reply, - unsigned char *val, size_t vsize); - - /* * Miscelaneous helper functions */ @@ -69,7 +63,7 @@ static int rep_send(const struct req_info *req, const unsigned char *buf, /* Send small replies, consisting in only a value. */ -static void udp_reply_mini(struct req_info *req, uint32_t reply) +static void udp_reply_mini(const struct req_info *req, uint32_t reply) { /* We use a mini buffer to speedup the small replies, to avoid the * malloc() overhead. */ @@ -89,12 +83,12 @@ static void udp_reply_mini(struct req_info *req, uint32_t reply) /* The udp_reply_* functions are used by the db code to send the network * replies. */ -void udp_reply_err(struct req_info *req, uint32_t reply) +static void udp_reply_err(const struct req_info *req, uint32_t reply) { rep_send_error(req, reply); } -void udp_reply_long(struct req_info *req, uint32_t reply, +static void udp_reply_long(const struct req_info *req, uint32_t reply, unsigned char *val, size_t vsize) { if (val == NULL) { @@ -196,11 +190,13 @@ void udp_recv(int fd, short event, void *arg) goto exit; } - if (rv < 2) { + if (rv < 8) { stats.net_broken_req++; goto exit; } + stats.msg_udp++; + req.fd = fd; req.type = REQTYPE_UDP; req.clisa = (struct sockaddr *) &clisa; diff --git a/tests/c/incr.c b/tests/c/incr.c index a0604d1..f0036a3 100644 --- a/tests/c/incr.c +++ b/tests/c/incr.c @@ -18,6 +18,7 @@ int main(int argc, char **argv) char *initval = "0"; size_t ksize; long long int increment; + int64_t newval; nmdb_t *db; if (argc != 3) { @@ -49,16 +50,17 @@ int main(int argc, char **argv) timer_start(); for (i = 0; i < times; i++) { - r = NINCR(db, (unsigned char *) key, ksize, increment); + r = NINCR(db, (unsigned char *) key, ksize, increment, + &newval); if (r != 2) { - printf("result: %d\n", r); + printf("result: %d %lld\n", r, (long long) newval); perror("Incr"); return 1; } } s_elapsed = timer_stop(); - printf("%lu\n", s_elapsed); + printf("%lu %lld\n", s_elapsed, (long long) newval); nmdb_free(db); diff --git a/tests/c/make.sh b/tests/c/make.sh index 3b07957..f8dda75 100755 --- a/tests/c/make.sh +++ b/tests/c/make.sh @@ -43,7 +43,10 @@ for p in TIPC TCP UDP SCTP MULT; do if [ "$CLEAN" == 1 ]; then rm -f $t-$OP else - cc -lnmdb $ALLCF $TF -o $t-$OP $t.c + # build only if src is newer than the binary + if [ "$t.c" -nt "$t-$OP" ]; then + cc -lnmdb $ALLCF $TF -o $t-$OP $t.c + fi fi done done diff --git a/tests/c/timer.h b/tests/c/timer.h index a0a8191..ef83bc0 100644 --- a/tests/c/timer.h +++ b/tests/c/timer.h @@ -1,7 +1,7 @@ /* * A simple timer for measuring delays. - * Alberto Bertogli (albertito@gmail.com) - September/2006 + * Alberto Bertogli (albertito@blitiri.com.ar) - September/2006 * * Use it like this: * unsigned long elapsed; @@ -10,7 +10,7 @@ * ... [code] ... * elapsed = timer_stop(); * ... - * printf("Time elapsed: %ul", elapsed); + * printf("Time elapsed: %lu", elapsed); * * Nested timers are not supported. The result is in usecs. * diff --git a/tests/perf/ag.sh b/tests/perf/ag.sh new file mode 100755 index 0000000..0ad9320 --- /dev/null +++ b/tests/perf/ag.sh @@ -0,0 +1,126 @@ +#!/bin/bash + +# Automated nmdb performance test - aggregated graphics +# Should be ran from somewhere inside the git repository + +TIMES=5000 # Times parameter used for the tests + +set -e + + +# aggregate the results prepending the version, but we need a table because +# gnuplot doesn't like strings as values +cd `git rev-parse --show-cdup` +cd tests/perf/out +mkdir -p ../ag-data +rm -f ../ag-data/* + +declare -a COMMITS +N=0 + +for i in *; do + COMMITS[$N]=$i + for f in 2-tipc-cache.out 3-tipc-cache.out; do + while read L; do + if [ "$L" != "" ]; then + echo $N $L >> ../ag-data/$f + else + # preserve data set delimiters for gnuplot + echo >> ../ag-data/$f + fi + done < $i/$f + done + N=$[ $N + 1 ] +done + + +# write the gnuplot script +cd `git rev-parse --show-cdup` +cd tests/perf +mkdir -p graph +DST=`pwd`/graph + +# 2-tipc-cache (3d) +cat > graph/ag-tmp.sh <> graph/ag-tmp.sh + N=$[ $N + 1 ] +done + +cat >> graph/ag-tmp.sh <> graph/ag-tmp.sh <> graph/ag-tmp.sh <> graph/ag-tmp.sh <> graph/ag-tmp.sh <> graph/ag-tmp.sh <> graph/ag-tmp.sh < /dev/null + + diff --git a/tests/perf/perf.sh b/tests/perf/perf.sh new file mode 100755 index 0000000..4824abf --- /dev/null +++ b/tests/perf/perf.sh @@ -0,0 +1,211 @@ +#!/bin/bash + +# Automated nmdb performance test +# Should be ran from somewhere inside the git repository + +COUNT=2 # How many times we should repeat the test +TIMES=5000 # Times parameter for the tests + + +set -e + +# set up the temporary directory, and the cleanup routine +TMP=`mktemp -d -t nmdb-perf-test.XXXXXXXXXX` || \ + ( echo "error: Can't create tmp directory"; exit 1 ) +function atexit() { + server_stop + rm -r "$TMP" +} +trap atexit EXIT + + +NMDBPID= +function server_start() { + # launch the server + techo "start nmdb" + rm -f $TMP/database $OUTDIR/nmdb.log + ./nmdb/nmdb -f -d $TMP/database -o $OUTDIR/nmdb.log & + NMDBPID=$! + + # wait a bit so the clients don't try to talk too early + sleep 0.5 +} + +function server_stop() { + if [ "$NMDBPID" != "" ]; then + techo "stop nmdb" + kill $NMDBPID || true + wait $NMDBPID + fi + NMDBPID= +} + + + +function techo() { + echo `date "+%H:%M:%S.%N"` "$@" +} + +# helper function used to run a single test that takes the same parameters: +# times, ksize, vsize; must be called from inside test_current +function run_test() { + t1=$1 + lcount=$2 + ltimes=$3 + + for t2 in mult tipc tcp udp sctp; do + for t3 in cache; do + t="$t1-$t2-$t3" + techo " " $t $@ + rm -f $OUTDIR/$t.out + for i in `seq 1 $lcount`; do + # restart server + server_start + echo -n " " + for s in 4 32 128 512 1024 \ + `seq 2048 2048 30000`; do + echo $s `tests/c/$t $ltimes $s $s` \ + >> $OUTDIR/$t.out + sleep 0.3 + echo -n " $s" + done + # gnuplot splits the data sets with this empty + # line, otherwise we get weird graphics when + # $COUNT > 2 + echo >> $OUTDIR/$t.out + echo + server_stop + done + done + done +} + + +# tests the current revision +function test_current() { + # go to the repo root + cd "`git rev-parse --show-cdup`" + + # create the output directory + OUTDIR="tests/perf/out/`git describe --always`$SUFFIX" + techo mkdir $OUTDIR + mkdir -p $OUTDIR + + # record the environment + ( + echo date `date --rfc-3339=seconds` + echo testing `git describe --always` + #techo using perf ${OLDCOMMIT:-`git describe --always`} + echo + uname -a + gcc --version | grep gcc + echo + cat /proc/cpuinfo | grep 'model name' + cat /proc/meminfo | grep MemTotal + ) > $OUTDIR/environment + + # build + techo build + #make BACKEND=bdb > $OUTDIR/make.out + # no porque asumimos en el sistema + #make > $OUTDIR/make.out + cd tests/c + ./make.sh build > $TMP/tests-make.out + cd - > /dev/null + + # record the vmstat output while the test runs + techo vmstat + vmstat 1 > $OUTDIR/vmstat.out & + VMSTATPID=$! + + # wait a bit for the server to come up + sleep 1 + + # run the tests + techo tests + + for t1 in 2 3; do + run_test $t1 $COUNT $TIMES + done + + # kill everything; the "|| true" is needed to prevent the script from + # dying if the process are already dead + techo "kill vmstat ($VMSTATPID)" + kill -9 $VMSTATPID || true + + # wait for them to die + wait $NMDBPID $VMSTATPID > /dev/null 2> /dev/null || true + + techo done +} + + + +function write_current_graph_scripts() { + SUFFIX="$1" + V="`git describe --always`$SUFFIX" + + # go to the results dir + cd "`git rev-parse --show-cdup`" + + mkdir -p tests/perf/graph + DST=`pwd`/tests/perf/graph + + gnuplot < /dev/null + +} + + +if [ "$2" == "" ]; then + SUFFIX="" +else + SUFFIX="-$2" +fi + +case $1 in + ""|"help"|"-h"|"--help") + echo "Usage: perf.sh test|graph [suffix]" + echo + echo "Run from somewhere inside the git repository; output will" + echo "be put in tests/pref/out/[-suffix]." + exit 1 + ;; + "test") + test_current $SUFFIX + ;; + "graph") + write_current_graph_scripts $SUFFIX + ;; + *) + echo "Unknown operation" + exit 1 +esac + diff --git a/tests/python/random1-cache.py b/tests/python/random1-cache.py index 3dca13c..33ca332 100755 --- a/tests/python/random1-cache.py +++ b/tests/python/random1-cache.py @@ -27,7 +27,10 @@ def checked(f): try: return f(k, *args, **kwargs) except: - print history[k] + if k in history: + print history[k] + else: + print 'No history for key', k raise newf.__name__ = f.__name__ return newf @@ -80,16 +83,20 @@ def getrand(): if __name__ == '__main__': - if len(sys.argv) != 2: - print 'Use: random1-cache.py number_of_keys' + if len(sys.argv) < 2: + print 'Use: random1-cache.py number_of_keys [key_prefix]' sys.exit(1) nkeys = int(sys.argv[1]) + if len(sys.argv) > 2: + key_prefix = sys.argv[2] + else: + key_prefix = '' # fill all the keys print 'populate' for i in xrange(nkeys): - set(getrand(), getrand()) + set(key_prefix + str(getrand()), getrand()) print 'missing', find_missing() diff --git a/tests/python/random1.py b/tests/python/random1.py index 0c53d28..172b4b5 100755 --- a/tests/python/random1.py +++ b/tests/python/random1.py @@ -27,7 +27,10 @@ def checked(f): try: return f(k, *args, **kwargs) except: - print history[k] + if k in history: + print history[k] + else: + print 'No history for key', k raise newf.__name__ = f.__name__ return newf @@ -99,16 +102,20 @@ def getrand(): if __name__ == '__main__': - if len(sys.argv) != 2: - print 'Use: random1.py number_of_keys' + if len(sys.argv) < 2: + print 'Use: random1.py number_of_keys [key_prefix]' sys.exit(1) nkeys = int(sys.argv[1]) + if len(sys.argv) > 2: + key_prefix = sys.argv[2] + else: + key_prefix = '' # fill all the keys print 'populate' for i in xrange(nkeys): - set(getrand(), getrand()) + set(key_prefix + str(getrand()), getrand()) lkeys = ldb.keys() diff --git a/tests/python3/README b/tests/python3/README new file mode 100644 index 0000000..9f857b7 --- /dev/null +++ b/tests/python3/README @@ -0,0 +1,4 @@ + +These tests are identical to the ones in tests/python, except they've been +modified to work under Python 3 (and obviously use the Python 3 bindings). + diff --git a/tests/python/random1-cache.py b/tests/python3/random1-cache.py similarity index 72% copy from tests/python/random1-cache.py copy to tests/python3/random1-cache.py index 3dca13c..71e8b9f 100755 --- a/tests/python/random1-cache.py +++ b/tests/python3/random1-cache.py @@ -27,7 +27,10 @@ def checked(f): try: return f(k, *args, **kwargs) except: - print history[k] + if k in history: + print(history[k]) + else: + print('No history for key', k) raise newf.__name__ = f.__name__ return newf @@ -53,7 +56,7 @@ def get(k): l = ldb[k] if l != n: - raise Mismatch, (n, l) + raise Mismatch((n, l)) history[k].append((get, k)) return True @@ -68,7 +71,7 @@ def delete(k): def find_missing(): misses = 0 - for k in ldb.keys(): + for k in list(ldb.keys()): if not get(k): misses += 1 return misses @@ -80,25 +83,29 @@ def getrand(): if __name__ == '__main__': - if len(sys.argv) != 2: - print 'Use: random1-cache.py number_of_keys' + if len(sys.argv) < 2: + print('Use: random1-cache.py number_of_keys [key_prefix]') sys.exit(1) nkeys = int(sys.argv[1]) + if len(sys.argv) > 2: + key_prefix = sys.argv[2] + else: + key_prefix = '' # fill all the keys - print 'populate' - for i in xrange(nkeys): - set(getrand(), getrand()) + print('populate') + for i in range(nkeys): + set(key_prefix + str(getrand()), getrand()) - print 'missing', find_missing() + print('missing', find_missing()) - lkeys = ldb.keys() + lkeys = list(ldb.keys()) # operate on them a bit - print 'random operations' + print('random operations') operations = ('set', 'get', 'delete') - for i in xrange(nkeys / 2): + for i in range(nkeys // 2): op = choice(operations) k = choice(lkeys) if op == 'set': @@ -109,13 +116,13 @@ if __name__ == '__main__': delete(k) lkeys.remove(k) - print 'missing', find_missing() + print('missing', find_missing()) - print 'delete' + print('delete') for k in lkeys: delete(k) - print 'missing', find_missing() + print('missing', find_missing()) sys.exit(0) diff --git a/tests/python/random1.py b/tests/python3/random1.py similarity index 72% copy from tests/python/random1.py copy to tests/python3/random1.py index 0c53d28..ae18bc6 100755 --- a/tests/python/random1.py +++ b/tests/python3/random1.py @@ -27,7 +27,10 @@ def checked(f): try: return f(k, *args, **kwargs) except: - print history[k] + if k in history: + print(history[k]) + else: + print('No history for key', k) raise newf.__name__ = f.__name__ return newf @@ -47,7 +50,7 @@ def get(k): n = ndb[k] l = ldb[k] if l != n: - raise Mismatch, (n, l) + raise Mismatch((n, l)) history[k].append((get, k)) return n @@ -62,7 +65,7 @@ def cas(k, ov, nv): prel = ldb[k] pren = ndb[k] n = ndb.cas(k, ov, nv) - if not ldb.has_key(k): + if k not in ldb: l = 0 elif ldb[k] == ov: ldb[k] = nv @@ -70,10 +73,10 @@ def cas(k, ov, nv): else: l = 1 if n != l: - print k, ldb[k], ndb[k] - print prel, pren - print history[k] - raise Mismatch, (n, l) + print(k, ldb[k], ndb[k]) + print(prel, pren) + print(history[k]) + raise Mismatch((n, l)) history[k].append((cas, k, ov, nv)) return n @@ -84,12 +87,12 @@ def check(): n = ndb[k] l = ldb[k] except: - print history[k] - raise Mismatch, (n, l) + print(history[k]) + raise Mismatch((n, l)) if n != n: - print history[k] - raise Mismatch, (n, l) + print(history[k]) + raise Mismatch((n, l)) # Use integers because the normal random() generates floating point numbers, @@ -99,23 +102,27 @@ def getrand(): if __name__ == '__main__': - if len(sys.argv) != 2: - print 'Use: random1.py number_of_keys' + if len(sys.argv) < 2: + print('Use: random1.py number_of_keys [key_prefix]') sys.exit(1) nkeys = int(sys.argv[1]) + if len(sys.argv) > 2: + key_prefix = sys.argv[2] + else: + key_prefix = '' # fill all the keys - print 'populate' - for i in xrange(nkeys): - set(getrand(), getrand()) + print('populate') + for i in range(nkeys): + set(key_prefix + str(getrand()), getrand()) - lkeys = ldb.keys() + lkeys = list(ldb.keys()) # operate on them a bit - print 'random operations' + print('random operations') operations = ('set', 'delete', 'cas0', 'cas1') - for i in xrange(nkeys / 2): + for i in range(nkeys // 2): op = choice(operations) k = choice(lkeys) if op == 'set': @@ -130,14 +137,14 @@ if __name__ == '__main__': # successful cas cas(k, ldb[k], getrand()) - print 'check' + print('check') check() - print 'delete' + print('delete') for k in lkeys: delete(k) - print 'check' + print('check') check() sys.exit(0) diff --git a/utils/Makefile b/utils/Makefile new file mode 100644 index 0000000..b56f6ae --- /dev/null +++ b/utils/Makefile @@ -0,0 +1,52 @@ + +CFLAGS += -std=c99 -pedantic -Wall -O3 +ALL_CFLAGS = -D_XOPEN_SOURCE=500 $(CFLAGS) + +ifdef DEBUG +ALL_CFLAGS += -g +endif + +ifdef PROFILE +ALL_CFLAGS += -g -pg -fprofile-arcs -ftest-coverage +endif + + +# prefix for installing the binaries +PREFIX=/usr/local + + +ifneq ($(V), 1) + NICE_CC = @echo " CC $@"; $(CC) +else + NICE_CC = $(CC) +endif + + +default: all + +all: nmdb-stats + +nmdb-stats: nmdb-stats.o + $(NICE_CC) $(ALL_CFLAGS) -L../libnmdb -lnmdb nmdb-stats.o \ + -o nmdb-stats + +.c.o: + $(NICE_CC) $(ALL_CFLAGS) -I../libnmdb -c $< -o $@ + +install-bin: nmdb-stats + install -d $(PREFIX)/bin + install -m 0755 nmdb-stats $(PREFIX)/bin + +install-man: + install -d $(PREFIX)/man/man1 + install -m 0644 nmdb-stats.1 $(PREFIX)/man/man1/ + +install: install-bin install-man + +clean: + rm -f nmdb-stats.o nmdb-stats + rm -f *.bb *.bbg *.da *.gcov *.gcda *.gcno gmon.out + +.PHONY: default all clean install-bin install-man install + + diff --git a/utils/nmdb-stats.1 b/utils/nmdb-stats.1 new file mode 100644 index 0000000..26e3681 --- /dev/null +++ b/utils/nmdb-stats.1 @@ -0,0 +1,33 @@ +.TH nmdb-stats 1 "11/Oct/2007" +.SH NAME +nmdb-stats - Get the stats of a nmdb server. +.SH SYNOPSYS +nmdb-stats [ tipc +.B port +| [tcp|udp|sctp] +.B host +.B port +] + +.SH DESCRIPTION + +This small application is used to query an nmdb server in order to get its +statistics. + +It takes the protocol as the first parameter (can be "tipc", "tcp", "udp", or +"sctp"), and then the server address. + +.SH INVOCATION EXAMPLE +.B "nmdb-stats tcp localhost 26010" + +.SH SEE ALSO +.BR nmdb (1). + +.SH AUTHORS +Created by Alberto Bertogli (albertito@blitiri.com.ar). + +.SH CONTACT + +If you want to report bugs, or have any questions or comments, just let me +know at albertito@blitiri.com.ar. + diff --git a/utils/nmdb-stats.c b/utils/nmdb-stats.c new file mode 100644 index 0000000..d9f1647 --- /dev/null +++ b/utils/nmdb-stats.c @@ -0,0 +1,150 @@ + +/* nmdb-stats.c + * Queries the stats of a nmdb server. + * Alberto Bertogli (albertito@blitiri.com.ar) + */ + +#include /* printf() */ +#include /* uint64_t */ +#include /* strcmp() */ +#include /* atoi() */ + +#include "nmdb.h" + + +/* ntohll() is not standard, so we define it using an UGLY trick because there + * is no standard way to check for endianness at runtime! (this is the same as + * the one in nmdb/parse.c, the infraestructure to keep these common is not + * worth it)*/ +static uint64_t ntohll(uint64_t x) +{ + static int endianness = 0; + + /* determine the endianness by checking how htonl() behaves; use -1 + * for little endian and 1 for big endian */ + if (endianness == 0) { + if (htonl(1) == 1) + endianness = 1; + else + endianness = -1; + } + + if (endianness == 1) { + /* big endian */ + return x; + } + + /* little endian */ + return ( ntohl( (x >> 32) & 0xFFFFFFFF ) | \ + ( (uint64_t) ntohl(x & 0xFFFFFFFF) ) << 32 ); +} + +#define MAX_STATS_SIZE 64 + +static void help(void) +{ + printf("Use: nmdb-stats [ tipc port | [tcp|udp|sctp] host port ]\n"); +} + +int main(int argc, char **argv) +{ + int i, j, k; + int rv; + uint64_t stats[MAX_STATS_SIZE]; + unsigned int nservers = 0, nstats = 0; + nmdb_t *db; + + db = nmdb_init(); + + if (argc < 3) { + help(); + return 1; + } + + if (strcmp("tipc", argv[1]) == 0) { + rv = nmdb_add_tipc_server(db, atoi(argv[2])); + } else { + if (argc != 4) { + help(); + return 1; + } + + if (strcmp("tcp", argv[1]) == 0) { + rv = nmdb_add_tcp_server(db, argv[2], atoi(argv[3])); + } else if (strcmp("udp", argv[1]) == 0) { + rv = nmdb_add_udp_server(db, argv[2], atoi(argv[3])); + } else if (strcmp("sctp", argv[1]) == 0) { + rv = nmdb_add_sctp_server(db, argv[2], atoi(argv[3])); + } else { + help(); + return 1; + } + } + + if (!rv) { + perror("Error adding server"); + return 1; + } + + rv = nmdb_stats(db, (unsigned char *) stats, sizeof(stats), + &nservers, &nstats); + if (rv <= 0) { + printf("Error %d\n", rv); + return 1; + } + + /* Macro to simplify showing the fields */ + #define shst(s, pos) \ + do { \ + printf("\t%ju\t%s\n", ntohll(stats[j + pos]), s); \ + } while(0) + + /* The following assumes it can be more than one server. This can + * never happen with the current code, but it can be useful as an + * example in the future. */ + j = 0; + for (i = 0; i < nservers; i++) { + printf("stats for server %d:\n", i); + + j = nstats * i; + + shst("cache get", 0); + shst("cache set", 1); + shst("cache del", 2); + shst("cache cas", 3); + shst("cache incr", 4); + + shst("db get", 5); + shst("db set", 6); + shst("db del", 7); + shst("db cas", 8); + shst("db incr", 9); + + shst("cache hits", 10); + shst("cache misses", 11); + + shst("db hits", 12); + shst("db misses", 13); + + shst("msg tipc", 14); + shst("msg tcp", 15); + shst("msg udp", 16); + shst("msg sctp", 17); + + shst("version mismatch", 18); + shst("broken requests", 19); + shst("unknown requests", 20); + + /* if there are any fields we don't know, show them anyway */ + for (k = 21; k < nstats; k++) { + shst("unknown field", k); + } + + printf("\n"); + } + + nmdb_free(db); + + return 0; +} +