LICENSE | 7 +- Makefile | 33 +++ README | 40 ++-- TODO | 12 - {python => bindings/d}/LICENSE | 0 bindings/d/nmdb.d | 253 ++++++++++++++++++ bindings/d/nmdb_ll.d | 65 +++++ bindings/d/sizeof.c | 15 + bindings/d/test1c.d | 45 ++++ bindings/d/testt.d | 22 ++ {python => bindings/newlisp}/LICENSE | 0 bindings/newlisp/nmdb.lsp | 140 ++++++++++ bindings/newlisp/test.lsp | 29 ++ {python => bindings/python}/LICENSE | 0 {python => bindings/python}/nmdb.py | 79 ++++-- {python => bindings/python}/nmdb_ll.c | 107 +++++++- {python => bindings/python}/setup.py | 0 doc/design.rst | 32 ++- doc/guide.rst | 110 +++++---- doc/network.rst | 18 +- libnmdb/Makefile | 56 ++--- libnmdb/internal.h | 27 ++ libnmdb/libnmdb.3 | 53 +++- libnmdb/libnmdb.c | 432 +++++++++++++++++++----------- libnmdb/net-const.h | 18 +- libnmdb/nmdb.h | 41 +++- libnmdb/tcp.c | 209 +++++++++++++++ libnmdb/tcp.h | 12 + libnmdb/test1d.c | 84 ------ libnmdb/test2cm.c | 114 -------- libnmdb/test2dm.c | 106 -------- libnmdb/test_functions.sh | 80 ------ libnmdb/tipc.c | 129 +++++++++ libnmdb/tipc.h | 12 + libnmdb/udp.c | 145 ++++++++++ libnmdb/udp.h | 12 + nmdb/Makefile | 45 +++- nmdb/cache.c | 49 ++++ nmdb/cache.h | 3 + nmdb/common.h | 18 +- nmdb/db.c | 52 +++- nmdb/main.c | 50 ++++- nmdb/net-const.h | 18 +- nmdb/net.c | 80 +++++- nmdb/nmdb.1 | 25 ++- nmdb/{tipc.c => parse.c} | 398 +++++++++++----------------- nmdb/parse.h | 11 + nmdb/queue.c | 8 +- nmdb/queue.h | 4 +- nmdb/req.h | 41 +++ nmdb/tcp-stub.c | 18 ++ nmdb/tcp.c | 469 +++++++++++++++++++++++++++++++++ nmdb/tcp.h | 10 + nmdb/tipc-stub.c | 18 ++ nmdb/tipc.c | 381 ++++----------------------- nmdb/tipc.h | 26 +-- nmdb/udp-stub.c | 18 ++ nmdb/udp.c | 240 +++++++++++++++++ nmdb/udp.h | 10 + libnmdb/test1c.c => tests/c/1.c | 15 +- libnmdb/test2c.c => tests/c/2.c | 15 +- libnmdb/test2d.c => tests/c/3.c | 40 ++-- tests/c/del.c | 70 +++++ libnmdb/test2d.c => tests/c/get.c | 48 +--- tests/c/make.sh | 50 ++++ tests/c/prototypes.h | 41 +++ libnmdb/test2d.c => tests/c/set.c | 43 +--- {libnmdb => tests/c}/timer.h | 4 +- tests/python/random1.py | 147 ++++++++++ 69 files changed, 3518 insertions(+), 1484 deletions(-) diff --git a/LICENSE b/LICENSE index 661eb2b..3b7cc0c 100644 --- a/LICENSE +++ b/LICENSE @@ -8,6 +8,9 @@ includes subdirectories). As a brief resume, here's how each sub-project is licensed: * nmdb: OSL 3.0 - * lib: BOLA (Public domain) - * python: BOLA (Public domain) + * libnmdb: BOLA (Public domain) + * bindings/python: BOLA (Public domain) + * bindings/d: BOLA (Public domain) + * bindings/newlisp: BOLA (Public domain) + diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..3fcab55 --- /dev/null +++ b/Makefile @@ -0,0 +1,33 @@ + +all: default + +default: nmdb libnmdb + +nmdb: + make -C nmdb + +libnmdb: + make -C libnmdb + +install: + make -C nmdb install + make -C libnmdb install + +clean: + make -C nmdb clean + make -C libnmdb clean + + +python: + cd bindings/python && python setup.py build + +python_install: + cd bindings/python && python setup.py install + +python_clean: + cd bindings/python && rm -rf build/ + + +.PHONY: default all clean nmdb libnmdb python python_install python_clean + + diff --git a/README b/README index 5b363dd..bde5926 100644 --- a/README +++ b/README @@ -1,10 +1,10 @@ -nmdb - A TIPC-based database manager +nmdb - A multiprotocol network database manager Alberto Bertogli (albertito@gmail.com) ------------------------------------------ +--------------------------------------------------- -nmdb is a network database that uses the TIPC protocol to communicate with -it's clients. +nmdb is a network database that can use several protocols to communicate with +it's clients. At the moment, it supports TIPC, TCP and UDP. It consists of an in-memory cache, that saves (key, value) pairs, and a persistent backend that stores the pairs on disk. @@ -13,9 +13,9 @@ Both work combined, but the use of the persistent backend is optional, so you can use the server only for cache queries, pretty much like memcached. This source distribution is composed of several parts: the server called -"nmdb", the library and the python bindings. Each one has a separate top level -directory, and is licensed individually. See the LICENSE file for more -information. +"nmdb", the library and bindings for Python, D and NewLISP. Each one has a +separate directory, and is licensed individually. See the LICENSE file for +more information. Documentation @@ -32,24 +32,26 @@ http://auriga.wearlab.de/~alb/nmdb. How to compile, test and install -------------------------------- -Requisites: - * Running Linux kernel >= 2.6.16 with TIPC enabled. - * Kernel headers also >= 2.6.16. Alternatively, you can run a TIPC patched - kernel, and have the tipc.h header be put somewhere in the include path - by hand. +General requisites: * libevent (http://www.monkey.org/~provos/libevent/). * qdbm (http://qdbm.sf.net/). -To compile the server and the library, you can just use "make" on their -directories. To install them, use "make install". +Requisites to enable TIPC: + * Running Linux kernel >= 2.6.16 with TIPC enabled. + * Kernel headers also >= 2.6.16. Alternatively, you can run a TIPC patched + kernel, and have the tipc.h header somewhere in the include path. + +To compile the server and the library, you can just use "make" on the top +level directory. To install them, use "make install". By default, all +protocols are enabled, including TIPC. To disable any of them, run something +like "make ENABLE_TIPC=0". -To run some tests, start the server and then go to the library directory. Run -"make tests", and then use the "test1c" and "test2c" to test the cache layer, -and "test1d" and "test2d" to test the database backend. +To run some tests, start the server and then go to the "tests/c/" directory. +Run "make.sh build" and then run any of the generated tests. To compile the Python bindings, you need to have the library already -installed. Go to the directory and run "python setup.py install", which will -build and install the modules. The module will be named "nmdb". +installed. Use "make python_install" at the top level directory to build and +install the modules. The module will be named "nmdb". Where to report bugs diff --git a/TODO b/TODO deleted file mode 100644 index f444ff3..0000000 --- a/TODO +++ /dev/null @@ -1,12 +0,0 @@ - -Server: -* Export stats -* Test it's endian-clean (it should work fine) -* Live switch from passive to active (command? signal?) - -Library: -* Support multithreading and/with non-blocking API - -Python: -* Write test-cases to test the binding, the library and the server - diff --git a/python/LICENSE b/bindings/d/LICENSE similarity index 100% copy from python/LICENSE copy to bindings/d/LICENSE diff --git a/bindings/d/nmdb.d b/bindings/d/nmdb.d new file mode 100644 index 0000000..e746d86 --- /dev/null +++ b/bindings/d/nmdb.d @@ -0,0 +1,253 @@ + +/* + * nmdb bindings for the Digital Mars D programming language. + * Alberto Bertogli (albertito@gmail.com) + */ + +module nmdb; + +import nmdb_ll; + + +/* Exception to raise when a key can't be found. It'd be better to use the + * same D arrays use (ArrayBoundsException) but it's designed for compiler + * use, and the lack of a standard exception hierarchy sucks. */ +class KeyNotFound : Exception { + this(char[] s) { + super(s); + } +} + + +/* Operation modes */ +enum { + MODE_NORMAL = 1, + MODE_CACHE = 2, + MODE_SYNC = 3, +} + + +class DB +{ + private nmdb_t *db; + int mode = MODE_NORMAL; + + + this() + { + db = nmdb_init(); + } + + ~this() + { + nmdb_free(db); + } + + void add_tipc_server(int port = -1) + { + int r = nmdb_add_tipc_server(db, port); + if (r == 0) { + throw new Exception("Can't add server"); + } + } + + void add_tcp_server(char[] addr, int port = -1) + { + int r = nmdb_add_tcp_server(db, cast(ubyte *) addr.ptr, port); + if (r == 0) { + throw new Exception("Can't add server"); + } + } + + void add_udp_server(char[] addr, int port = -1) + { + int r = nmdb_add_udp_server(db, cast(ubyte *) addr.ptr, port); + if (r == 0) { + throw new Exception("Can't add server"); + } + } + + private char[] do_get(char[] key, int mode) + { + ptrdiff_t size; + ubyte* k = cast(ubyte *) key.ptr; + auto v = new char[64 * 1024]; + + if (mode == MODE_NORMAL || mode == MODE_SYNC) { + size = nmdb_get(db, k, key.length, + cast(ubyte *) v, v.sizeof); + } else if (mode == MODE_CACHE) { + size = nmdb_cache_get(db, k, key.length, + cast(ubyte *) v, v.sizeof); + } else { + throw new Exception("Invalid mode"); + } + + if (size == 0) { + throw new KeyNotFound("Key not found: " ~ key); + } else if (size < 0) { + throw new Exception("Can't get value"); + } + + return v[0 .. cast(size_t) size]; + } + + private void do_set(char[] key, char[] val, int mode) + { + ubyte* k = cast(ubyte *) key.ptr; + ubyte* v = cast(ubyte *) val.ptr; + int res = 0; + + if (mode == MODE_NORMAL) { + res = nmdb_set(db, k, key.length, v, val.length); + } else if (mode == MODE_SYNC) { + res = nmdb_set_sync(db, k, key.length, + v, val.length); + } else if (mode == MODE_CACHE) { + res = nmdb_cache_set(db, k, key.length, + v, val.length); + } else { + throw new Exception("Invalid mode"); + } + + if (res != 1) { + throw new Exception("Can't set value"); + } + } + + private int do_del(char[] key, int mode) + { + ubyte* k = cast(ubyte *) key.ptr; + int res = 0; + + if (mode == MODE_NORMAL) { + res = nmdb_del(db, k, key.length); + } else if (mode == MODE_SYNC) { + res = nmdb_del_sync(db, k, key.length); + } else if (mode == MODE_CACHE) { + res = nmdb_cache_del(db, k, key.length); + } else { + throw new Exception("Invalid mode"); + } + return res; + } + + private int do_cas(char[] key, char[] oldval, char[] newval, + int mode) + { + ubyte* k = cast(ubyte *) key.ptr; + ubyte* ov = cast(ubyte *) oldval.ptr; + ubyte* nv = cast(ubyte *) newval.ptr; + int res = 0; + + if (mode == MODE_NORMAL || mode == MODE_SYNC) { + res = nmdb_cas(db, k, key.length, + ov, oldval.length, + nv, newval.length); + } else if (mode == MODE_CACHE) { + res = nmdb_cache_cas(db, k, key.length, + ov, oldval.length, + nv, newval.length); + } else { + throw new Exception("Invalid mode"); + } + return res; + } + + + char[] get(char[] key) + { + return do_get(key, mode); + } + + char[] get_normal(char[] key) + { + return do_get(key, MODE_NORMAL); + } + + char[] cache_get(char[] key) + { + return do_get(key, MODE_CACHE); + } + + + void set(char[] key, char[] val) + { + return do_set(key, val, mode); + } + + void set_normal(char[] key, char[] val) + { + return do_set(key, val, MODE_NORMAL); + } + + void set_sync(char[] key, char[] val) + { + return do_set(key, val, MODE_SYNC); + } + + void cache_set(char[] key, char[] val) + { + return do_set(key, val, MODE_CACHE); + } + + + int remove(char[] key) + { + return do_del(key, mode); + } + + int remove_normal(char[] key) + { + return do_del(key, MODE_NORMAL); + } + + int remove_sync(char[] key) + { + return do_del(key, MODE_SYNC); + } + + int cache_remove(char[] key) + { + return do_del(key, MODE_CACHE); + } + + + int cas(char[] key, char[] oldval, char[] newval) + { + return do_cas(key, oldval, newval, mode); + } + + int cas_normal(char[] key, char[] oldval, char[] newval) + { + return do_cas(key, oldval, newval, MODE_NORMAL); + } + + int cache_cas(char[] key, char[] oldval, char[] newval) + { + return do_cas(key, oldval, newval, MODE_CACHE); + } + + + char[] opIndex(char[] key) + { + return get(key); + } + + void opIndexAssign(char[] val, char[] key) + { + return set(key, val); + } + + bool opIn_r(char[] key) + { + try { + get(key); + } catch (KeyNotFound(s)) { + return false; + } + return true; + } + +} + diff --git a/bindings/d/nmdb_ll.d b/bindings/d/nmdb_ll.d new file mode 100644 index 0000000..6fc3852 --- /dev/null +++ b/bindings/d/nmdb_ll.d @@ -0,0 +1,65 @@ + +/* Low-level bindings, used by nmdb.d. */ + +module nmdb_ll; + + +/* We need to define the nmdb_t type used by the C API. + * + * One possiblity is to import the C struct here (alignment and stuff is + * supposed to be the same), but since it includes some OS structures like + * sockaddr_tipc, it means several lines of code and it's difficult to + * maintain. Because to us it's an opaque type, we define it to be an ubyte + * array of the same length as C's struct nmdb_t. That way we retain ABI + * compatibility but minimize the clutter. + * + * To port this to another architecture, just compile and run the "sizeof.c" + * program. It should output some lines like "sizeof(struct nmdb_t) = 16". + * Then use that information to define the aliases for your platform. + * + * Should nmdb_t change, the numbers must be updated to reflect the new sizes. + */ + +version (X86) { + /* Generated on a Pentium II running Ubuntu. It should be the same on + * all x86 Linux boxes. */ + alias ubyte[12] nmdb_t; +} + +version (X86_64) { + /* Generated on a Pentium D running Gentoo in 64 bits mode. It should + * be the same on all Linux amd64 boxes. */ + alias ubyte[16] nmdb_t; +} + + +/* nmdb structures and prototypes, these shouldn't need any changes + * unless libnmdb/nmdb.h is updated */ + +extern (C) nmdb_t *nmdb_init(); +extern (C) int nmdb_add_tipc_server(nmdb_t *db, int port); +extern (C) int nmdb_add_tcp_server(nmdb_t *db, ubyte *addr, int port); +extern (C) int nmdb_add_udp_server(nmdb_t *db, ubyte *addr, int port); +extern (C) int nmdb_free(nmdb_t *db); + +extern (C) ptrdiff_t nmdb_get(nmdb_t *db, ubyte *key, size_t ksize, + ubyte *val, size_t vsize); +extern (C) ptrdiff_t nmdb_cache_get(nmdb_t *db, ubyte *key, size_t ksize, + ubyte *val, size_t vsize); + +extern (C) int nmdb_set(nmdb_t *db, ubyte *key, size_t ksize, + ubyte *val, size_t vsize); +extern (C) int nmdb_set_sync(nmdb_t *db, ubyte *key, size_t ksize, + ubyte *val, size_t vsize); +extern (C) int nmdb_cache_set(nmdb_t *db, ubyte *key, size_t ksize, + ubyte *val, size_t vsize); + +extern (C) int nmdb_del(nmdb_t *db, ubyte *key, size_t ksize); +extern (C) int nmdb_del_sync(nmdb_t *db, ubyte *key, size_t ksize); +extern (C) int nmdb_cache_del(nmdb_t *db, ubyte *key, size_t ksize); + +extern (C) int nmdb_cas(nmdb_t *db, ubyte *key, size_t ksize, + ubyte *oldval, size_t ovsize, ubyte *newval, size_t nvsize); +extern (C) int nmdb_cache_cas(nmdb_t *db, ubyte *key, size_t ksize, + ubyte *oldval, size_t ovsize, ubyte *newval, size_t nvsize); + diff --git a/bindings/d/sizeof.c b/bindings/d/sizeof.c new file mode 100644 index 0000000..2a8e7b5 --- /dev/null +++ b/bindings/d/sizeof.c @@ -0,0 +1,15 @@ + +/* sizeof.c + * Used to find out the size of some intresting structures and data types, to + * help defining D's bindings. + */ + +#include +#include + +int main() +{ + printf("sizeof(struct nmdb_t) = %lu\n", sizeof(struct nmdb_t)); + return 0; +} + diff --git a/bindings/d/test1c.d b/bindings/d/test1c.d new file mode 100644 index 0000000..a3f3caf --- /dev/null +++ b/bindings/d/test1c.d @@ -0,0 +1,45 @@ + +import nmdb; +import std.stdio; +import std.stream; +import std.string; +import std.perf; + + +int main(char [][] argv) +{ + if (argv.length != 2) { + writefln("Usage: test1d TIMES"); + return 1; + } + + auto times = atoi(argv[1]); + char[] val; + + nmdb.DB db = new nmdb.DB(); + db.add_tipc_server(); + db.mode = nmdb.MODE_CACHE; + + auto counter = new PerformanceCounter; + + counter.start(); + for (int i = 0; i < times; i++) + db["1"] = "D"; + counter.stop(); + writefln("d set: ", counter.microseconds()); + + counter.start(); + for (int i = 0; i < times; i++) + val = db["1"]; + counter.stop(); + writefln("d get: ", counter.microseconds()); + + counter.start(); + for (int i = 0; i < times; i++) + db.remove("1"); + counter.stop(); + writefln("d del: ", counter.microseconds()); + + return 0; +} + diff --git a/bindings/d/testt.d b/bindings/d/testt.d new file mode 100644 index 0000000..e2b820c --- /dev/null +++ b/bindings/d/testt.d @@ -0,0 +1,22 @@ + +import nmdb; +import std.stdio; +import std.stream; + + +int main() +{ + char[] val1; + + nmdb.DB db = new nmdb.DB(); + db.add_tipc_server(); + + db.mode = MODE_CACHE; + db["1"] = "D"; + val1 = db["1"]; + + writefln(val1); + + return 0; +} + diff --git a/python/LICENSE b/bindings/newlisp/LICENSE similarity index 100% copy from python/LICENSE copy to bindings/newlisp/LICENSE diff --git a/bindings/newlisp/nmdb.lsp b/bindings/newlisp/nmdb.lsp new file mode 100644 index 0000000..187e1dc --- /dev/null +++ b/bindings/newlisp/nmdb.lsp @@ -0,0 +1,140 @@ + +; +; nmdb bindings for newlisp (http://www.newlisp.org/) +; Alberto Bertogli (albertito@gmail.com) +; +; Functions: +; +; (nmdb:init) -> Creates a new database object. +; (nmdb:add-tipc-server port) -> Adds a new TIPC server to the database. +; (nmdb:add-tcp-server addr port) -> Adds a new TCP server to the database. +; (nmdb:add-udp-server addr port) -> Adds a new UDP server to the database. +; (nmdb:free) -> Closes the database. +; +; (nmdb:db-get key) -> Gets the value associated to the given key, or -1. +; (nmdb:cache-get key) -> Like dbget but only get from the cache. +; +; (nmdb:db-set key val ) -> Sets the given key to the given value. +; (nmdb:sync-set key val ) -> Like db-set but synchronous. +; (nmdb:cache-set key val ) -> Like db-set but only set to the the cache. +; +; (nmdb:db-del key ) -> Removes the given key from the database. +; (nmdb:sync-del key ) -> Like db-del but synchronous. +; (nmdb:cache-del key ) -> Like db-del but only delete from the cache. +; +; +; Example: +; (load "nmdb.lsp") +; (nmdb:init) +; (nmdb:add-tipc-server 10) +; (nmdb:db-set "Hello" "Newlisp!") +; (nmdb:db-get "Hello") +; (nmdb:db-del "Hello") +; (nmdb:free) +; +; For more information check the nmdb docs. +; + + +(context 'nmdb) + +; library loading +(set 'libnmdb "libnmdb.so") + +(import libnmdb "nmdb_init") +(import libnmdb "nmdb_add_tipc_server") +(import libnmdb "nmdb_add_tcp_server") +(import libnmdb "nmdb_add_udp_server") +(import libnmdb "nmdb_free") + +(import libnmdb "nmdb_set") +(import libnmdb "nmdb_set_sync") +(import libnmdb "nmdb_cache_set") + +(import libnmdb "nmdb_get") +(import libnmdb "nmdb_cache_get") + +(import libnmdb "nmdb_del") +(import libnmdb "nmdb_del_sync") +(import libnmdb "nmdb_cache_del") + +(import libnmdb "nmdb_cas") +(import libnmdb "nmdb_cache_cas") + + +; main functions + +(define (init port) + (set 'NMDB (nmdb_init port)) + (if (= NMDB 0) (set NMDB nil)) + (not (= NMDB nil))) + +(define (add-tipc-server port) + (nmdb_add_tipc_server NMDB port)) + +(define (add-tcp-server addr port) + (nmdb_add_tcp_server NMDB addr port)) + +(define (add-udp-server addr port) + (nmdb_add_udp_server NMDB addr port)) + +(define (free) + (nmdb_free NMDB)) + + +; *-get functions + +(define (priv-get func key) + (letn ( (keylen (length key)) + (vallen (* 64 1024)) + (val (dup "\000" vallen)) + ) + (set 'rv (func NMDB key keylen val vallen)) + (if (> rv 0) + (slice val 0 rv) + -1) ) ) + +(define (db-get key) (priv-get nmdb_get key)) +(define (cache-get key) (priv-get nmdb_cache_get key)) + + +; *-set functions + +(define (priv-set func key val) + (letn ( (keylen (length key)) + (vallen (length val)) + ) + (func NMDB key keylen val vallen) ) ) + +(define (db-set key val) (priv-set nmdb_set key val)) +(define (sync-set key val) (priv-set nmdb_set_sync key val)) +(define (cache-set key val) (priv-set nmdb_cache_set key val)) + + +; *-del functions +(define (priv-del func key) + (letn ( (keylen (length key)) ) + (func NMDB key keylen) ) ) + +(define (db-del key) (priv-del nmdb_del key)) +(define (sync-del key) (priv-del nmdb_del_sync key)) +(define (cache-del key) (priv-del nmdb_cache_del key)) + + +; *-cas functions +(define (priv-cas func key oldval newval) + (letn ( (keylen (length key)) + (ovlen (length oldval)) + (nvlen (length newval)) + ) + (func NMDB key keylen oldval ovlen newval nvlen) ) ) + +(define (db-cas key oval nval) (priv-cas nmdb_cas key oval nval)) +(define (cache-cas key oval nval) (priv-cas nmdb_cache_cas key oval nval)) + + + +(context MAIN) + + + diff --git a/bindings/newlisp/test.lsp b/bindings/newlisp/test.lsp new file mode 100644 index 0000000..6a1c784 --- /dev/null +++ b/bindings/newlisp/test.lsp @@ -0,0 +1,29 @@ + +; To run: newlisp test.lsp + +(load "nmdb.lsp") + +(println "init\t\t" (nmdb:init)) +(println "add-tipc-server\t" (nmdb:add-tipc-server 10)) +;(println "add-tipc-server\t" (nmdb:add-tipc-server 11)) +;(println "add-tipc-server\t" (nmdb:add-tipc-server 12)) +;(println "add-tcp-server\t" (nmdb:add-tcp-server "127.0.0.1" -1)) +;(println "add-udp-server\t" (nmdb:add-udp-server "127.0.0.1" -1)) +(println) +(println "db-set D1 V1\t" (nmdb:db-set "D1" "D1")) +(println "sync-set S2 V2\t" (nmdb:sync-set "S2" "V2")) +(println "cache-set C3 C3\t" (nmdb:cache-set "C3" "C3")) +(println) +(println "db-get D1\t" (nmdb:db-get "D1")) +(println "db-get S2\t" (nmdb:db-get "S2")) +(println "cache-get C3\t" (nmdb:cache-get "C3")) +(println) +(println "db-cas D1\t" (nmdb:db-cas "D1" "D1" "DX")) +(println "cache-cas C3\t" (nmdb:cache-cas "C3" "C3" "CX")) +(println) +(println "db-del D1\t" (nmdb:db-del "D1")) +(println "sync-del S2\t" (nmdb:sync-del "S2")) +(println "cache-del C3\t" (nmdb:cache-del "C3")) + +(exit) + diff --git a/python/LICENSE b/bindings/python/LICENSE similarity index 100% rename from python/LICENSE rename to bindings/python/LICENSE diff --git a/python/nmdb.py b/bindings/python/nmdb.py similarity index 65% rename from python/nmdb.py rename to bindings/python/nmdb.py index caa5663..4d0dccc 100644 --- a/python/nmdb.py +++ b/bindings/python/nmdb.py @@ -24,8 +24,7 @@ Here is an example using the DB class: >>> import nmdb >>> db = nmdb.DB() ->>> import nmdb ->>> db = nmdb.DB() +>>> db.add_tipc_server() >>> db[1] = { 'english': 'one', 'castellano': 'uno', 'quechua': 'huk' } >>> print db[1] {'english': 'one', 'castellano': 'uno', 'quechua': 'huk'} @@ -50,16 +49,31 @@ class NetworkError (Exception): class _nmdbDict (object): - def __init__(self, db, op_get, op_set, op_delete): - self.db = db - self.get = op_get - self.set = op_set - self.delete = op_delete + def __init__(self, db, op_get, op_set, op_delete, op_cas): + self._db = db + self._get = op_get + self._set = op_set + self._delete = op_delete + self._cas = op_cas self.autopickle = True - def add_server(self, port): - "Adds a server to the server pool." - rv = self.db.add_server(port) + def add_tipc_server(self, port = -1): + "Adds a TIPC server to the server pool." + rv = self._db.add_tipc_server(port) + if not rv: + raise NetworkError + return rv + + def add_tcp_server(self, addr, port = -1): + "Adds a TCP server to the server pool." + rv = self._db.add_tcp_server(addr, port) + if not rv: + raise NetworkError + return rv + + def add_udp_server(self, addr, port = -1): + "Adds an UDP server to the server pool." + rv = self._db.add_udp_server(addr, port) if not rv: raise NetworkError return rv @@ -69,7 +83,7 @@ class _nmdbDict (object): if self.autopickle: key = str(hash(key)) try: - r = self.get(key) + r = self._get(key) except: raise NetworkError if not r: @@ -83,7 +97,7 @@ class _nmdbDict (object): if self.autopickle: key = str(hash(key)) val = cPickle.dumps(val, protocol = -1) - r = self.set(key, val) + r = self._set(key, val) if r <= 0: raise NetworkError return 1 @@ -92,7 +106,7 @@ class _nmdbDict (object): "del d[k] Deletes the key k." if self.autopickle: key = str(hash(key)) - r = self.delete(key) + r = self._delete(key) if r < 0: raise NetworkError elif r == 0: @@ -104,7 +118,7 @@ class _nmdbDict (object): if self.autopickle: key = str(hash(key)) try: - r = self.get(key) + r = self._get(key) except KeyError: return False if not r: @@ -115,22 +129,41 @@ class _nmdbDict (object): "Returns True if the key is in the database, False otherwise." return self.__contains__(key) + def cas(self, key, oldval, newval): + "Perform a compare-and-swap." + if self.autopickle: + key = str(hash(key)) + oldval = cPickle.dumps(oldval, protocol = -1) + newval = cPickle.dumps(newval, protocol = -1) + r = self._cas(key, oldval, newval) + if r == 2: + # success + return 2 + elif r == 1: + # no match + return 1 + elif r == 0: + # not in + raise KeyError + else: + raise NetworkError + class Cache (_nmdbDict): - def __init__(self, port = -1): - db = nmdb_ll.connect(port) + def __init__(self): + db = nmdb_ll.connect() _nmdbDict.__init__(self, db, db.cache_get, db.cache_set, - db.cache_delete) + db.cache_delete, db.cache_cas) class DB (_nmdbDict): - def __init__(self, port = -1): - db = nmdb_ll.connect(port) - _nmdbDict.__init__(self, db, db.get, db.set, db.delete) + def __init__(self): + db = nmdb_ll.connect() + _nmdbDict.__init__(self, db, db.get, db.set, db.delete, db.cas) class SyncDB (_nmdbDict): - def __init__(self, port = -1): - db = nmdb_ll.connect(port) + def __init__(self): + db = nmdb_ll.connect() _nmdbDict.__init__(self, db, db.get, db.set_sync, - db.delete_sync) + db.delete_sync, db.cas) diff --git a/python/nmdb_ll.c b/bindings/python/nmdb_ll.c similarity index 72% rename from python/nmdb_ll.c rename to bindings/python/nmdb_ll.c index 47f5b2c..e37b22a 100644 --- a/python/nmdb_ll.c +++ b/bindings/python/nmdb_ll.c @@ -35,18 +35,54 @@ static void db_dealloc(nmdbobject *db) } -/* add server */ -static PyObject *db_add_server(nmdbobject *db, PyObject *args) +/* add tipc server */ +static PyObject *db_add_tipc_server(nmdbobject *db, PyObject *args) { int port; int rv; - if (!PyArg_ParseTuple(args, "i:add_server", &port)) { + if (!PyArg_ParseTuple(args, "i:add_tipc_server", &port)) { return NULL; } Py_BEGIN_ALLOW_THREADS - rv = nmdb_add_server(db->db, port); + rv = nmdb_add_tipc_server(db->db, port); + Py_END_ALLOW_THREADS + + return PyLong_FromLong(rv); +} + +/* add tcp server */ +static PyObject *db_add_tcp_server(nmdbobject *db, PyObject *args) +{ + int port; + char *addr; + int rv; + + if (!PyArg_ParseTuple(args, "si:add_tcp_server", &addr, &port)) { + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + rv = nmdb_add_tcp_server(db->db, addr, port); + Py_END_ALLOW_THREADS + + return PyLong_FromLong(rv); +} + +/* add udp server */ +static PyObject *db_add_udp_server(nmdbobject *db, PyObject *args) +{ + int port; + char *addr; + int rv; + + if (!PyArg_ParseTuple(args, "si:add_udp_server", &addr, &port)) { + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + rv = nmdb_add_udp_server(db->db, addr, port); Py_END_ALLOW_THREADS return PyLong_FromLong(rv); @@ -124,6 +160,28 @@ static PyObject *db_cache_delete(nmdbobject *db, PyObject *args) return PyLong_FromLong(rv); } +/* cache cas */ +static PyObject *db_cache_cas(nmdbobject *db, PyObject *args) +{ + unsigned char *key, *oldval, *newval; + int ksize, ovsize, nvsize; + int rv; + + if (!PyArg_ParseTuple(args, "s#s#s#:cache_cas", &key, &ksize, + &oldval, &ovsize, + &newval, &nvsize)) { + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + rv = nmdb_cache_cas(db->db, key, ksize, oldval, ovsize, + newval, nvsize); + Py_END_ALLOW_THREADS + + return PyLong_FromLong(rv); +} + + /* db set */ static PyObject *db_set(nmdbobject *db, PyObject *args) { @@ -196,6 +254,26 @@ static PyObject *db_delete(nmdbobject *db, PyObject *args) return PyLong_FromLong(rv); } +/* db cas */ +static PyObject *db_cas(nmdbobject *db, PyObject *args) +{ + unsigned char *key, *oldval, *newval; + int ksize, ovsize, nvsize; + int rv; + + if (!PyArg_ParseTuple(args, "s#s#s#:cas", &key, &ksize, + &oldval, &ovsize, + &newval, &nvsize)) { + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + rv = nmdb_cas(db->db, key, ksize, oldval, ovsize, newval, nvsize); + Py_END_ALLOW_THREADS + + return PyLong_FromLong(rv); +} + /* db set sync */ static PyObject *db_set_sync(nmdbobject *db, PyObject *args) @@ -239,13 +317,20 @@ static PyObject *db_delete_sync(nmdbobject *db, PyObject *args) /* nmdb method table */ static PyMethodDef nmdb_methods[] = { - { "add_server", (PyCFunction) db_add_server, METH_VARARGS, NULL }, + { "add_tipc_server", (PyCFunction) db_add_tipc_server, + METH_VARARGS, NULL }, + { "add_tcp_server", (PyCFunction) db_add_tcp_server, + METH_VARARGS, NULL }, + { "add_udp_server", (PyCFunction) db_add_udp_server, + METH_VARARGS, NULL }, { "cache_set", (PyCFunction) db_cache_set, METH_VARARGS, NULL }, { "cache_get", (PyCFunction) db_cache_get, METH_VARARGS, NULL }, { "cache_delete", (PyCFunction) db_cache_delete, METH_VARARGS, NULL }, + { "cache_cas", (PyCFunction) db_cache_cas, METH_VARARGS, NULL }, { "set", (PyCFunction) db_set, METH_VARARGS, NULL }, { "get", (PyCFunction) db_get, METH_VARARGS, NULL }, { "delete", (PyCFunction) db_delete, METH_VARARGS, NULL }, + { "cas", (PyCFunction) db_cas, METH_VARARGS, NULL }, { "set_sync", (PyCFunction) db_set_sync, METH_VARARGS, NULL }, { "delete_sync", (PyCFunction) db_delete_sync, METH_VARARGS, NULL }, @@ -277,18 +362,16 @@ static PyTypeObject nmdbType = { static PyObject *db_connect(PyObject *self, PyObject *args) { nmdbobject *db; - long port; - - if (!PyArg_ParseTuple(args, "i:connect", &port)) { - return NULL; - } - db = PyObject_New(nmdbobject, &nmdbType); if (db == NULL) return NULL; - db->db = nmdb_init(port); + if (!PyArg_ParseTuple(args, ":connect")) { + return NULL; + } + + db->db = nmdb_init(); if (db->db == NULL) { return PyErr_NoMemory(); } diff --git a/python/setup.py b/bindings/python/setup.py similarity index 100% rename from python/setup.py rename to bindings/python/setup.py diff --git a/doc/design.rst b/doc/design.rst index 2acb447..105370c 100644 --- a/doc/design.rst +++ b/doc/design.rst @@ -1,16 +1,16 @@ -===================================== -nmdb - A TIPC-based database manager -===================================== +================================================= +nmdb - A multiprotocol network database manager +================================================= :Author: Alberto Bertogli (albertito@gmail.com) Introduction ============ -nmdb_ is a simple and fast cache and database for TIPC clusters. It allows -applications in the cluster to use a centralized, shared cache and database in -a very easy way. It stores *(key, value)* pairs, with each key having only one +nmdb_ is a simple and fast cache and database for controlled networks. It +allows applications to use a centralized, shared cache and database in a very +easy way. It stores *(key, value)* pairs, with each key having only one associated value. This document explains how the server works internally, and why it works that @@ -20,10 +20,13 @@ way. Network interface ================= -The server communicates with its clients using TIPC_ messages, which are -limited to 66000 bytes in size. It's completely connectionless, and uses the -reliable datagram layer provided by TIPC_. The network protocol is specified -in another document, and will not be subject of analysis here. +The server communicates with its clients using messages, which can be +delivered through TIPC_, TCP or UDP. Messages are limited by design to 64k so +they stay inside within TIPC_'s limits. + +TIPC_ is completely connectionless, and uses the reliable datagram layer +provided by TIPC_. The network protocol is specified in another document, and +will not be subject of analysis here. The interaction is very simple: the client sends a request message for an action, and the server replies to it. There is only one message per request, @@ -62,6 +65,9 @@ cache_set *key* *value* cache_del *key* Like *del*, but only affects the cache and not the database. +cache_cas *key* *oldvalue* *newvalue* + Do a compare-and-swap, using *oldvalue* to compare with the value stored in + the database, and replacing it with *newvalue* if they match. As you can see, it's possible to operate exclusively with the cache, ignoring the database completely. This is very similar to what memcached_ does. Note @@ -149,12 +155,14 @@ acts upon them internally, but never sends any replies. It is used for redundancy purposes, allowing the administrator to have an up-to-date copy of the database in case the main one fails. +It only makes sense if used with TIPC_ because it can multicast messages. + The implementation is quite simple, because the code paths are exactly the same, with the exception of skipping the network replies, so they're done conditionally depending on the passive setting. -Live switching of a server from passive to active (and vice-versa) should be -possible, although it is not yet implemented. +Live switching of a server from passive to active (and vice-versa) can be done +at runtime by sending a *SIGUSR2* signal to the server. The cache layer diff --git a/doc/guide.rst b/doc/guide.rst index f0de916..4431722 100644 --- a/doc/guide.rst +++ b/doc/guide.rst @@ -8,10 +8,10 @@ nmdb User Guide Introduction ============ -nmdb_ is a simple and fast cache and database for TIPC_ clusters. It allows -applications in the cluster to use a centralized, shared cache and database in -a very easy way. It stores *(key, value)* pairs, with each key having only one -associated value. +nmdb_ is a simple and fast cache and database for TIPC_, TCP and UDP clusters. +It allows applications in the cluster to use a centralized, shared cache and +database in a very easy way. It stores *(key, value)* pairs, with each key +having only one associated value. This document explains how to setup nmdb and a simple guide to writing clients. It also includes a "quick start" section for the anxious. @@ -30,9 +30,12 @@ Prerequisites Before you install nmdb, you will need the following software: - libevent_, a library for fast event handling. -- `Linux kernel`_ 2.6.16 or newer, compiled with TIPC_ support. - QDBM_, for the database backend. +And, if you're going to use TIPC_: + +- `Linux kernel`_ 2.6.16 or newer, compiled with TIPC_ support. + Compiling and installing ------------------------ @@ -40,9 +43,11 @@ Compiling and installing There are three components of the nmdb tarball: the server in the *nmdb/* directory, the C library in *libnmdb/*, and the Python module in *python/*. -- To install the server, run ``cd nmdb; make install``. -- To install the C library, run ``cd libnmdb; make install; ldconfig``. -- To install the Python module, run ``cd python; python setup.py install``. +To install the server and the C library, run ``make install; ldconfig``. To +install the Python module, run ``make python_install``. + +If you want to disable support for some protocol (i.e. TIPC), you can do so by +running ``make ENABLE_TIPC=0 install``. Quick start @@ -61,7 +66,8 @@ simple way to test it is to use the python module, like this:: [GCC 4.1.1 (Gentoo 4.1.1)] on linux2 Type "help", "copyright", "credits" or "license" for more information. >>> import nmdb # import the module - >>> db = nmdb.DB() # connect to the server + >>> db = nmdb.DB() # create a DB object + >>> db.add_tcp_server("localhost") # connect to the TCP server >>> db['x'] = 1 # store some data >>> db[(1, 2)] = (2, 6) >>> print db['x'], db[(1, 2)] # retreive the values @@ -78,9 +84,10 @@ out how to setup a simple TIPC cluster. TIPC setup ========== -If you want to use the server and the clients in different machines, you need -to setup your TIPC network. If you just want to run everything in one machine, -or you already have a TIPC network set up, you can skip this section. +If you want to use the server and the clients in different machines using +TIPC, you need to setup your TIPC network. If you just want to run everything +in one machine, you already have a TIPC network set up, or you only want to +use TCP or UDP connections, you can skip this section. Before we begin, all the machines should already be connected in an Ethernet LAN, and have the tipc-config application that should come with your Linux @@ -123,25 +130,11 @@ Starting the server Before starting the server, there are some things you need to know about it: -Port numbers - Each server instance in your network (even the ones running in the same - machine) should get a **unique** port to listen to requests. Ports identify - an application instance inside the whole network, not just the machine as in - TCP/IP. - - The port space is very very large, and it's private to nmdb, so you can - choose numbers without fear of colliding with other TIPC applications. The - default port is 10. - - So, if you are going to start more than one nmdb server, **be careful**. If - you assign two active servers the same port you will get no error, but - everything will act weird. - Cache size nmdb's cache is a main component of the server. In fact you can use it exclusively for caching purposes, like memcached_. So the size becomes an important issue if you have performance requirements. - + It is only possible to limit the cache size by the maximum number of objects in the cache. @@ -172,18 +165,32 @@ Distributed queries the queries among them. This is entirely done on the client side and the server doesn't know about it. +TIPC Port numbers + With TIPC, each server instance in your network (even the ones running in + the same machine) should get a **unique** port to listen to requests. Ports + identify an application instance inside the whole network, not just the + machine as in TCP/IP. + + The port space is very very large, and it's private to nmdb, so you can + choose numbers without fear of colliding with other TIPC applications. The + default port is 10. + + So, if you are going to start more than one nmdb server, **be careful**. If + you assign two active servers the same port you will get no error, but + everything will act weird. + Now that you know all that, starting a server should be quite simple: first create the database as explained above, and then run the daemon with ``nmdb -d /path/to/the/database``. -To change the port, use ``-l port``, to change the cache size, use ``-c nobj`` -(where *nobj* is the number of objects in thousands), to make the server -passive, use ``-p``. Of course you won't remember all that (I know I don't), -that's why ``-h`` is your friend. +There are several options you can change at start time. Of course you won't +remember all that (I know I don't), so check out ``nmdb -h`` to see a complete +list. -Nothing prevents you from starting more than one server in the same machine, -so be careful to select different ports and databases for each one. +Nothing prevents you from starting more than one TIPC server in the same +machine, so be careful to select different TIPC ports and databases for each +one. Example @@ -214,9 +221,10 @@ Thread safety that needs it. Available operations - You can request the server to do three operations: *set* a value to a key, - *get* the value associated with the given key, and *delete* a given key - (with its associated value). + You can request the server to do four operations: *set* a value to a key, + *get* the value associated with the given key, *delete* a given key (with + its associated value), and perform a *compare-and-swap* of the values + associated with the given key. Request modes For each operation, you will have three different modes available: @@ -245,7 +253,8 @@ Distributed queries For all examples we will assume that you have three servers running in your -network, in ports 11, 12 and 13. +network, two in TIPC ports 11 and 12, and one TCP listening on localhost on +the default port. The Python module @@ -261,12 +270,14 @@ objects as values, unless you know what you are doing. To start a connection to the servers, you must first decide which mode you are going to use: the normal database-backed mode, database-backed with synchronous access, or cache only. Let's say you want to use the normal mode -and connect to the server at port 11, and then add the other two servers:: +and connect to the TIPC servers at port 11, 12, and a TCP server on localhost +at the default port:: import nmdb - db = nmdb.DB(11) - db.add_server(12) - db.add_server(13) + db = nmdb.DB() + db.add_tipc_server(11) + db.add_tipc_server(12) + db.add_tcp_server("127.0.0.1") Now you're ready to use it. Let's suppose you want to write a recursive function to calculate the factorial of a number. But before doing the @@ -315,9 +326,10 @@ similar to what we did before, and has the advantage of not having to write our own cache management routines:: import nmdb - db = nmdb.Cache(11) - db.add_server(12) - db.add_server(13) + db = nmdb.Cache() + db.add_tipc_server(11) + db.add_tipc_server(12) + db.add_tcp_server("127.0.0.1") Let's write the decorator:: @@ -359,15 +371,17 @@ The C library is in essence similar to the Python module, so we won't make a very long example here, only a brief display of the available functions. Let's begin by creating a "nmdb descriptor" which is of type *nmdb_t*, and -connecting it to your three servers:: +connecting it to your three servers (two TIPC at ports 11 and 12, one TCP on +localhost, default port):: unsigned char *key, *val; size_t ksize, vsize; nmdb_t *db; - db = nmdb_connect(11); - nmdb_add_server(db, 12); - nmdb_add_server(db, 13); + db = nmdb_init(); + nmdb_add_tipc_server(db, 11); + nmdb_add_tipc_server(db, 12); + nmdb_add_tcp_server(db, "127.0.0.1", -1); Now you can do some operations (allocations and checks are not shown for brevity):: diff --git a/doc/network.rst b/doc/network.rst index 78b26a1..7df5cae 100644 --- a/doc/network.rst +++ b/doc/network.rst @@ -6,6 +6,12 @@ nmdb_ Network Protocol **NOTE:** All integers are in network byte order. +The nmdb network protocol relies on a message passing underlying transport +protocol. It normally uses TIPC, but can use UDP, or TCP with a messaging +layer too. This document describes the protocol in a transport-independent +way, assuming the transport protocol can send and receive messages reliably +and preserve message boundaries. No ordering guarantees are required. + Requests ======== @@ -51,6 +57,8 @@ REQ_SET_SYNC 0x105 REQ_DEL_SYNC 0x106 REQ_SET_ASYNC 0x107 REQ_DEL_ASYNC 0x108 +REQ_CACHE_CAS 0x109 +REQ__CAS 0x110 ============== ====== @@ -67,6 +75,9 @@ REQ_SET_* and REQ_CACHE_SET REQ_DEL_* and REQ_CACHE_DEL You guessed it, they share the payload format too: first the key size (32 bits), and then the key. +REQ_CAS and REQ_CACHE_CAS + First the key size, then the old value size, then the new value size, and + then the key, the old value and the new value. Replies @@ -107,6 +118,7 @@ REP_CACHE_HIT 0x801 REP_CACHE_MISS 0x802 REP_OK 0x803 REP_NOTIN 0x804 +REP_NOMATCH 0x805 ================ ====== @@ -115,14 +127,14 @@ Reply payload formats REP_ERR The payload is a 32-bit error code, according to the table below. -REP_CACHE_MISS and REP_NOTIN +REP_CACHE_MISS, REP_NOTIN and REP_NOMATCH These replies have no payload. REP_CACHE_HIT The first 32 bits are the value size, then the value. REP_OK Depending on the request, this reply does or doesn't have an associated - value. For *REQ_SET** or *REQ_DEL** there is no payload. But for *REQ_GET* - the first 32 bits are the value size, and then the value. + value. For *REQ_SET**, *REQ_DEL** and *REQ_CAS** there is no payload. But + for *REQ_GET* the first 32 bits are the value size, and then the value. Reply error codes diff --git a/libnmdb/Makefile b/libnmdb/Makefile index 39a5d5d..9206f10 100644 --- a/libnmdb/Makefile +++ b/libnmdb/Makefile @@ -1,19 +1,31 @@ -CFLAGS += -std=c99 -Wall -D_XOPEN_SOURCE=500 -O3 -fPIC +ENABLE_TIPC = 1 +ENABLE_TCP = 1 +ENABLE_UDP = 1 + +CFLAGS += -std=c99 -Wall -O3 +ALL_CFLAGS = -D_XOPEN_SOURCE=500 -fPIC $(CFLAGS) +ALL_CFLAGS += -DENABLE_TIPC=$(ENABLE_TIPC) \ + -DENABLE_TCP=$(ENABLE_TCP) \ + -DENABLE_UDP=$(ENABLE_UDP) ifdef DEBUG -CFLAGS += -g -pg -fprofile-arcs -ftest-coverage +ALL_CFLAGS += -g +endif + +ifdef PROFILE +ALL_CFLAGS += -g -pg -fprofile-arcs -ftest-coverage endif ifdef STRICT -CFLAGS += -ansi -pedantic +ALL_CFLAGS += -ansi -pedantic endif # prefix for installing the binaries PREFIX=/usr/local -OBJS = libnmdb.o test1c.o test1d.o test2c.o test2cm.o test2d.o test2dm.o +OBJS = libnmdb.o tcp.o tipc.o udp.o default: all @@ -23,32 +35,11 @@ all: libs libs: libnmdb.so libnmdb.a -libnmdb.so: libnmdb.o - $(CC) $(CFLAGS) libnmdb.o -shared -fPIC -o libnmdb.so - -libnmdb.a: libnmdb.o - $(AR) cr libnmdb.a libnmdb.o - - -tests: test1c test1d test2c test2cm test2d test2dm - -test1c: test1c.o libnmdb.a - $(CC) $(CFLAGS) test1c.o libnmdb.a -o test1c - -test1d: test1d.o libnmdb.a - $(CC) $(CFLAGS) test1d.o libnmdb.a -o test1d - -test2c: test2c.o libnmdb.a - $(CC) $(CFLAGS) test2c.o libnmdb.a -o test2c - -test2cm: test2cm.o libnmdb.a - $(CC) $(CFLAGS) test2cm.o libnmdb.a -o test2cm - -test2d: test2d.o libnmdb.a - $(CC) $(CFLAGS) test2d.o libnmdb.a -o test2d +libnmdb.so: $(OBJS) + $(CC) $(ALL_CFLAGS) -shared -fPIC $(OBJS) -o libnmdb.so -test2dm: test2dm.o libnmdb.a - $(CC) $(CFLAGS) test2dm.o libnmdb.a -o test2dm +libnmdb.a: $(OBJS) + $(AR) cr libnmdb.a $(OBJS) install: libs @@ -65,13 +56,12 @@ install: libs .c.o: - $(CC) $(CFLAGS) -c $< -o $@ + $(CC) $(ALL_CFLAGS) -c $< -o $@ clean: rm -f $(OBJS) libnmdb.so libnmdb.a - rm -f test1c test1d test2c test2cm test2d test2dm - rm -f *.bb *.bbg *.da *.gcov gmon.out + rm -f *.bb *.bbg *.da *.gcov *.gcda *.gcno gmon.out -.PHONY: default all libs tests install clean +.PHONY: default all libs install clean diff --git a/libnmdb/internal.h b/libnmdb/internal.h new file mode 100644 index 0000000..4c2f93c --- /dev/null +++ b/libnmdb/internal.h @@ -0,0 +1,27 @@ + +#ifndef _INTERNAL_H +#define _INTERNAL_H + +/* Different connection types. Used internally to differentiate between TIPC + * and TCP connections in struct nmdb_srv. */ +#define TIPC_CONN 1 +#define TCP_CONN 2 +#define UDP_CONN 3 + +/* The ID code for requests is hardcoded for now, until asynchronous requests + * * are implemented. */ +#define ID_CODE 1 + +/* For a given buffer, how much into it should the generic library code write + * the message contents. */ +#define TIPC_MSG_OFFSET 0 +#define TCP_MSG_OFFSET 4 +#define UDP_MSG_OFFSET 0 + +/* Functions used internally but shared among the different files. */ +int compare_servers(const void *s1, const void *s2); +ssize_t srecv(int fd, unsigned char *buf, size_t count, int flags); +ssize_t ssend(int fd, const unsigned char *buf, size_t count, int flags); + +#endif + diff --git a/libnmdb/libnmdb.3 b/libnmdb/libnmdb.3 index 27306d8..72cb400 100644 --- a/libnmdb/libnmdb.3 +++ b/libnmdb/libnmdb.3 @@ -5,8 +5,10 @@ libnmdb - Library for interacting with a nmdb server .nf .B #include .sp -.BI "nmdb_t *nmdb_init(int " port ");" -.BI "int nmdb_add_server(nmdb_t *" db ", int " port ");" +.BI "nmdb_t *nmdb_init();" +.BI "int nmdb_add_tipc_server(nmdb_t *" db ", int " port ");" +.BI "int nmdb_add_tcp_server(nmdb_t *" db ", const char * " addr ", int " port ");" +.BI "int nmdb_add_udp_server(nmdb_t *" db ", const char * " addr ", int " port ");" .BI "int nmdb_free(nmdb_t *" db ");" .sp .BI "int nmdb_set(nmdb_t *" db "," @@ -32,6 +34,15 @@ libnmdb - Library for interacting with a nmdb server .BI " const unsigned char *" key ", size_t " ksize ");" .BI "int nmdb_cache_del(nmdb_t *" db "," .BI " const unsigned char *" key ", size_t " ksize ");" +.sp +.BI "int nmdb_cas(nmdb_t *" db "," +.BI " const unsigned char *" key " , size_t " ksize "," +.BI " const unsigned char *" oldval ", size_t " ovsize "," +.BI " const unsigned char *" newval ", size_t " nvsize ")" +.BI "int nmdb_cache_cas(nmdb_t *" db "," +.BI " const unsigned char *" key " , size_t " ksize "," +.BI " const unsigned char *" oldval ", size_t " ovsize "," +.BI " const unsigned char *" newval ", size_t " nvsize ")" .fi .SH DESCRIPTION @@ -43,15 +54,20 @@ The first step to access a server is to call to initialize the data. It will return an opaque pointer of .B nmdb_t type, which well be used as the first argument for the rest of the functions -and identifies a connection to the server. It takes a single argument, which -is the TIPC port to use. If you are in doubt, use -1 which will use the -default value. +and identifies a connection to the server. Optionally, you can add more servers to the server pool, using -.BR nmdb_add_server (). +.BR nmdb_add_*_server (). You can add any number of servers, and each time a request is made, one will be selected. Be aware that you should add all the servers before start using -the database. +the database. For +.BR nmdb_add_tipc_server() , +.B nmdb_add_tcp_server() +and +.BR nmdb_add_udp_server() , +if you pass -1 as the port, it will select the default one. They return 1 on +success or 0 on error (or if the specified protocol was not compiled in). + To dispose a connection, use .BR nmdb_free (). @@ -70,14 +86,19 @@ of the keys and values apply: keys can't exceed 64Kb, values can't exceed 64Kb, and the size of a key + the size of it's associated value can't exceed 64Kb. -There are three kinds of operations: +There are four kinds of operations: .IR set , -.I get -and +which sets pairs; +.IR get , +which gets pairs; .IR del , -with their obvious meaning. The normal set and del operations return as soon -as they've been queued on the server for asynchronous completion. Note that in -this case no message is sent to the client when the operation completes. +which removes pairs; +and +.IR cas , +which compares-and-sets values. The normal set and del operations return as +soon as they've been queued on the server for asynchronous completion. Note +that in this case no message is sent to the client when the operation +completes. Each operation has variants, that make it behave in a different way. All three have "cache" variants, that only affect the cache and not the database; and @@ -113,6 +134,12 @@ returns 1 if it was queued successfuly, or < 0 on failure. The cache and synchronous variant return 1 if the key was removed successfuly, 0 if the key was not in the database/cache, or < 0 on failure. +.BR nmdb_cas () +is used to compare-and-swap a key's value. It takes an old value to compare +with the one in the database, and if they match, it sets the key to the given +new value. Returns 2 if the swap was performed, 1 if the values didn't match, +0 if the key was not on in the database/cache, and < 0 on failure. + .SH SEE ALSO .BR nmdb (1), diff --git a/libnmdb/libnmdb.c b/libnmdb/libnmdb.c index ceb7b05..f43dd10 100644 --- a/libnmdb/libnmdb.c +++ b/libnmdb/libnmdb.c @@ -2,183 +2,207 @@ #include /* socket defines */ #include /* socket functions */ #include /* malloc() */ -#include /* tipc stuff */ #include /* uint32_t and friends */ #include /* htonls() and friends */ #include /* memcpy() */ #include /* close() */ -#include - #include "nmdb.h" #include "net-const.h" - -/* The ID code for requests is hardcoded for now, until asynchronous requests - * are implemented. */ -#define ID_CODE 1 +#include "tipc.h" +#include "tcp.h" +#include "udp.h" +#include "internal.h" -/* Create a nmdb_t and set the first server to port. If port is < 0, the - * standard port is used. */ -nmdb_t *nmdb_init(int port) +/* Compare two servers by their connection identifiers. It is used internally + * to keep the server array sorted with qsort(). */ +int compare_servers(const void *s1, const void *s2) { - int fd; - nmdb_t *db; - struct nmdb_srv *server; + struct nmdb_srv *srv1 = (struct nmdb_srv *) s1; + struct nmdb_srv *srv2 = (struct nmdb_srv *) s2; - db = malloc(sizeof(nmdb_t)); - if (db == NULL) { - return NULL; + if (srv1->type != srv2->type) { + if (srv1->type < srv2->type) + return -1; + else + return 1; } - server = malloc(sizeof(struct nmdb_srv)); - if (server == NULL) { - free(db); - return NULL; +#if ENABLE_TIPC + if (srv1->type == TIPC_CONN) { + if (srv1->info.tipc.port < srv2->info.tipc.port) + return -1; + else if (srv1->info.tipc.port == srv2->info.tipc.port) + return 0; + else + return 1; + } +#endif +#if ENABLE_TCP || ENABLE_UDP + if (srv1->type == TCP_CONN || srv1->type == UDP_CONN) { + in_addr_t a1, a2; + a1 = srv1->info.tcp.srvsa.sin_addr.s_addr; + a2 = srv2->info.tcp.srvsa.sin_addr.s_addr; + + if (a1 < a2) { + return -1; + } else if (a1 == a2) { + in_port_t p1, p2; + p1 = srv1->info.tcp.srvsa.sin_port; + p2 = srv2->info.tcp.srvsa.sin_port; + + if (p1 < p2) + return -1; + else if (p1 == p2) + return 0; + else + return 1; + } else { + return 1; + } } +#endif - db->servers = server; - db->nservers = 1; + /* We should never get here */ + return 0; +} - /* the fd is shared among the different servers, because we use - * sendto() directly instead of connecting sockets */ - fd = socket(AF_TIPC, SOCK_RDM, 0); - if (fd < 0) { - free(db->servers); - free(db); - return NULL; - } - db->fd = fd; - if (port < 0) - port = SERVER_INST; +/* Like recv() but either fails, or returns a complete read; if we return less + * than count is because EOF was reached */ +ssize_t srecv(int fd, unsigned char *buf, size_t count, int flags) +{ + ssize_t rv, c; - server->port = port; - server->srvsa.family = AF_TIPC; - server->srvsa.addrtype = TIPC_ADDR_NAMESEQ; - server->srvsa.addr.nameseq.type = SERVER_TYPE; - server->srvsa.addr.nameseq.lower = port; - server->srvsa.addr.nameseq.upper = port; - server->srvsa.scope = TIPC_CLUSTER_SCOPE; - server->srvlen = (socklen_t) sizeof(server->srvsa); + c = 0; - return db; -} + while (c < count) { + rv = recv(fd, buf + c, count - c, flags); -/* Compare two servers, using their ports. It is used internally to keep the - * server array sorted with qsort(). */ -static int compare_servers(const void *s1, const void *s2) -{ - struct nmdb_srv *srv1 = (struct nmdb_srv *) s1; - struct nmdb_srv *srv2 = (struct nmdb_srv *) s2; + if (rv == count) + return count; + else if (rv < 0) + return rv; + else if (rv == 0) + return c; - if (srv1->port < srv2->port) - return -1; - else if (srv1->port == srv2->port) - return 0; - else - return 1; + c += rv; + } + return count; } -/* Add a server to the db connection. Requests will select which server to use - * by hashing the key. */ -int nmdb_add_server(nmdb_t *db, int port) +/* Like srecv() but for send() */ +ssize_t ssend(int fd, const unsigned char *buf, size_t count, int flags) { - struct nmdb_srv *newsrv, *newarray; + ssize_t rv, c; - newarray = realloc(db->servers, - sizeof(struct nmdb_srv) * (db->nservers + 1)); - if (newarray == NULL) { - return 0; - } - db->servers = newarray; - db->nservers++; + c = 0; + + while (c < count) { + rv = send(fd, buf + c, count - c, flags); + + if (rv == count) + return count; + else if (rv < 0) + return rv; + else if (rv == 0) + return c; - newsrv = &(db->servers[db->nservers - 1]); + c += rv; + } + return count; +} - if (port < 0) - port = SERVER_INST; +/* Create a nmdb_t and set the first server to port. If port is < 0, the + * standard port is used. */ +nmdb_t *nmdb_init() +{ + nmdb_t *db; - newsrv->port = port; - newsrv->srvsa.family = AF_TIPC; - newsrv->srvsa.addrtype = TIPC_ADDR_NAMESEQ; - newsrv->srvsa.addr.nameseq.type = SERVER_TYPE; - newsrv->srvsa.addr.nameseq.lower = port; - newsrv->srvsa.addr.nameseq.upper = port; - newsrv->srvsa.scope = TIPC_CLUSTER_SCOPE; - newsrv->srvlen = (socklen_t) sizeof(newsrv->srvsa); + db = malloc(sizeof(nmdb_t)); + if (db == NULL) { + return NULL; + } - /* keep the list sorted by port, so we can do a reliable selection */ - qsort(db->servers, db->nservers, sizeof(struct nmdb_srv), - compare_servers); + db->servers = NULL; + db->nservers = 0; - return 1; + return db; } - /* Frees a nmdb_t structure created with nmdb_init(). */ int nmdb_free(nmdb_t *db) { - close(db->fd); - if (db->servers != NULL) + if (db->servers != NULL) { + int i; + for (i = 0; i < db->nservers; i++) + close(db->servers[i].fd); free(db->servers); + } free(db); return 1; } - -/* Used internally to send a buffer to the given server. */ -static int srv_send(nmdb_t *db, struct nmdb_srv *srv, +/* Used internally to send a buffer to the given server. Calls the appropriate + * sender according to the server protocol. */ +static int srv_send(struct nmdb_srv *srv, const unsigned char *buf, size_t bsize) { - ssize_t rv; - rv = sendto(db->fd, buf, bsize, 0, (struct sockaddr *) &(srv->srvsa), - srv->srvlen); - if (rv <= 0) + if (srv == NULL) return 0; - return 1; -} -/* Used internally to receive a buffer. */ -static ssize_t srv_recv(nmdb_t *db, struct nmdb_srv *srv, - unsigned char *buf, size_t bsize) -{ - ssize_t rv; - rv = recv(db->fd, buf, bsize, 0); - return rv; + switch (srv->type) { + case TIPC_CONN: + return tipc_srv_send(srv, buf, bsize); + case TCP_CONN: + return tcp_srv_send(srv, buf, bsize); + case UDP_CONN: + return udp_srv_send(srv, buf, bsize); + default: + return 0; + } } -/* Used internally to get and parse replies from the server. */ -static uint32_t get_rep(nmdb_t *db, struct nmdb_srv *srv, +static uint32_t get_rep(struct nmdb_srv *srv, unsigned char *buf, size_t bsize, unsigned char **payload, size_t *psize) { - ssize_t t; - uint32_t id, reply; - - t = srv_recv(db, srv, buf, bsize); - if (t < 4 + 4) { + if (srv == NULL) return -1; - } - id = * (uint32_t *) buf; - id = ntohl(id); - reply = * ((uint32_t *) buf + 1); - reply = ntohl(reply); - - if (id != ID_CODE) { - return -1; + switch (srv->type) { + case TIPC_CONN: + return tipc_get_rep(srv, buf, bsize, payload, psize); + case TCP_CONN: + return tcp_get_rep(srv, buf, bsize, payload, psize); + case UDP_CONN: + return udp_get_rep(srv, buf, bsize, payload, psize); + default: + return 0; } +} + +static int srv_get_msg_offset(struct nmdb_srv *srv) +{ + if (srv == NULL) + return 0; - if (payload != NULL) { - *payload = buf + 4 + 4; - *psize = t - 4 - 4; + switch (srv->type) { + case TIPC_CONN: + return TIPC_MSG_OFFSET; + case TCP_CONN: + return TCP_MSG_OFFSET; + case UDP_CONN: + return UDP_MSG_OFFSET; + default: + return 0; } - return reply; } + /* Hash function used internally by select_srv(). See RFC 1071. */ -uint32_t checksum(const unsigned char *buf, size_t bsize) +static uint32_t checksum(const unsigned char *buf, size_t bsize) { uint32_t sum = 0; @@ -202,6 +226,9 @@ static struct nmdb_srv *select_srv(nmdb_t *db, { uint32_t n; + if (db->nservers == 0) + return NULL; + n = checksum(key, ksize) % db->nservers; return &(db->servers[n]); } @@ -211,9 +238,10 @@ static ssize_t do_get(nmdb_t *db, const unsigned char *key, size_t ksize, unsigned char *val, size_t vsize, int impact_db) { + int moff; ssize_t rv, t; unsigned char *buf, *p; - size_t bsize, reqsize, psize; + size_t bsize, reqsize, psize = 0; uint32_t request, reply; struct nmdb_srv *srv; @@ -223,35 +251,38 @@ static ssize_t do_get(nmdb_t *db, request = REQ_CACHE_GET; } + srv = select_srv(db, key, ksize); + moff = srv_get_msg_offset(srv); + /* Use the same buffer for the request and the reply. * Request: 4 bytes ver+id, 4 bytes request code, 4 bytes ksize, * ksize bytes key. * Reply: 4 bytes id, 4 bytes reply code, 4 bytes vsize, * vsize bytes key. * - * We don't know vsize beforehand, but we do know TIPC's max packet is - * 66000. We malloc 70k just in case. + * We don't know vsize beforehand, but we do know our max packet size + * is 64kb. We malloc 68kb just in case. */ - bsize = 70 * 1024; + bsize = 68 * 1024; buf = malloc(bsize); if (buf == NULL) return -1; - * (uint32_t *) buf = htonl( (PROTO_VER << 28) | ID_CODE ); - * ((uint32_t *) buf + 1) = htonl(request); - * ((uint32_t *) buf + 2) = htonl(ksize); - p = buf + 3 * 4; - memcpy(p, key, ksize); + p = buf + moff; + + * (uint32_t *) p = htonl( (PROTO_VER << 28) | ID_CODE ); + * ((uint32_t *) p + 1) = htonl(request); + * ((uint32_t *) p + 2) = htonl(ksize); + memcpy(p + 3 * 4, key, ksize); reqsize = 3 * 4 + ksize; - srv = select_srv(db, key, ksize); - t = srv_send(db, srv, buf, reqsize); + t = srv_send(srv, buf, moff + reqsize); if (t <= 0) { rv = -1; goto exit; } - reply = get_rep(db, srv, buf, bsize, &p, &psize); + reply = get_rep(srv, buf, bsize, &p, &psize); if (reply == REP_CACHE_MISS || reply == REP_NOTIN) { rv = 0; @@ -297,6 +328,7 @@ static int do_set(nmdb_t *db, const unsigned char *key, size_t ksize, const unsigned char *val, size_t vsize, int impact_db, int async) { + int moff; ssize_t rv, t; unsigned char *buf, *p; size_t bsize; @@ -312,34 +344,35 @@ static int do_set(nmdb_t *db, const unsigned char *key, size_t ksize, request = REQ_CACHE_SET; } + srv = select_srv(db, key, ksize); + moff = srv_get_msg_offset(srv); /* Use the same buffer for the request and the reply. * Request: 4 bytes ver+id, 4 bytes request code, 4 bytes ksize, 4 * bytes vsize, ksize bytes key, vsize bytes val. * Reply: 4 bytes id, 4 bytes reply code. */ - bsize = 4 + 4 + 4 + 4 + ksize + vsize; + bsize = moff + 4 + 4 + 4 + 4 + ksize + vsize; buf = malloc(bsize); if (buf == NULL) return -1; - * (uint32_t *) buf = htonl( (PROTO_VER << 28) | ID_CODE ); - * ((uint32_t *) buf + 1) = htonl(request); - * ((uint32_t *) buf + 2) = htonl(ksize); - * ((uint32_t *) buf + 3) = htonl(vsize); - p = buf + 4 * 4; - memcpy(p, key, ksize); - p += ksize; - memcpy(p, val, vsize); + p = buf + moff; - srv = select_srv(db, key, ksize); - t = srv_send(db, srv, buf, bsize); + * (uint32_t *) p = htonl( (PROTO_VER << 28) | ID_CODE ); + * ((uint32_t *) p + 1) = htonl(request); + * ((uint32_t *) p + 2) = htonl(ksize); + * ((uint32_t *) p + 3) = htonl(vsize); + memcpy(p + 4 * 4, key, ksize); + memcpy(p + 4 * 4 + ksize, val, vsize); + + t = srv_send(srv, buf, bsize); if (t <= 0) { rv = -1; goto exit; } - reply = get_rep(db, srv, buf, bsize, NULL, NULL); + reply = get_rep(srv, buf, bsize, NULL, NULL); if (reply == REP_OK) { rv = 1; @@ -375,11 +408,12 @@ int nmdb_cache_set(nmdb_t *db, const unsigned char *key, size_t ksize, -int do_del(nmdb_t *db, const unsigned char *key, size_t ksize, +static int do_del(nmdb_t *db, const unsigned char *key, size_t ksize, int impact_db, int async) { + int moff; ssize_t rv, t; - unsigned char *buf; + unsigned char *buf, *p; size_t bsize; uint32_t request, reply; struct nmdb_srv *srv; @@ -393,30 +427,33 @@ int do_del(nmdb_t *db, const unsigned char *key, size_t ksize, request = REQ_CACHE_DEL; } + srv = select_srv(db, key, ksize); + moff = srv_get_msg_offset(srv); /* Use the same buffer for the request and the reply. * Request: 4 bytes ver+id, 4 bytes request code, 4 bytes ksize, * ksize bytes key. * Reply: 4 bytes id, 4 bytes reply code. */ - bsize = 8 + 4 + ksize; + bsize = moff + 8 + 4 + ksize; buf = malloc(bsize); if (buf == NULL) return -1; - * (uint32_t *) buf = htonl( (PROTO_VER << 28) | ID_CODE ); - * ((uint32_t *) buf + 1) = htonl(request); - * ((uint32_t *) buf + 2) = htonl(ksize); - memcpy(buf + 3 * 4, key, ksize); + p = buf + moff; - srv = select_srv(db, key, ksize); - t = srv_send(db, srv, buf, bsize); + * (uint32_t *) p = htonl( (PROTO_VER << 28) | ID_CODE ); + * ((uint32_t *) p + 1) = htonl(request); + * ((uint32_t *) p + 2) = htonl(ksize); + memcpy(p + 3 * 4, key, ksize); + + t = srv_send(srv, buf, bsize); if (t <= 0) { rv = -1; goto exit; } - reply = get_rep(db, srv, buf, bsize, NULL, NULL); + reply = get_rep(srv, buf, bsize, NULL, NULL); if (reply == REP_OK) { rv = 1; @@ -451,3 +488,90 @@ int nmdb_cache_del(nmdb_t *db, const unsigned char *key, size_t ksize) } +static int do_cas(nmdb_t *db, const unsigned char *key, size_t ksize, + const unsigned char *oldval, size_t ovsize, + const unsigned char *newval, size_t nvsize, + int impact_db) +{ + int moff; + ssize_t rv, t; + unsigned char *buf, *p, *q; + size_t bsize; + uint32_t request, reply; + struct nmdb_srv *srv; + + request = REQ_CACHE_CAS; + if (impact_db) + request = REQ_CAS; + + srv = select_srv(db, key, ksize); + moff = srv_get_msg_offset(srv); + + /* Use the same buffer for the request and the reply. + * Request: 4 bytes ver+id, 4 bytes request code, 4 bytes ksize, 4 + * bytes ovsize, 4 bytes nvsize, ksize bytes key, + * ovsize bytes oldval, nvsize bytes newval. + * Reply: 4 bytes id, 4 bytes reply code. + */ + bsize = moff + 4 + 4 + 4 + 4 + 4 + ksize + ovsize + nvsize; + buf = malloc(bsize); + if (buf == NULL) + return -1; + + p = buf + moff; + + * (uint32_t *) p = htonl( (PROTO_VER << 28) | ID_CODE ); + * ((uint32_t *) p + 1) = htonl(request); + * ((uint32_t *) p + 2) = htonl(ksize); + * ((uint32_t *) p + 3) = htonl(ovsize); + * ((uint32_t *) p + 4) = htonl(nvsize); + q = p + 5 * 4; + memcpy(q, key, ksize); + q += ksize; + memcpy(q, oldval, ovsize); + q += ovsize; + memcpy(q, newval, nvsize); + + srv = select_srv(db, key, ksize); + t = srv_send(srv, buf, bsize); + if (t <= 0) { + rv = -1; + goto exit; + } + + reply = get_rep(srv, buf, bsize, NULL, NULL); + + if (reply == REP_OK) { + rv = 2; + goto exit; + } else if (reply == REP_NOMATCH) { + rv = 1; + goto exit; + } else if (reply == REP_NOTIN) { + rv = 0; + goto exit; + } + + /* REP_ERR or invalid response */ + rv = -1; + +exit: + free(buf); + return rv; + +} + +int nmdb_cas(nmdb_t *db, const unsigned char *key, size_t ksize, + const unsigned char *oldval, size_t ovsize, + const unsigned char *newval, size_t nvsize) +{ + return do_cas(db, key, ksize, oldval, ovsize, newval, nvsize, 1); +} + +int nmdb_cache_cas(nmdb_t *db, const unsigned char *key, size_t ksize, + const unsigned char *oldval, size_t ovsize, + const unsigned char *newval, size_t nvsize) +{ + return do_cas(db, key, ksize, oldval, ovsize, newval, nvsize, 0); +} + diff --git a/libnmdb/net-const.h b/libnmdb/net-const.h index a382490..bdaa29d 100644 --- a/libnmdb/net-const.h +++ b/libnmdb/net-const.h @@ -7,9 +7,17 @@ * Isolated so it's shared between the server and the library code. */ -/* TIPC server type and instance -- Hardcoded for now. */ -#define SERVER_TYPE 26001 -#define SERVER_INST 10 +/* TIPC server type (hardcoded) and default instance. */ +#define TIPC_SERVER_TYPE 26001 +#define TIPC_SERVER_INST 10 + +/* TCP default listen address and port. */ +#define TCP_SERVER_ADDR "0.0.0.0" +#define TCP_SERVER_PORT 26010 + +/* UDP default listen address and port. */ +#define UDP_SERVER_ADDR "0.0.0.0" +#define UDP_SERVER_PORT 26010 /* Protocol version, for checking in the network header. */ #define PROTO_VER 1 @@ -23,6 +31,8 @@ #define REQ_DEL_SYNC 0x106 #define REQ_SET_ASYNC 0x107 #define REQ_DEL_ASYNC 0x108 +#define REQ_CACHE_CAS 0x109 +#define REQ_CAS 0x110 /* Network replies (different namespace from requests) */ #define REP_ERR 0x800 @@ -30,6 +40,7 @@ #define REP_CACHE_MISS 0x802 #define REP_OK 0x803 #define REP_NOTIN 0x804 +#define REP_NOMATCH 0x805 /* Network error replies */ #define ERR_VER 0x101 /* Version mismatch */ @@ -39,5 +50,6 @@ #define ERR_MEM 0x105 /* Memory allocation error */ #define ERR_DB 0x106 /* Database error */ + #endif diff --git a/libnmdb/nmdb.h b/libnmdb/nmdb.h index 8d0a294..9184e8c 100644 --- a/libnmdb/nmdb.h +++ b/libnmdb/nmdb.h @@ -5,21 +5,38 @@ #include /* socket defines */ #include /* socklen_t */ #include /* struct sockaddr_tipc */ +#include /* struct sockaddr_in */ + struct nmdb_srv { - int port; - struct sockaddr_tipc srvsa; - socklen_t srvlen; + int fd; + int type; + union { + struct { + unsigned int port; + struct sockaddr_tipc srvsa; + socklen_t srvlen; + } tipc; + struct { + struct sockaddr_in srvsa; + socklen_t srvlen; + } tcp; + struct { + struct sockaddr_in srvsa; + socklen_t srvlen; + } udp; + } info; }; typedef struct nmdb_t { - int fd; unsigned int nservers; struct nmdb_srv *servers; } nmdb_t; -nmdb_t *nmdb_init(int port); -int nmdb_add_server(nmdb_t *db, int port); +nmdb_t *nmdb_init(); +int nmdb_add_tipc_server(nmdb_t *db, int port); +int nmdb_add_tcp_server(nmdb_t *db, const char *addr, int port); +int nmdb_add_udp_server(nmdb_t *db, const char *addr, int port); int nmdb_free(nmdb_t *db); ssize_t nmdb_get(nmdb_t *db, const unsigned char *key, size_t ksize, @@ -29,14 +46,22 @@ ssize_t nmdb_cache_get(nmdb_t *db, const unsigned char *key, size_t ksize, int nmdb_set(nmdb_t *db, const unsigned char *key, size_t ksize, const unsigned char *val, size_t vsize); -int nmdb_set_async(nmdb_t *db, const unsigned char *key, size_t ksize, +int nmdb_set_sync(nmdb_t *db, const unsigned char *key, size_t ksize, const unsigned char *val, size_t vsize); int nmdb_cache_set(nmdb_t *db, const unsigned char *key, size_t ksize, const unsigned char *val, size_t vsize); int nmdb_del(nmdb_t *db, const unsigned char *key, size_t ksize); -int nmdb_del_async(nmdb_t *db, const unsigned char *key, size_t ksize); +int nmdb_del_sync(nmdb_t *db, const unsigned char *key, size_t ksize); int nmdb_cache_del(nmdb_t *db, const unsigned char *key, size_t ksize); +int nmdb_cas(nmdb_t *db, const unsigned char *key, size_t ksize, + const unsigned char *oldval, size_t ovsize, + const unsigned char *newval, size_t nvsize); +int nmdb_cache_cas(nmdb_t *db, const unsigned char *key, size_t ksize, + const unsigned char *oldval, size_t ovsize, + const unsigned char *newval, size_t nvsize); + + #endif diff --git a/libnmdb/tcp.c b/libnmdb/tcp.c new file mode 100644 index 0000000..6e701ed --- /dev/null +++ b/libnmdb/tcp.c @@ -0,0 +1,209 @@ + +#if ENABLE_TCP + +#include /* socket defines */ +#include /* socket functions */ +#include /* malloc() */ +#include /* uint32_t and friends */ +#include /* htonls() and friends */ +#include /* memcpy() */ +#include /* close() */ + +#include /* TCP stuff */ +#include /* gethostbyname() */ + +#include "nmdb.h" +#include "net-const.h" +#include "internal.h" + + +/* Used internally to really add the server once we have an IP address. */ +static int add_tcp_server_addr(nmdb_t *db, in_addr_t *inetaddr, int port) +{ + int rv, fd; + struct nmdb_srv *newsrv, *newarray; + + fd = socket(AF_INET, SOCK_STREAM, 0); + if (fd < 0) + return 0; + + newarray = realloc(db->servers, + sizeof(struct nmdb_srv) * (db->nservers + 1)); + if (newarray == NULL) { + close(fd); + return 0; + } + + db->servers = newarray; + db->nservers++; + + if (port < 0) + port = TCP_SERVER_PORT; + + newsrv = &(db->servers[db->nservers - 1]); + + newsrv->fd = fd; + newsrv->info.tcp.srvsa.sin_family = AF_INET; + newsrv->info.tcp.srvsa.sin_port = htons(port); + newsrv->info.tcp.srvsa.sin_addr.s_addr = *inetaddr; + + rv = connect(fd, (struct sockaddr *) &(newsrv->info.tcp.srvsa), + sizeof(newsrv->info.tcp.srvsa)); + if (rv < 0) + goto error_exit; + + /* Disable Nagle algorithm because we often send small packets. Huge + * gain in performance. */ + rv = 1; + if (setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &rv, sizeof(rv)) < 0 ) + goto error_exit; + + newsrv->type = TCP_CONN; + + /* keep the list sorted by port, so we can do a reliable selection */ + qsort(db->servers, db->nservers, sizeof(struct nmdb_srv), + compare_servers); + + return 1; + +error_exit: + close(fd); + newarray = realloc(db->servers, + sizeof(struct nmdb_srv) * (db->nservers - 1)); + if (newarray == NULL) { + db->servers = NULL; + db->nservers = 0; + return 0; + } + + db->servers = newarray; + db->nservers -= 1; + + return 0; +} + +/* Same as nmdb_add_tipc_server() but for TCP connections. */ +int nmdb_add_tcp_server(nmdb_t *db, const char *addr, int port) +{ + int rv; + struct hostent *he; + struct in_addr ia; + + /* We try to resolve and then pass it to add_tcp_server_addr(). */ + rv = inet_pton(AF_INET, addr, &ia); + if (rv <= 0) { + he = gethostbyname(addr); + if (he == NULL) + return 0; + + ia.s_addr = *( (in_addr_t *) (he->h_addr_list[0]) ); + } + + return add_tcp_server_addr(db, &(ia.s_addr), port); +} + +int tcp_srv_send(struct nmdb_srv *srv, + unsigned char *buf, size_t bsize) +{ + ssize_t rv; + uint32_t len; + + len = htonl(bsize); + memcpy(buf, (const void *) &len, 4); + + rv = ssend(srv->fd, buf, bsize, 0); + if (rv != bsize) + return 0; + return 1; +} + +static ssize_t recv_msg(int fd, unsigned char *buf, size_t bsize) +{ + ssize_t rv, t; + uint32_t msgsize; + + rv = recv(fd, buf, bsize, 0); + if (rv <= 0) + return rv; + + if (rv < 4) { + t = srecv(fd, buf + rv, 4 - rv, 0); + if (t <= 0) { + return t; + } + + rv = rv + t; + } + + msgsize = * ((uint32_t *) buf); + msgsize = ntohl(msgsize); + + if (msgsize > bsize) + return -1; + + if (rv < msgsize) { + t = srecv(fd, buf + rv, msgsize - rv, 0); + if (t <= 0) { + return t; + } + + rv = rv + t; + } + + return rv; +} + + +/* Used internally to get and parse replies from the server. */ +uint32_t tcp_get_rep(struct nmdb_srv *srv, + unsigned char *buf, size_t bsize, + unsigned char **payload, size_t *psize) +{ + ssize_t rv; + uint32_t id, reply; + + rv = recv_msg(srv->fd, buf, bsize); + if (rv <= 0) + return -1; + + id = * ((uint32_t *) buf + 1); + id = ntohl(id); + reply = * ((uint32_t *) buf + 2); + reply = ntohl(reply); + + if (id != ID_CODE) { + return -1; + } + + if (payload != NULL) { + *payload = buf + 4 + 4 + 4; + *psize = rv - 4 - 4 - 4; + } + return reply; +} + +#else +/* Stubs to use when TCP is not enabled. */ + +#include "nmdb.h" + +int nmdb_add_tcp_server(nmdb_t *db, const char *addr, int port) +{ + return 0; +} + +int tcp_srv_send(struct nmdb_srv *srv, + const unsigned char *buf, size_t bsize) +{ + return 0; +} + +uint32_t tcp_get_rep(struct nmdb_srv *srv, + unsigned char *buf, size_t bsize, + unsigned char **payload, size_t *psize) +{ + return -1; +} + +#endif /* ENABLE_TCP */ + diff --git a/libnmdb/tcp.h b/libnmdb/tcp.h new file mode 100644 index 0000000..4252353 --- /dev/null +++ b/libnmdb/tcp.h @@ -0,0 +1,12 @@ + +#ifndef _TCP_H +#define _TCP_H + +int tcp_srv_send(struct nmdb_srv *srv, + const unsigned char *buf, size_t bsize); +uint32_t tcp_get_rep(struct nmdb_srv *srv, + unsigned char *buf, size_t bsize, + unsigned char **payload, size_t *psize); + +#endif + diff --git a/libnmdb/test1d.c b/libnmdb/test1d.c deleted file mode 100644 index 51e4f83..0000000 --- a/libnmdb/test1d.c +++ /dev/null @@ -1,84 +0,0 @@ - -#include -#include -#include -#include -#include - -#include "nmdb.h" -#include "timer.h" - - -int main(int argc, char **argv) -{ - int i, r, times; - unsigned char *key, *val, *gval; - size_t ksize, vsize; - unsigned long elapsed, misses = 0; - nmdb_t *db; - - if (argc != 4) { - printf("Usage: test1 TIMES KEY VAL\n"); - return 1; - } - - times = atoi(argv[1]); - key = (unsigned char *) argv[2]; - ksize = strlen((char *) key); - val = (unsigned char *) argv[3]; - vsize = strlen((char *) val); - - db = nmdb_init(-1); - if (db == NULL) { - perror("nmdb_init() failed"); - return 1; - } - - printf("set... "); - timer_start(); - for (i = 0; i < times; i++) { - r = nmdb_set(db, key, ksize, val, vsize); - if (r < 0) { - perror("Set"); - return 1; - } - } - elapsed = timer_stop(); - printf("%lu\n", elapsed); - - gval = malloc(70 * 1024); - printf("get... "); - timer_start(); - for (i = 0; i < times; i++) { - r = nmdb_get(db, key, ksize, gval, vsize); - if (r < 0) { - perror("Get"); - return 1; - } else if (r == 0) { - misses++; - } - } - elapsed = timer_stop(); - printf("%lu\n", elapsed); - free(gval); - - printf("get misses: %ld\n", misses); - - printf("del... "); - timer_start(); - for (i = 0; i < times; i++) { - r = nmdb_del(db, key, ksize); - if (r < 0) { - perror("Del"); - return 1; - } - } - elapsed = timer_stop(); - printf("%lu\n", elapsed); - - - nmdb_free(db); - - return 0; -} - diff --git a/libnmdb/test2cm.c b/libnmdb/test2cm.c deleted file mode 100644 index 222ff3f..0000000 --- a/libnmdb/test2cm.c +++ /dev/null @@ -1,114 +0,0 @@ - -#include -#include -#include -#include -#include - -#include "nmdb.h" -#include "timer.h" - - -int main(int argc, char **argv) -{ - int i, r, times; - unsigned char *key, *val, *gval; - size_t ksize, vsize; - unsigned long s_elapsed, g_elapsed, d_elapsed, misses = 0; - nmdb_t *db; - - if (argc != 4) { - printf("Usage: test2 TIMES KSIZE VSIZE\n"); - return 1; - } - - times = atoi(argv[1]); - ksize = atoi(argv[2]); - vsize = atoi(argv[3]); - if (times < 1) { - printf("Error: TIMES must be >= 1\n"); - return 1; - } - if (ksize < sizeof(int) || vsize < sizeof(int)) { - printf("Error: KSIZE and VSIZE must be >= sizeof(int)\n"); - return 1; - } - - key = malloc(ksize); - memset(key, 0, ksize); - val = malloc(vsize); - memset(val, 0, vsize); - - if (key == NULL || val == NULL) { - perror("Error: malloc()"); - return 1; - } - - db = nmdb_init(-1); - if (db == NULL) { - perror("nmdb_init() failed"); - return 1; - } - - if (!nmdb_add_server(db, 11) || - !nmdb_add_server(db, 12) || - !nmdb_add_server(db, 13)) { - perror("nmdb_add_server() failed"); - return 1; - } - - timer_start(); - for (i = 0; i < times; i++) { - * (int *) key = i; - * (int *) val = i; - r = nmdb_cache_set(db, key, ksize, val, vsize); - if (r < 0) { - perror("Set"); - return 1; - } - } - s_elapsed = timer_stop(); - - memset(key, 0, ksize); - gval = malloc(70 * 1024); - timer_start(); - for (i = 0; i < times; i++) { - * (int *) key = i; - r = nmdb_cache_get(db, key, ksize, gval, vsize); - if (r < 0) { - perror("Get"); - return 1; - } else if (r == 0) { - misses++; - continue; - } - * (int *) val = i; - if (memcmp((void *) val, (void *) gval, vsize) != 0) { - printf("Values differ for key %s: %s - %s\n", - key, val, gval); - printf("i: %d\n", i); - return 1; - } - } - g_elapsed = timer_stop(); - free(gval); - free(val); - - timer_start(); - for (i = 0; i < times; i++) { - * (int *) key = i; - r = nmdb_cache_del(db, key, ksize); - if (r < 0) { - perror("Del"); - return 1; - } - } - d_elapsed = timer_stop(); - printf("%lu %lu %lu %lu\n", s_elapsed, g_elapsed, d_elapsed, misses); - - free(key); - nmdb_free(db); - - return 0; -} - diff --git a/libnmdb/test2dm.c b/libnmdb/test2dm.c deleted file mode 100644 index 0ed89fd..0000000 --- a/libnmdb/test2dm.c +++ /dev/null @@ -1,106 +0,0 @@ - -#include -#include -#include -#include -#include - -#include "nmdb.h" -#include "timer.h" - - -int main(int argc, char **argv) -{ - int i, r, times; - unsigned char *key, *val; - size_t ksize, vsize; - unsigned long s_elapsed, g_elapsed, d_elapsed, misses = 0; - nmdb_t *db; - - if (argc != 4) { - printf("Usage: test2 TIMES KSIZE VSIZE\n"); - return 1; - } - - times = atoi(argv[1]); - ksize = atoi(argv[2]); - vsize = atoi(argv[3]); - if (times < 1) { - printf("Error: TIMES must be >= 1\n"); - return 1; - } - if (ksize < sizeof(int) || vsize < sizeof(int)) { - printf("Error: KSIZE and VSIZE must be >= sizeof(int)\n"); - return 1; - } - - key = malloc(ksize); - memset(key, 0, ksize); - val = malloc(vsize); - memset(val, 0, vsize); - - if (key == NULL || val == NULL) { - perror("Error: malloc()"); - return 1; - } - - db = nmdb_init(-1); - if (db == NULL) { - perror("nmdb_init() failed"); - return 1; - } - - if (!nmdb_add_server(db, 11) || - !nmdb_add_server(db, 12) || - !nmdb_add_server(db, 13)) { - perror("nmdb_add_server() failed"); - return 1; - } - - timer_start(); - for (i = 0; i < times; i++) { - * (int *) key = i; - * (int *) val = i; - r = nmdb_set(db, key, ksize, val, vsize); - if (r < 0) { - perror("Set"); - return 1; - } - } - s_elapsed = timer_stop(); - - memset(key, 0, ksize); - free(val); - val = malloc(70 * 1024); - timer_start(); - for (i = 0; i < times; i++) { - * (int *) key = i; - r = nmdb_get(db, key, ksize, val, vsize); - if (r < 0) { - perror("Get"); - return 1; - } else if (r == 0) { - misses++; - } - } - g_elapsed = timer_stop(); - free(val); - - timer_start(); - for (i = 0; i < times; i++) { - * (int *) key = i; - r = nmdb_del(db, key, ksize); - if (r < 0) { - perror("Del"); - return 1; - } - } - d_elapsed = timer_stop(); - printf("%lu %lu %lu %lu\n", s_elapsed, g_elapsed, d_elapsed, misses); - - free(key); - nmdb_free(db); - - return 0; -} - diff --git a/libnmdb/test_functions.sh b/libnmdb/test_functions.sh deleted file mode 100644 index d8e5c97..0000000 --- a/libnmdb/test_functions.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash - -# Useful functions for performing automated performance testing. - - -# run test2[c|d] varying KSIZE and VSIZE (which are equal). -# $1 == "c" or "d" depending on which test2 to run -# $2 == TIMES to pass to test2 -function var_sizes { - if [ "$1" == "" ] && [ "$2" == ""]; then - return; - fi; - - for i in 8 16 64 128 512 1024 2048 5120 8192 10240 20480 32768; do - for j in 1 2 3; do - echo $i `./test2$1 $2 $i $i`; - done; - done; -} - - -# run test2[c|d] varying TIMES. -# $1 == "c" or "d" depending on which test2 to run -# $2 == KSIZE and VSIZE to pass to test2 -function var_times { - if [ "$1" == "" ] && [ "$2" == ""]; then - return; - fi; - - for i in `seq 500 500 30000`; do - for j in 1 2 3; do - echo $i `./test2$1 $i $2 $2`; - done; - done; -} - - -# run test2[c|d] and return the amount of operations per second -# $1 == "c" or "d" depending on which test2 to run -# $2 == list of TIMES to use (eg. "1000 2000 5000") -# $3 == KSIZE and VSIZE -function ops_per_sec { - if [ "$1" == "" ] && [ "$2" == ""] && [ "$3" == ""]; then - return; - fi; - - for i in $2; do - for j in 1 2 3; do - OUT=`./test2$1 $i $3 $3`; - ST=`echo $OUT | cut -d ' ' -f 1`; - GT=`echo $OUT | cut -d ' ' -f 2`; - DT=`echo $OUT | cut -d ' ' -f 3`; - MS=`echo $OUT | cut -d ' ' -f 4`; - - SO=$[ $ST / $i ]; - GO=$[ $GT / $i ]; - DO=$[ $DT / $i ]; - - echo $i $SO $GO $DO $MS; - done; - done; -} - - -# -# sample invocations -# -# srv:X means the server is running with -c X - -# var_sizes c 5000 > var_sizes_c_5000_srv:1 -# var_sizes c 5000 > var_sizes_c_5000_srv:128 -# ops_per_sec c "`seq 2000 1000 10000`" 32 > ops_per_sec_c_32_srv:1 -# ops_per_sec c "`seq 2000 1000 10000`" 32 > ops_per_sec_c_32_srv:128 -# ops_per_sec d "`seq 2000 1000 10000`" 32 > ops_per_sec_d_32_srv:1 -# ops_per_sec d "`seq 2000 1000 10000`" 32 > ops_per_sec_d_32_srv:128 -# var_times c 32 > var_times_c_32_srv:1 -# var_times c 32 > var_times_c_32_srv:16 -# var_times c 32 > var_times_c_32_srv:126 - - diff --git a/libnmdb/tipc.c b/libnmdb/tipc.c new file mode 100644 index 0000000..cea13ff --- /dev/null +++ b/libnmdb/tipc.c @@ -0,0 +1,129 @@ + +#if ENABLE_TIPC + +#include /* socket defines */ +#include /* socket functions */ +#include /* malloc() */ +#include /* uint32_t and friends */ +#include /* htonls() and friends */ +#include /* memcpy() */ +#include /* close() */ + +#include /* TIPC stuff */ + +#include "nmdb.h" +#include "net-const.h" +#include "internal.h" + + +/* Add a TIPC server to the db connection. Requests will select which server + * to use by hashing the key. */ +int nmdb_add_tipc_server(nmdb_t *db, int port) +{ + int fd; + struct nmdb_srv *newsrv, *newarray; + + fd = socket(AF_TIPC, SOCK_RDM, 0); + if (fd < 0) + return 0; + + newarray = realloc(db->servers, + sizeof(struct nmdb_srv) * (db->nservers + 1)); + if (newarray == NULL) { + close(fd); + return 0; + } + + db->servers = newarray; + db->nservers++; + + if (port < 0) + port = TIPC_SERVER_INST; + + newsrv = &(db->servers[db->nservers - 1]); + + newsrv->fd = fd; + newsrv->info.tipc.port = port; + newsrv->info.tipc.srvsa.family = AF_TIPC; + newsrv->info.tipc.srvsa.addrtype = TIPC_ADDR_NAMESEQ; + newsrv->info.tipc.srvsa.addr.nameseq.type = TIPC_SERVER_TYPE; + newsrv->info.tipc.srvsa.addr.nameseq.lower = port; + newsrv->info.tipc.srvsa.addr.nameseq.upper = port; + newsrv->info.tipc.srvsa.scope = TIPC_CLUSTER_SCOPE; + newsrv->info.tipc.srvlen = (socklen_t) sizeof(newsrv->info.tipc.srvsa); + + newsrv->type = TIPC_CONN; + + /* keep the list sorted by port, so we can do a reliable selection */ + qsort(db->servers, db->nservers, sizeof(struct nmdb_srv), + compare_servers); + + return 1; +} + +int tipc_srv_send(struct nmdb_srv *srv, + const unsigned char *buf, size_t bsize) +{ + ssize_t rv; + rv = sendto(srv->fd, buf, bsize, 0, + (struct sockaddr *) &(srv->info.tipc.srvsa), + srv->info.tipc.srvlen); + if (rv <= 0) + return 0; + return 1; +} + +/* Used internally to get and parse replies from the server. */ +uint32_t tipc_get_rep(struct nmdb_srv *srv, + unsigned char *buf, size_t bsize, + unsigned char **payload, size_t *psize) +{ + ssize_t rv; + uint32_t id, reply; + + rv = recv(srv->fd, buf, bsize, 0); + if (rv < 4 + 4) { + return -1; + } + + id = * (uint32_t *) buf; + id = ntohl(id); + reply = * ((uint32_t *) buf + 1); + reply = ntohl(reply); + + if (id != ID_CODE) { + return -1; + } + + if (payload != NULL) { + *payload = buf + 4 + 4; + *psize = rv - 4 - 4; + } + return reply; +} + +#else +/* Stubs to use when TIPC is not enabled. */ + +#include "nmdb.h" + +int nmdb_add_tipc_server(nmdb_t *db, int port) +{ + return 0; +} + +int tipc_srv_send(struct nmdb_srv *srv, + const unsigned char *buf, size_t bsize) +{ + return 0; +} + +uint32_t tipc_get_rep(struct nmdb_srv *srv, + unsigned char *buf, size_t bsize, + unsigned char **payload, size_t *psize) +{ + return -1; +} + +#endif /* ENABLE_TIPC */ + diff --git a/libnmdb/tipc.h b/libnmdb/tipc.h new file mode 100644 index 0000000..82b8cb2 --- /dev/null +++ b/libnmdb/tipc.h @@ -0,0 +1,12 @@ + +#ifndef _TIPC_H +#define _TIPC_H + +int tipc_srv_send(struct nmdb_srv *srv, + const unsigned char *buf, size_t bsize); +uint32_t tipc_get_rep(struct nmdb_srv *srv, + unsigned char *buf, size_t bsize, + unsigned char **payload, size_t *psize); + +#endif + diff --git a/libnmdb/udp.c b/libnmdb/udp.c new file mode 100644 index 0000000..5ece6e8 --- /dev/null +++ b/libnmdb/udp.c @@ -0,0 +1,145 @@ + +#if ENABLE_UDP + +#include /* socket defines */ +#include /* socket functions */ +#include /* malloc() */ +#include /* uint32_t and friends */ +#include /* htonls() and friends */ +#include /* memcpy() */ +#include /* close() */ + +#include /* UDP stuff */ +#include /* gethostbyname() */ + +#include "nmdb.h" +#include "net-const.h" +#include "internal.h" + + +/* Used internally to really add the server once we have an IP address. */ +static int add_udp_server_addr(nmdb_t *db, in_addr_t *inetaddr, int port) +{ + int fd; + struct nmdb_srv *newsrv, *newarray; + + fd = socket(AF_INET, SOCK_DGRAM, 0); + if (fd < 0) + return 0; + + newarray = realloc(db->servers, + sizeof(struct nmdb_srv) * (db->nservers + 1)); + if (newarray == NULL) { + close(fd); + return 0; + } + + db->servers = newarray; + db->nservers++; + + if (port < 0) + port = UDP_SERVER_PORT; + + newsrv = &(db->servers[db->nservers - 1]); + + newsrv->fd = fd; + newsrv->info.udp.srvsa.sin_family = AF_INET; + newsrv->info.udp.srvsa.sin_port = htons(port); + newsrv->info.udp.srvsa.sin_addr.s_addr = *inetaddr; + newsrv->info.udp.srvlen = sizeof(struct sockaddr_in); + + newsrv->type = UDP_CONN; + + /* keep the list sorted by port, so we can do a reliable selection */ + qsort(db->servers, db->nservers, sizeof(struct nmdb_srv), + compare_servers); + + return 1; +} + +/* Same as nmdb_add_tcp_server() but for UDP. */ +int nmdb_add_udp_server(nmdb_t *db, const char *addr, int port) +{ + int rv; + struct hostent *he; + struct in_addr ia; + + /* We try to resolve and then pass it to add_udp_server_addr(). */ + rv = inet_pton(AF_INET, addr, &ia); + if (rv <= 0) { + he = gethostbyname(addr); + if (he == NULL) + return 0; + + ia.s_addr = *( (in_addr_t *) (he->h_addr_list[0]) ); + } + + return add_udp_server_addr(db, &(ia.s_addr), port); +} + +int udp_srv_send(struct nmdb_srv *srv, + const unsigned char *buf, size_t bsize) +{ + ssize_t rv; + rv = sendto(srv->fd, buf, bsize, 0, + (struct sockaddr *) &(srv->info.udp.srvsa), + srv->info.udp.srvlen); + if (rv <= 0) + return 0; + return 1; +} + +/* Used internally to get and parse replies from the server. */ +uint32_t udp_get_rep(struct nmdb_srv *srv, + unsigned char *buf, size_t bsize, + unsigned char **payload, size_t *psize) +{ + ssize_t rv; + uint32_t id, reply; + + rv = recv(srv->fd, buf, bsize, 0); + if (rv < 4 + 4) { + return -1; + } + + id = * (uint32_t *) buf; + id = ntohl(id); + reply = * ((uint32_t *) buf + 1); + reply = ntohl(reply); + + if (id != ID_CODE) { + return -1; + } + + if (payload != NULL) { + *payload = buf + 4 + 4; + *psize = rv - 4 - 4; + } + return reply; +} + +#else +/* Stubs to use when UDP is not enabled. */ + +#include "nmdb.h" + +int nmdb_add_udp_server(nmdb_t *db, int port) +{ + return 0; +} + +int udp_srv_send(struct nmdb_srv *srv, + const unsigned char *buf, size_t bsize) +{ + return 0; +} + +uint32_t udp_get_rep(struct nmdb_srv *srv, + unsigned char *buf, size_t bsize, + unsigned char **payload, size_t *psize) +{ + return -1; +} + +#endif /* ENABLE_UDP */ + diff --git a/libnmdb/udp.h b/libnmdb/udp.h new file mode 100644 index 0000000..7515941 --- /dev/null +++ b/libnmdb/udp.h @@ -0,0 +1,12 @@ + +#ifndef _UDP_H +#define _UDP_H + +int udp_srv_send(struct nmdb_srv *srv, + const unsigned char *buf, size_t bsize); +uint32_t udp_get_rep(struct nmdb_srv *srv, + unsigned char *buf, size_t bsize, + unsigned char **payload, size_t *psize); + +#endif + diff --git a/nmdb/Makefile b/nmdb/Makefile index fe76217..5c58c5a 100644 --- a/nmdb/Makefile +++ b/nmdb/Makefile @@ -1,29 +1,60 @@ -CFLAGS += -std=c99 -Wall -D_XOPEN_SOURCE=500 -O3 +ENABLE_TIPC = 1 +ENABLE_TCP = 1 +ENABLE_UDP = 1 + +CFLAGS += -std=c99 -Wall -O3 +ALL_CFLAGS = -D_XOPEN_SOURCE=500 $(CFLAGS) +ALL_CFLAGS += -DENABLE_TIPC=$(ENABLE_TIPC) \ + -DENABLE_TCP=$(ENABLE_TCP) \ + -DENABLE_UDP=$(ENABLE_UDP) ifdef DEBUG -CFLAGS += -g -pg -fprofile-arcs -ftest-coverage +ALL_CFLAGS += -g +endif + +ifdef PROFILE +ALL_CFLAGS += -g -pg -fprofile-arcs -ftest-coverage endif ifdef STRICT -CFLAGS += -ansi -pedantic +ALL_CFLAGS += -ansi -pedantic endif # prefix for installing the binaries PREFIX=/usr/local -OBJS = be-qdbm.o cache.o db.o queue.o net.o tipc.o main.o +OBJS = be-qdbm.o cache.o db.o queue.o net.o parse.o main.o + +ifeq ($(ENABLE_TIPC), 1) + OBJS += tipc.o +else + OBJS += tipc-stub.o +endif + +ifeq ($(ENABLE_TCP), 1) + OBJS += tcp.o +else + OBJS += tcp-stub.o +endif + +ifeq ($(ENABLE_UDP), 1) + OBJS += udp.o +else + OBJS += udp-stub.o +endif + default: all all: nmdb nmdb: $(OBJS) - $(CC) $(CFLAGS) $(OBJS) -levent -lpthread -lrt -lqdbm -o nmdb + $(CC) $(ALL_CFLAGS) $(OBJS) -levent -lpthread -lrt -lqdbm -o nmdb .c.o: - $(CC) $(CFLAGS) -c $< -o $@ + $(CC) $(ALL_CFLAGS) -c $< -o $@ install: all install -d $(PREFIX)/bin @@ -33,7 +64,7 @@ install: all clean: rm -f $(OBJS) nmdb - rm -f *.bb *.bbg *.da *.gcov gmon.out + rm -f *.bb *.bbg *.da *.gcov *.gcda *.gcno gmon.out .PHONY: default all clean diff --git a/nmdb/cache.c b/nmdb/cache.c index 27f426f..59538b3 100644 --- a/nmdb/cache.c +++ b/nmdb/cache.c @@ -307,3 +307,52 @@ exit: return rv; } + +/* Performs a cache compare-and-swap. + * Returns -2 if there was an error, -1 if the key is not in the cache, 0 if + * the old value does not match, and 1 if the CAS was successful. */ +int cache_cas(struct cache *cd, const unsigned char *key, size_t ksize, + const unsigned char *oldval, size_t ovsize, + const unsigned char *newval, size_t nvsize) +{ + int rv = 1; + uint32_t h = 0; + struct cache_chain *c; + struct cache_entry *e; + unsigned char *buf; + + h = hash(key, ksize) % cd->hashlen; + c = cd->table + h; + + e = find_in_chain(c, key, ksize); + + if (e == NULL) { + rv = -1; + goto exit; + } + + if (e->vsize != ovsize) { + rv = 0; + goto exit; + } + + if (memcmp(e->val, oldval, ovsize) != 0) { + rv = 0; + goto exit; + } + + buf = malloc(nvsize); + if (buf == NULL) { + rv = -2; + goto exit; + } + + memcpy(buf, newval, nvsize); + free(e->val); + e->val = buf; + e->vsize = nvsize; + +exit: + return rv; +} + diff --git a/nmdb/cache.h b/nmdb/cache.h index fd31900..66c705d 100644 --- a/nmdb/cache.h +++ b/nmdb/cache.h @@ -44,6 +44,9 @@ int cache_get(struct cache *cd, const unsigned char *key, size_t ksize, int cache_set(struct cache *cd, const unsigned char *k, size_t ksize, const unsigned char *v, size_t vsize); int cache_del(struct cache *cd, const unsigned char *key, size_t ksize); +int cache_cas(struct cache *cd, const unsigned char *key, size_t ksize, + const unsigned char *oldval, size_t ovsize, + const unsigned char *newval, size_t nvsize); #endif diff --git a/nmdb/common.h b/nmdb/common.h index 81a220d..2d6cf56 100644 --- a/nmdb/common.h +++ b/nmdb/common.h @@ -6,27 +6,33 @@ /* The cache table */ #include "cache.h" -struct cache *cache_table; +extern struct cache *cache_table; /* The queue for database operations */ #include "queue.h" -struct queue *op_queue; +extern struct queue *op_queue; /* Settings */ -struct { +struct settings { int tipc_lower; int tipc_upper; + char *tcp_addr; + int tcp_port; + char *udp_addr; + int udp_port; int numobjs; int foreground; int passive; char *dbname; -} settings; +}; +extern struct settings settings; /* Statistics */ -struct { +struct stats { unsigned long net_version_mismatch; unsigned long net_broken_req; unsigned long net_unk_req; -} stats; +}; +extern struct stats stats; #endif diff --git a/nmdb/db.c b/nmdb/db.c index 0589bc2..defc881 100644 --- a/nmdb/db.c +++ b/nmdb/db.c @@ -3,12 +3,14 @@ #include /* nanosleep() */ #include /* ETIMEDOUT */ #include /* perror() */ +#include /* memcmp() */ #include "common.h" #include "db.h" #include "be.h" #include "queue.h" #include "net-const.h" +#include "req.h" static void *db_loop(void *arg); @@ -99,43 +101,77 @@ static void process_op(db_t *db, struct queue_entry *e) if (e->operation == REQ_SET_SYNC) { rv = db_set(db, e->key, e->ksize, e->val, e->vsize); if (!rv) { - tipc_reply_err(e->req, ERR_DB); + e->req->reply_err(e->req, ERR_DB); return; } - tipc_reply_set(e->req, REP_OK); + e->req->reply_set(e->req, REP_OK); } else if (e->operation == REQ_SET_ASYNC) { db_set(db, e->key, e->ksize, e->val, e->vsize); } else if (e->operation == REQ_GET) { unsigned char *val; - size_t vsize = 128 * 1024; + size_t vsize = 64 * 1024; val = malloc(vsize); if (val == NULL) { - tipc_reply_err(e->req, ERR_MEM); + e->req->reply_err(e->req, ERR_MEM); return; } rv = db_get(db, e->key, e->ksize, val, &vsize); if (rv == 0) { - tipc_reply_get(e->req, REP_NOTIN, NULL, 0); + e->req->reply_get(e->req, REP_NOTIN, NULL, 0); free(val); return; } - tipc_reply_get(e->req, REP_OK, val, vsize); + e->req->reply_get(e->req, REP_OK, val, vsize); free(val); } else if (e->operation == REQ_DEL_SYNC) { rv = db_del(db, e->key, e->ksize); if (rv == 0) { - tipc_reply_del(e->req, REP_NOTIN); + e->req->reply_del(e->req, REP_NOTIN); return; } - tipc_reply_del(e->req, REP_OK); + e->req->reply_del(e->req, REP_OK); } else if (e->operation == REQ_DEL_ASYNC) { db_del(db, e->key, e->ksize); + } else if (e->operation == REQ_CAS) { + unsigned char *dbval; + size_t dbvsize = 64 * 1024; + + /* Compare */ + dbval = malloc(dbvsize); + if (dbval == NULL) { + e->req->reply_err(e->req, ERR_MEM); + return; + } + rv = db_get(db, e->key, e->ksize, dbval, &dbvsize); + if (rv == 0) { + e->req->reply_get(e->req, REP_NOTIN, NULL, 0); + free(dbval); + return; + } + + if (e->vsize == dbvsize && + memcmp(e->val, dbval, dbvsize) == 0) { + /* Swap */ + rv = db_set(db, e->key, e->ksize, e->newval, e->nvsize); + if (!rv) { + e->req->reply_err(e->req, ERR_DB); + return; + } + + e->req->reply_cas(e->req, REP_OK); + free(dbval); + return; + } + + e->req->reply_cas(e->req, REP_NOMATCH); + free(dbval); + } else { printf("Unknown op 0x%x\n", e->operation); } diff --git a/nmdb/main.c b/nmdb/main.c index eae7f3a..4e75eb7 100644 --- a/nmdb/main.c +++ b/nmdb/main.c @@ -15,12 +15,23 @@ #define DEFDBNAME "database" -static void help() { +/* Define the common structures that are used throughout the whole server. */ +struct settings settings; +struct stats stats; +struct cache *cache_table; +struct queue *op_queue; + + +static void help(void) { char h[] = \ "nmdb [options]\n" " -d dbpath database path ('database', must be created with dpmgr)\n" - " -l lower lower TIPC port number (10)\n" - " -L upper upper TIPC port number (= lower)\n" + " -l lower TIPC lower port number (10)\n" + " -L upper TIPC upper port number (= lower)\n" + " -t port TCP listening port (26010)\n" + " -T addr TCP listening address (all local addresses)\n" + " -u port UDP listening port (26010)\n" + " -U addr UDP listening address (all local addresses)\n" " -c nobj max. number of objects to be cached, in thousands (128)\n" " -f don't fork and stay in the foreground\n" " -p enable passive mode, for redundancy purposes (read docs.)\n" @@ -38,6 +49,10 @@ static int load_settings(int argc, char **argv) settings.tipc_lower = -1; settings.tipc_upper = -1; + settings.tcp_addr = NULL; + settings.tcp_port = -1; + settings.udp_addr = NULL; + settings.udp_port = -1; settings.numobjs = -1; settings.foreground = 0; settings.passive = 0; @@ -45,19 +60,35 @@ static int load_settings(int argc, char **argv) settings.dbname = malloc(strlen(DEFDBNAME) + 1); strcpy(settings.dbname, DEFDBNAME); - while ((c = getopt(argc, argv, "d:l:L:c:fph?")) != -1) { + while ((c = getopt(argc, argv, "d:l:L:a:P:c:fph?")) != -1) { switch(c) { case 'd': free(settings.dbname); settings.dbname = malloc(strlen(optarg) + 1); strcpy(settings.dbname, optarg); break; + case 'l': settings.tipc_lower = atoi(optarg); break; case 'L': settings.tipc_upper = atoi(optarg); break; + + case 't': + settings.tcp_port = atoi(optarg); + break; + case 'T': + settings.tcp_addr = optarg; + break; + + case 'u': + settings.udp_port = atoi(optarg); + break; + case 'U': + settings.udp_addr = optarg; + break; + case 'c': settings.numobjs = atoi(optarg) * 1024; break; @@ -67,6 +98,7 @@ static int load_settings(int argc, char **argv) case 'p': settings.passive = 1; break; + case 'h': case '?': help(); @@ -78,9 +110,17 @@ static int load_settings(int argc, char **argv) } if (settings.tipc_lower == -1) - settings.tipc_lower = SERVER_INST; + settings.tipc_lower = TIPC_SERVER_INST; if (settings.tipc_upper == -1) settings.tipc_upper = settings.tipc_lower; + if (settings.tcp_addr == NULL) + settings.tcp_addr = TCP_SERVER_ADDR; + if (settings.tcp_port == -1) + settings.tcp_port = TCP_SERVER_PORT; + if (settings.udp_addr == NULL) + settings.udp_addr = UDP_SERVER_ADDR; + if (settings.udp_port == -1) + settings.udp_port = UDP_SERVER_PORT; if (settings.numobjs == -1) settings.numobjs = 128 * 1024; diff --git a/nmdb/net-const.h b/nmdb/net-const.h index a382490..bdaa29d 100644 --- a/nmdb/net-const.h +++ b/nmdb/net-const.h @@ -7,9 +7,17 @@ * Isolated so it's shared between the server and the library code. */ -/* TIPC server type and instance -- Hardcoded for now. */ -#define SERVER_TYPE 26001 -#define SERVER_INST 10 +/* TIPC server type (hardcoded) and default instance. */ +#define TIPC_SERVER_TYPE 26001 +#define TIPC_SERVER_INST 10 + +/* TCP default listen address and port. */ +#define TCP_SERVER_ADDR "0.0.0.0" +#define TCP_SERVER_PORT 26010 + +/* UDP default listen address and port. */ +#define UDP_SERVER_ADDR "0.0.0.0" +#define UDP_SERVER_PORT 26010 /* Protocol version, for checking in the network header. */ #define PROTO_VER 1 @@ -23,6 +31,8 @@ #define REQ_DEL_SYNC 0x106 #define REQ_SET_ASYNC 0x107 #define REQ_DEL_ASYNC 0x108 +#define REQ_CACHE_CAS 0x109 +#define REQ_CAS 0x110 /* Network replies (different namespace from requests) */ #define REP_ERR 0x800 @@ -30,6 +40,7 @@ #define REP_CACHE_MISS 0x802 #define REP_OK 0x803 #define REP_NOTIN 0x804 +#define REP_NOMATCH 0x805 /* Network error replies */ #define ERR_VER 0x101 /* Version mismatch */ @@ -39,5 +50,6 @@ #define ERR_MEM 0x105 /* Memory allocation error */ #define ERR_DB 0x106 /* Database error */ + #endif diff --git a/nmdb/net.c b/nmdb/net.c index 6260460..a376c17 100644 --- a/nmdb/net.c +++ b/nmdb/net.c @@ -9,39 +9,93 @@ typedef unsigned char u_char; #include #include +#include "common.h" #include "tipc.h" +#include "tcp.h" +#include "udp.h" +#include "net.h" -static void signal_handler(int fd, short event, void *arg) +static void exit_sighandler(int fd, short event, void *arg) { printf("Got signal! Puf!\n"); event_loopexit(NULL); } +static void passive_to_active_sighandler(int fd, short event, void *arg) +{ + printf("Passive toggle!\n"); + settings.passive = !settings.passive; +} void net_loop(void) { - int tipc_fd; - struct event srv_evt, sigterm_evt, sigint_evt; + int tipc_fd = -1; + int tcp_fd = -1; + int udp_fd = -1; + struct event tipc_evt, tcp_evt, udp_evt, + sigterm_evt, sigint_evt, sigusr2_evt; + + event_init(); - tipc_fd = tipc_init(); - if (tipc_fd < 0) { - perror("Error initializing TIPC"); - exit(1); + /* ENABLE_* are preprocessor constants defined on the command line by + * make. */ + + if (ENABLE_TIPC) { + tipc_fd = tipc_init(); + if (tipc_fd < 0) { + perror("Error initializing TIPC"); + exit(1); + } + + event_set(&tipc_evt, tipc_fd, EV_READ | EV_PERSIST, tipc_recv, + &tipc_evt); + event_add(&tipc_evt, NULL); } - event_init(); + if (ENABLE_TCP) { + tcp_fd = tcp_init(); + if (tcp_fd < 0) { + perror("Error initializing TCP"); + exit(1); + } - event_set(&srv_evt, tipc_fd, EV_READ | EV_PERSIST, tipc_recv, - &srv_evt); - event_add(&srv_evt, NULL); + event_set(&tcp_evt, tcp_fd, EV_READ | EV_PERSIST, tcp_newconnection, + &tcp_evt); + event_add(&tcp_evt, NULL); + } + + if (ENABLE_UDP) { + udp_fd = udp_init(); + if (udp_fd < 0) { + perror("Error initializing UDP"); + exit(1); + } + + event_set(&udp_evt, udp_fd, EV_READ | EV_PERSIST, udp_recv, + &udp_evt); + event_add(&udp_evt, NULL); + } - signal_set(&sigterm_evt, SIGTERM, signal_handler, &sigterm_evt); + signal_set(&sigterm_evt, SIGTERM, exit_sighandler, &sigterm_evt); signal_add(&sigterm_evt, NULL); - signal_set(&sigint_evt, SIGINT, signal_handler, &sigint_evt); + signal_set(&sigint_evt, SIGINT, exit_sighandler, &sigint_evt); signal_add(&sigint_evt, NULL); + signal_set(&sigusr2_evt, SIGUSR2, passive_to_active_sighandler, + &sigusr2_evt); + signal_add(&sigusr2_evt, NULL); event_dispatch(); + + event_del(&tipc_evt); + event_del(&tcp_evt); + signal_del(&sigterm_evt); + signal_del(&sigint_evt); + signal_del(&sigusr2_evt); + + tipc_close(tipc_fd); + tcp_close(tcp_fd); + udp_close(udp_fd); } diff --git a/nmdb/nmdb.1 b/nmdb/nmdb.1 index d08455d..36d4418 100644 --- a/nmdb/nmdb.1 +++ b/nmdb/nmdb.1 @@ -1,14 +1,13 @@ .TH nmdb 1 "11/Sep/2006" .SH NAME -nmdb - A TIPC-based database manager +nmdb - A multiprotocol network database manager .SH SYNOPSYS -nmdb [-d dbpath] [-l lower] [-L upper] [-c nobj] [-f] [-p] [-h] +nmdb [-d dbpath] [-l lower] [-L upper] [-t tcpport] [-T tcpaddr] +[-u udpport] [-U udpaddr] [-c nobj] [-f] [-p] [-h] .SH DESCRIPTION -nmdb is a TIPC-based database manager. - -It allows all the applications in the cluster to store (key, value) pairs in a -central cache and database in a transparent way. +nmdb is a network database that can use several protocols to communicate with +it's clients. At the moment, it supports TIPC, TCP and UDP. It can also be used as a generic caching system (pretty much like memcached), because it has a very fast cache that can be used without impacting on the @@ -46,7 +45,19 @@ run more than one nmdb instance in the same TIPC cluster. .B "-L upper" Upper TIPC port number to bind to. Defaults to the same value .B lower -is defined. Useful mainly for passive mode (which is not implemented yet). +is defined. Useful mainly for passive mode. +.TP +.B "-t tcpport" +TCP listening port. Defaults to 26010. +.TP +.B "-T tcpaddr" +IP listening address for TCP. Defaults to all local addresses. +.TP +.B "-u udpport" +UDP listening port. Defaults to 26010. +.TP +.B "-U udpaddr" +IP listening address for UDP. Defaults to all local addresses. .TP .B "-c nobj" Sets the maximum number of objects the cache will held, in thousands. Note diff --git a/nmdb/tipc.c b/nmdb/parse.c similarity index 53% copy from nmdb/tipc.c copy to nmdb/parse.c index 3a216bd..1acd07f 100644 --- a/nmdb/tipc.c +++ b/nmdb/parse.c @@ -1,100 +1,33 @@ -#include /* socket defines */ -#include /* socket functions */ #include /* malloc() */ -#include /* tipc stuff */ -#include /* perror() */ #include /* uint32_t and friends */ -#include /* htonls() and friends */ #include /* memcpy() */ +#include /* htonl() and friends */ -#include "tipc.h" -#include "common.h" + +#include "parse.h" +#include "req.h" #include "queue.h" #include "net-const.h" +#include "common.h" -static void parse_msg(struct req_info *req, unsigned char *buf, - size_t bsize); static void parse_get(struct req_info *req, int impact_db); static void parse_set(struct req_info *req, int impact_db, int async); static void parse_del(struct req_info *req, int impact_db, int async); - - -/* - * Miscelaneous helper functions - */ - -static void rep_send_error(const struct req_info *req, const unsigned int code) -{ - int r, c; - unsigned char minibuf[3 * 4]; - - if (settings.passive) - return; - - /* Network format: ID (4), REP_ERR (4), error code (4) */ - r = htonl(REP_ERR); - c = htonl(code); - memcpy(minibuf, &(req->id), 4); - memcpy(minibuf + 4, &r, 4); - memcpy(minibuf + 8, &c, 4); - - /* If this send fails, there's nothing to be done */ - r = sendto(req->fd, minibuf, 3 * 4, 0, (struct sockaddr *) req->clisa, - req->clilen); - - if (r < 0) { - perror("rep_send_error() failed"); - } -} - - -static int rep_send(const struct req_info *req, const unsigned char *buf, - const size_t size) -{ - int rv; - - if (settings.passive) - return 1; - - rv = sendto(req->fd, buf, size, 0, - (struct sockaddr *) req->clisa, req->clilen); - if (rv < 0) { - rep_send_error(req, ERR_SEND); - return 0; - } - return 1; -} - - -/* Send small replies, consisting in only a value. */ -static void mini_reply(struct req_info *req, uint32_t reply) -{ - /* We use a mini buffer to speedup the small replies, to avoid the - * malloc() overhead. */ - unsigned char minibuf[8]; - - if (settings.passive) - return; - - reply = htonl(reply); - memcpy(minibuf, &(req->id), 4); - memcpy(minibuf + 4, &reply, 4); - rep_send(req, minibuf, 8); - return; -} +static void parse_cas(struct req_info *req, int impact_db); /* Create a queue entry structure based on the parameters passed. Memory * allocated here will be free()'d in queue_entry_free(). It's not the * cleanest way, but the alternatives are even messier. */ -static struct queue_entry *make_queue_entry(struct req_info *req, +static struct queue_entry *make_queue_long_entry(struct req_info *req, uint32_t operation, const unsigned char *key, size_t ksize, - const unsigned char *val, size_t vsize) + const unsigned char *val, size_t vsize, + const unsigned char *newval, size_t nvsize) { struct queue_entry *e; - unsigned char *kcopy, *vcopy; + unsigned char *kcopy, *vcopy, *nvcopy; e = queue_entry_create(); if (e == NULL) { @@ -123,11 +56,27 @@ static struct queue_entry *make_queue_entry(struct req_info *req, memcpy(vcopy, val, vsize); } + nvcopy = NULL; + if (newval != NULL) { + nvcopy = malloc(nvsize); + if (nvcopy == NULL) { + queue_entry_free(e); + if (kcopy != NULL) + free(kcopy); + if (vcopy != NULL) + free(vcopy); + return NULL; + } + memcpy(nvcopy, newval, nvsize); + } + e->operation = operation; e->key = kcopy; e->ksize = ksize; e->val = vcopy; e->vsize = vsize; + e->newval = nvcopy; + e->nvsize = nvsize; /* Create a copy of req, including clisa */ e->req = malloc(sizeof(struct req_info)); @@ -137,12 +86,12 @@ static struct queue_entry *make_queue_entry(struct req_info *req, } memcpy(e->req, req, sizeof(struct req_info)); - e->req->clisa = malloc(sizeof(struct sockaddr_tipc)); + e->req->clisa = malloc(req->clilen); if (e->req->clisa == NULL) { queue_entry_free(e); return NULL; } - memcpy(e->req->clisa, req->clisa, sizeof(struct sockaddr_tipc)); + memcpy(e->req->clisa, req->clisa, req->clilen); /* clear out unused fields */ e->req->payload = NULL; @@ -151,170 +100,56 @@ static struct queue_entry *make_queue_entry(struct req_info *req, return e; } - -/* The tipc_reply_* functions are used by the db code to send the network - * replies. */ - -void tipc_reply_err(struct req_info *req, uint32_t reply) -{ - rep_send_error(req, reply); -} - -void tipc_reply_get(struct req_info *req, uint32_t reply, - unsigned char *val, size_t vsize) -{ - if (val == NULL) { - /* miss */ - mini_reply(req, reply); - } else { - unsigned char *buf; - size_t bsize; - uint32_t t; - - reply = htonl(reply); - - /* The reply length is: - * 4 id - * 4 reply code - * 4 vsize - * vsize val - */ - bsize = 4 + 4 + 4 + vsize; - buf = malloc(bsize); - - t = htonl(vsize); - - memcpy(buf, &(req->id), 4); - memcpy(buf + 4, &reply, 4); - memcpy(buf + 8, &t, 4); - memcpy(buf + 12, val, vsize); - - rep_send(req, buf, bsize); - free(buf); - } - return; - -} - - -void tipc_reply_set(struct req_info *req, uint32_t reply) -{ - mini_reply(req, reply); -} - - -void tipc_reply_del(struct req_info *req, uint32_t reply) +/* Like make_queue_long_entry() but with few parameters because most actions + * do not need newval. */ +static struct queue_entry *make_queue_entry(struct req_info *req, + uint32_t operation, const unsigned char *key, size_t ksize, + const unsigned char *val, size_t vsize) { - mini_reply(req, reply); + return make_queue_long_entry(req, operation, key, ksize, val, vsize, + NULL, 0); } -/* - * Main functions for receiving and parsing - */ - -int tipc_init(void) +/* Parse an incoming message. Note that the protocol might have sent this + * directly over the network (ie. TIPC) or might have wrapped it around (ie. + * TCP). Here we only deal with the clean, stripped, non protocol-specific + * message. */ +int parse_message(struct req_info *req, + const unsigned char *buf, size_t len) { - int fd, rv; - static struct sockaddr_tipc srvsa; - - srvsa.family = AF_TIPC; - if (settings.tipc_lower == settings.tipc_upper) - srvsa.addrtype = TIPC_ADDR_NAME; - else - srvsa.addrtype = TIPC_ADDR_NAMESEQ; - - srvsa.addr.nameseq.type = SERVER_TYPE; - srvsa.addr.nameseq.lower = settings.tipc_lower; - srvsa.addr.nameseq.upper = settings.tipc_upper; - srvsa.scope = TIPC_CLUSTER_SCOPE; - - fd = socket(AF_TIPC, SOCK_RDM, 0); - if (fd < 0) - return -1; - - rv = bind(fd, (struct sockaddr *) &srvsa, sizeof(srvsa)); - if (rv < 0) - return -1; - - return fd; -} + uint32_t hdr, ver, id, cmd; + const unsigned char *payload; + size_t psize; -/* Called by libevent for each receive event */ -void tipc_recv(int fd, short event, void *arg) -{ - int rv; - struct req_info req; - struct sockaddr_tipc clisa; - socklen_t clilen; - size_t bsize; - - /* Allocate enough to hold the max msg length of 66000 bytes. - * Originally, this was malloc()ed, but using the stack made it go - * from 27 usec for each set operation, to 23 usec. While it may sound - * worthless, it made test1 go from 3.213s to 2.345s for 37618 - * operations. - * TODO: check for negative impacts (beside being ugly, obviously) + /* The header is: + * 4 bytes Version + ID + * 4 bytes Command + * Variable Payload */ - unsigned char buf[128 * 1024]; - bsize = 128 * 1024; - - clilen = sizeof(clisa); - rv = recvfrom(fd, buf, bsize, 0, (struct sockaddr *) &clisa, - &clilen); - if (rv <= 0) { - /* rv == 0 means "return of an undeliverable message", which - * we ignore; -1 means other error. */ - goto exit; - } - - if (rv < 2) { - stats.net_broken_req++; - goto exit; - } - - req.fd = fd; - req.clisa = &clisa; - req.clilen = clilen; - - /* parse the message */ - parse_msg(&req, buf, rv); - -exit: - return; -} - - -/* Main message parsing and dissecting */ -static void parse_msg(struct req_info *req, unsigned char *buf, size_t bsize) -{ - uint32_t hdr, id, cmd; - uint8_t ver; - size_t psize; - unsigned char *payload; - - hdr = * (uint32_t *) buf; + hdr = * ((uint32_t *) buf); hdr = htonl(hdr); /* FIXME: little endian-only */ ver = (hdr & 0xF0000000) >> 28; id = hdr & 0x0FFFFFFF; + req->id = id; cmd = ntohl(* ((uint32_t *) buf + 1)); if (ver != PROTO_VER) { stats.net_version_mismatch++; - rep_send_error(req, ERR_VER); - return; + req->reply_err(req, ERR_VER); + return 0; } /* We define payload as the stuff after buf. But be careful because - * there might be none (if bsize == 1). Doing the pointer arithmetic + * there might be none (if len == 1). Doing the pointer arithmetic * isn't problematic, but accessing the payload should be done only if * we know we have enough data. */ payload = buf + 8; - psize = bsize - 8; + psize = len - 8; /* Store the id encoded in network byte order, so that we don't have * to calculate it at send time. */ @@ -339,32 +174,32 @@ static void parse_msg(struct req_info *req, unsigned char *buf, size_t bsize) parse_set(req, 1, 1); else if (cmd == REQ_DEL_ASYNC) parse_del(req, 1, 1); + else if (cmd == REQ_CACHE_CAS) + parse_cas(req, 0); + else if (cmd == REQ_CAS) + parse_cas(req, 1); else { stats.net_unk_req++; - rep_send_error(req, ERR_UNKREQ); - return; + req->reply_err(req, ERR_UNKREQ); } - return; + return 1; } static void parse_get(struct req_info *req, int impact_db) { int hit; - unsigned char *key; + const unsigned char *key; uint32_t ksize; unsigned char *val = NULL; size_t vsize = 0; - if (settings.passive) - return; - ksize = * (uint32_t *) req->payload; ksize = ntohl(ksize); if (req->psize < ksize) { stats.net_broken_req++; - rep_send_error(req, ERR_BROKEN); + req->reply_err(req, ERR_BROKEN); return; } @@ -373,13 +208,13 @@ static void parse_get(struct req_info *req, int impact_db) hit = cache_get(cache_table, key, ksize, &val, &vsize); if (!hit && !impact_db) { - mini_reply(req, REP_CACHE_MISS); + req->mini_reply(req, REP_CACHE_MISS); return; } else if (!hit && impact_db) { struct queue_entry *e; e = make_queue_entry(req, REQ_GET, key, ksize, NULL, 0); if (e == NULL) { - rep_send_error(req, ERR_MEM); + req->reply_err(req, ERR_MEM); return; } queue_lock(op_queue); @@ -388,7 +223,7 @@ static void parse_get(struct req_info *req, int impact_db) queue_signal(op_queue); return; } else { - tipc_reply_get(req, REP_CACHE_HIT, val, vsize); + req->reply_get(req, REP_CACHE_HIT, val, vsize); return; } } @@ -397,7 +232,7 @@ static void parse_get(struct req_info *req, int impact_db) static void parse_set(struct req_info *req, int impact_db, int async) { int rv; - unsigned char *key, *val; + const unsigned char *key, *val; uint32_t ksize, vsize; const int max = 65536; @@ -421,7 +256,7 @@ static void parse_set(struct req_info *req, int impact_db, int async) (ksize > max) || (vsize > max) || ( (ksize + vsize) > max) ) { stats.net_broken_req++; - rep_send_error(req, ERR_BROKEN); + req->reply_err(req, ERR_BROKEN); return; } @@ -430,7 +265,7 @@ static void parse_set(struct req_info *req, int impact_db, int async) rv = cache_set(cache_table, key, ksize, val, vsize); if (!rv) { - rep_send_error(req, ERR_MEM); + req->reply_err(req, ERR_MEM); return; } @@ -444,7 +279,7 @@ static void parse_set(struct req_info *req, int impact_db, int async) e = make_queue_entry(req, request, key, ksize, val, vsize); if (e == NULL) { - rep_send_error(req, ERR_MEM); + req->reply_err(req, ERR_MEM); return; } queue_lock(op_queue); @@ -452,7 +287,7 @@ static void parse_set(struct req_info *req, int impact_db, int async) queue_unlock(op_queue); if (async) { - mini_reply(req, REP_OK); + req->mini_reply(req, REP_OK); } else { /* Signal the DB thread it has work only if it's a * synchronous operation, asynchronous don't mind @@ -463,7 +298,7 @@ static void parse_set(struct req_info *req, int impact_db, int async) } return; } else { - mini_reply(req, REP_OK); + req->mini_reply(req, REP_OK); } return; @@ -473,14 +308,14 @@ static void parse_set(struct req_info *req, int impact_db, int async) static void parse_del(struct req_info *req, int impact_db, int async) { int hit; - unsigned char *key; + const unsigned char *key; uint32_t ksize; ksize = * (uint32_t *) req->payload; ksize = ntohl(ksize); if (req->psize < ksize) { stats.net_broken_req++; - rep_send_error(req, ERR_BROKEN); + req->reply_err(req, ERR_BROKEN); return; } @@ -489,9 +324,9 @@ static void parse_del(struct req_info *req, int impact_db, int async) hit = cache_del(cache_table, key, ksize); if (!impact_db && hit) { - mini_reply(req, REP_OK); + req->mini_reply(req, REP_OK); } else if (!impact_db && !hit) { - mini_reply(req, REP_NOTIN); + req->mini_reply(req, REP_NOTIN); } else if (impact_db) { struct queue_entry *e; uint32_t request; @@ -502,7 +337,7 @@ static void parse_del(struct req_info *req, int impact_db, int async) e = make_queue_entry(req, request, key, ksize, NULL, 0); if (e == NULL) { - rep_send_error(req, ERR_MEM); + req->reply_err(req, ERR_MEM); return; } queue_lock(op_queue); @@ -510,7 +345,7 @@ static void parse_del(struct req_info *req, int impact_db, int async) queue_unlock(op_queue); if (async) { - mini_reply(req, REP_OK); + req->mini_reply(req, REP_OK); } else { /* See comment on parse_set(). */ queue_signal(op_queue); @@ -522,4 +357,83 @@ static void parse_del(struct req_info *req, int impact_db, int async) return; } +static void parse_cas(struct req_info *req, int impact_db) +{ + int rv; + const unsigned char *key, *oldval, *newval; + uint32_t ksize, ovsize, nvsize; + const int max = 65536; + + /* Request format: + * 4 ksize + * 4 ovsize + * 4 nvsize + * ksize key + * ovsize oldval + * nvsize newval + */ + ksize = * (uint32_t *) req->payload; + ksize = ntohl(ksize); + ovsize = * ( ((uint32_t *) req->payload) + 1); + ovsize = ntohl(ovsize); + nvsize = * ( ((uint32_t *) req->payload) + 2); + nvsize = ntohl(nvsize); + + /* Sanity check on sizes: + * - ksize, ovsize and nvsize must all be < req->psize + * - ksize, ovsize and nvsize must all be < 2^16 = 64k + * - ksize + ovsize + mvsize < 2^16 = 64k + */ + if ( (req->psize < ksize) || (req->psize < ovsize) || + (req->psize < nvsize) || + (ksize > max) || (ovsize > max) || + (nvsize > max) || + ( (ksize + ovsize + nvsize) > max) ) { + stats.net_broken_req++; + req->reply_err(req, ERR_BROKEN); + return; + } + + key = req->payload + sizeof(uint32_t) * 3; + oldval = key + ksize; + newval = oldval + ovsize; + + rv = cache_cas(cache_table, key, ksize, oldval, ovsize, + newval, nvsize); + if (rv == 0) { + /* If the cache doesn't match, there is no need to bother the + * DB even if we were asked to impact. */ + req->mini_reply(req, REP_NOMATCH); + return; + } + + if (!impact_db) { + if (rv == -1) { + req->mini_reply(req, REP_NOTIN); + return; + } else { + req->mini_reply(req, REP_OK); + return; + } + } else { + /* impact_db = 1 and the key is either not in the cache, or + * cache_cas() was successful. We now need to queue the CAS in + * the database. */ + struct queue_entry *e; + + e = make_queue_long_entry(req, REQ_CAS, key, ksize, + oldval, ovsize, newval, nvsize); + if (e == NULL) { + req->reply_err(req, ERR_MEM); + return; + } + + queue_lock(op_queue); + queue_put(op_queue, e); + queue_unlock(op_queue); + queue_signal(op_queue); + } + return; +} + diff --git a/nmdb/parse.h b/nmdb/parse.h new file mode 100644 index 0000000..b27cf40 --- /dev/null +++ b/nmdb/parse.h @@ -0,0 +1,11 @@ + +#ifndef _PARSE_H +#define _PARSE_H + +#include "req.h" + +int parse_message(struct req_info *req, + const unsigned char *buf, size_t len); + +#endif + diff --git a/nmdb/queue.c b/nmdb/queue.c index 7eeda67..eb56b09 100644 --- a/nmdb/queue.c +++ b/nmdb/queue.c @@ -5,7 +5,7 @@ #include "queue.h" -struct queue *queue_create() +struct queue *queue_create(void) { struct queue *q; pthread_mutexattr_t attr; @@ -66,7 +66,7 @@ int queue_timedwait(struct queue *q, struct timespec *ts) } -struct queue_entry *queue_entry_create() +struct queue_entry *queue_entry_create(void) { struct queue_entry *e; @@ -77,8 +77,10 @@ struct queue_entry *queue_entry_create() e->operation = 0; e->key = NULL; e->val = NULL; + e->newval = NULL; e->ksize = 0; e->vsize = 0; + e->nvsize = 0; e->prev = NULL; return e; @@ -93,6 +95,8 @@ void queue_entry_free(struct queue_entry *e) { free(e->key); if (e->val) free(e->val); + if (e->newval) + free(e->newval); free(e); return; } diff --git a/nmdb/queue.h b/nmdb/queue.h index 69a0002..f50abe5 100644 --- a/nmdb/queue.h +++ b/nmdb/queue.h @@ -4,7 +4,7 @@ #include /* for mutexes */ #include /* for uint32_t */ -#include "tipc.h" /* for req_info */ +#include "req.h" /* for req_info */ struct queue { pthread_mutex_t lock; @@ -20,8 +20,10 @@ struct queue_entry { unsigned char *key; unsigned char *val; + unsigned char *newval; size_t ksize; size_t vsize; + size_t nvsize; struct queue_entry *prev; /* A pointer to the next element on the list is actually not diff --git a/nmdb/req.h b/nmdb/req.h new file mode 100644 index 0000000..454f0f8 --- /dev/null +++ b/nmdb/req.h @@ -0,0 +1,41 @@ + +#ifndef _REQ_H +#define _REQ_H + +#include /* uint32_t */ +#include /* size_t */ +#include /* socklen_t */ + + +/* req_info types, according to the protocol */ +#define REQTYPE_TIPC 1 +#define REQTYPE_TCP 2 +#define REQTYPE_UDP 3 + + +struct req_info { + /* network information */ + int fd; + int type; + + struct sockaddr *clisa; + socklen_t clilen; + + /* operation information */ + uint32_t id; + uint32_t cmd; + const unsigned char *payload; + size_t psize; + + /* operations */ + void (*mini_reply)(struct req_info *req, uint32_t reply); + void (*reply_err)(struct req_info *req, uint32_t reply); + void (*reply_get)(struct req_info *req, uint32_t reply, + unsigned char *val, size_t vsize); + void (*reply_set)(struct req_info *req, uint32_t reply); + void (*reply_del)(struct req_info *req, uint32_t reply); + void (*reply_cas)(struct req_info *req, uint32_t reply); +}; + +#endif + diff --git a/nmdb/tcp-stub.c b/nmdb/tcp-stub.c new file mode 100644 index 0000000..d3fffdc --- /dev/null +++ b/nmdb/tcp-stub.c @@ -0,0 +1,18 @@ + +/* TCP stub file, used when TCP is not compiled in. */ + +int tcp_init(void) +{ + return -1; +} + +void tcp_close(int fd) +{ + return; +} + +void tcp_recv(int fd, short event, void *arg) +{ + return; +} + diff --git a/nmdb/tcp.c b/nmdb/tcp.c new file mode 100644 index 0000000..7c86a99 --- /dev/null +++ b/nmdb/tcp.c @@ -0,0 +1,469 @@ + +#include /* socket defines */ +#include /* socket functions */ +#include /* malloc() */ +#include /* perror() */ +#include /* uint32_t and friends */ +#include /* htonls() and friends */ +#include /* INET stuff */ +#include /* TCP stuff */ +#include /* memcpy() */ +#include /* fcntl() */ +#include /* fcntl() */ +#include /* errno */ + +/* Workaround for libevent 1.1a: the header assumes u_char is typedef'ed to an + * unsigned char, and that "struct timeval" is in scope. */ +typedef unsigned char u_char; +#include +#include /* libevent stuff */ + +#include "tcp.h" +#include "common.h" +#include "net-const.h" +#include "req.h" +#include "parse.h" + + +/* TCP socket structure. Used mainly to hold buffers from incomplete + * recv()s. */ +struct tcp_socket { + int fd; + struct sockaddr_in clisa; + socklen_t clilen; + struct event *evt; + + unsigned char *buf; + size_t pktsize; + size_t len; + struct req_info req; + size_t excess; +}; + +static void tcp_recv(int fd, short event, void *arg); +static void process_buf(struct tcp_socket *tcpsock, + unsigned char *buf, size_t len); + +static void tcp_mini_reply(struct req_info *req, uint32_t reply); +static void tcp_reply_err(struct req_info *req, uint32_t reply); +static void tcp_reply_get(struct req_info *req, uint32_t reply, + unsigned char *val, size_t vsize); +static void tcp_reply_set(struct req_info *req, uint32_t reply); +static void tcp_reply_del(struct req_info *req, uint32_t reply); +static void tcp_reply_cas(struct req_info *req, uint32_t reply); + + +/* + * Miscelaneous helper functions + */ + +static void tcp_socket_free(struct tcp_socket *tcpsock) +{ + if (tcpsock->evt) + free(tcpsock->evt); + if (tcpsock->buf) + free(tcpsock->buf); + free(tcpsock); +} + +static void init_req(struct tcp_socket *tcpsock) +{ + tcpsock->req.fd = tcpsock->fd; + tcpsock->req.type = REQTYPE_TCP; + tcpsock->req.clisa = (struct sockaddr *) &tcpsock->clisa; + tcpsock->req.clilen = tcpsock->clilen; + tcpsock->req.mini_reply = tcp_mini_reply; + tcpsock->req.reply_err = tcp_reply_err; + tcpsock->req.reply_get = tcp_reply_get; + tcpsock->req.reply_set = tcp_reply_set; + tcpsock->req.reply_del = tcp_reply_del; + tcpsock->req.reply_cas = tcp_reply_cas; +} + +static void rep_send_error(const struct req_info *req, const unsigned int code) +{ + uint32_t l, r, c; + unsigned char minibuf[4 * 4]; + + if (settings.passive) + return; + + /* Network format: length (4), ID (4), REP_ERR (4), error code (4) */ + l = htonl(4 + 4 + 4 + 4); + r = htonl(REP_ERR); + c = htonl(code); + memcpy(minibuf, &l, 4); + memcpy(minibuf + 4, &(req->id), 4); + memcpy(minibuf + 8, &r, 4); + memcpy(minibuf + 12, &c, 4); + + /* If this send fails, there's nothing to be done */ + r = send(req->fd, minibuf, 4 * 4, 0); + + if (r < 0) { + perror("rep_send_error() failed"); + } +} + + +static int rep_send(const struct req_info *req, const unsigned char *buf, + const size_t size) +{ + int rv; + + if (settings.passive) + return 1; + + rv = send(req->fd, buf, size, 0); + if (rv < 0) { + rep_send_error(req, ERR_SEND); + return 0; + } + return 1; +} + + +/* Send small replies, consisting in only a value. */ +void tcp_mini_reply(struct req_info *req, uint32_t reply) +{ + /* We use a mini buffer to speedup the small replies, to avoid the + * malloc() overhead. */ + uint32_t len; + unsigned char minibuf[12]; + + if (settings.passive) + return; + + len = htonl(12); + reply = htonl(reply); + memcpy(minibuf, &len, 4); + memcpy(minibuf + 4, &(req->id), 4); + memcpy(minibuf + 8, &reply, 4); + rep_send(req, minibuf, 12); + return; +} + + +void tcp_reply_err(struct req_info *req, uint32_t reply) +{ + rep_send_error(req, reply); +} + +void tcp_reply_get(struct req_info *req, uint32_t reply, + unsigned char *val, size_t vsize) +{ + if (val == NULL) { + /* miss */ + tcp_mini_reply(req, reply); + } else { + unsigned char *buf; + size_t bsize; + uint32_t t; + + reply = htonl(reply); + + /* The reply length is: + * 4 total length + * 4 id + * 4 reply code + * 4 vsize + * vsize val + */ + bsize = 4 + 4 + 4 + 4 + vsize; + buf = malloc(bsize); + + t = htonl(bsize); + memcpy(buf, &t, 4); + + memcpy(buf + 4, &(req->id), 4); + memcpy(buf + 8, &reply, 4); + + t = htonl(vsize); + memcpy(buf + 12, &t, 4); + memcpy(buf + 16, val, vsize); + + rep_send(req, buf, bsize); + free(buf); + } + return; + +} + + +void tcp_reply_set(struct req_info *req, uint32_t reply) +{ + tcp_mini_reply(req, reply); +} + + +void tcp_reply_del(struct req_info *req, uint32_t reply) +{ + tcp_mini_reply(req, reply); +} + +void tcp_reply_cas(struct req_info *req, uint32_t reply) +{ + tcp_mini_reply(req, reply); +} + + +/* + * Main functions for receiving and parsing + */ + +int tcp_init(void) +{ + int fd, rv; + struct sockaddr_in srvsa; + struct in_addr ia; + + rv = inet_pton(AF_INET, settings.tcp_addr, &ia); + if (rv <= 0) + return -1; + + srvsa.sin_family = AF_INET; + srvsa.sin_addr.s_addr = ia.s_addr; + srvsa.sin_port = htons(settings.tcp_port); + + + fd = socket(AF_INET, SOCK_STREAM, 0); + if (fd < 0) + return -1; + + rv = 1; + if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &rv, sizeof(rv)) < 0 ) { + close(fd); + return -1; + } + + rv = bind(fd, (struct sockaddr *) &srvsa, sizeof(srvsa)); + if (rv < 0) { + close(fd); + return -1; + } + + rv = listen(fd, 1024); + if (rv < 0) { + close(fd); + return -1; + } + + /* Disable nagle algorithm, as we often handle small amounts of data + * it can make I/O quite slow. */ + rv = 1; + if (setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &rv, sizeof(rv)) < 0 ) { + close(fd); + return -1; + } + + return fd; +} + + +void tcp_close(int fd) +{ + close(fd); +} + + +/* Called by libevent for each receive event on our listen fd */ +void tcp_newconnection(int fd, short event, void *arg) +{ + int newfd; + struct tcp_socket *tcpsock; + struct event *new_event; + + tcpsock = malloc(sizeof(struct tcp_socket)); + if (tcpsock == NULL) { + return; + } + tcpsock->clilen = sizeof(tcpsock->clisa); + + new_event = malloc(sizeof(struct event)); + if (new_event == NULL) { + free(tcpsock); + return; + } + + newfd = accept(fd, + (struct sockaddr *) &(tcpsock->clisa), + &(tcpsock->clilen)); + + if (fcntl(newfd, F_SETFL, O_NONBLOCK) != 0) { + close(newfd); + free(new_event); + free(tcpsock); + return; + } + + tcpsock->fd = newfd; + tcpsock->evt = new_event; + tcpsock->buf = NULL; + tcpsock->pktsize = 0; + tcpsock->len = 0; + tcpsock->excess = 0; + + event_set(new_event, newfd, EV_READ | EV_PERSIST, tcp_recv, + (void *) tcpsock); + event_add(new_event, NULL); + + return; +} + + +/* Static common buffer to avoid unnecessary allocation on the common case + * where we get an entire single message on each recv(). + * Allocate a little bit more over the max. message size, which is 64kb. */ +#define SBSIZE (68 * 1024) +static unsigned char static_buf[SBSIZE]; + +/* Called by libevent for each receive event */ +static void tcp_recv(int fd, short event, void *arg) +{ + int rv; + struct tcp_socket *tcpsock; + + tcpsock = (struct tcp_socket *) arg; + + if (tcpsock->buf == NULL) { + /* New incoming message */ + rv = recv(fd, static_buf, SBSIZE, 0); + if (rv < 0 && errno == EAGAIN) { + /* We were awoken but have no data to read, so we do + * nothing */ + return; + } else if (rv <= 0) { + /* Orderly shutdown or error; close the file + * descriptor in either case. */ + goto error_exit; + } + + init_req(tcpsock); + process_buf(tcpsock, static_buf, rv); + + } else { + /* We already got a partial message, complete it. */ + size_t maxtoread = tcpsock->pktsize - tcpsock->len; + + rv = recv(fd, tcpsock->buf + tcpsock->len, maxtoread, 0); + if (rv < 0 && errno == EAGAIN) { + return; + } else if (rv <= 0) { + goto error_exit; + } + + tcpsock->len += rv; + + process_buf(tcpsock,tcpsock->buf, tcpsock->len); + } + + return; + +error_exit: + close(fd); + event_del(tcpsock->evt); + tcp_socket_free(tcpsock); + return; +} + + +/* Main message unwrapping */ +static void process_buf(struct tcp_socket *tcpsock, + unsigned char *buf, size_t len) +{ + uint32_t totaltoget = 0; + + if (len >= 4) { + totaltoget = * (uint32_t *) buf; + totaltoget = ntohl(totaltoget); + if (totaltoget > (64 * 1024) || totaltoget <= 12) { + /* Message too big or too small, close the connection. */ + goto error_exit; + } + + } else { + /* If we didn't even read 4 bytes, we try to read 4 first and + * then care about the rest. */ + totaltoget = 4; + } + + if (totaltoget > len) { + if (tcpsock->buf == NULL) { + /* The first incomplete recv(). + * Create a temporary buffer and copy the contents of + * our current one (which is static_buf, otherwise + * tcpsock->buf wouldn't be NULL) to it. */ + tcpsock->buf = malloc(SBSIZE); + if (tcpsock->buf == NULL) + goto error_exit; + + memcpy(tcpsock->buf, buf, len); + tcpsock->len = len; + tcpsock->pktsize = totaltoget; + + } else { + /* We already had an incomplete recv() and this is + * just another one. */ + tcpsock->len = len; + tcpsock->pktsize = totaltoget; + } + return; + } + + if (totaltoget < len) { + /* Got more than one message in the same recv(); save the + * amount of bytes exceeding so we can process it later. */ + tcpsock->excess = len - totaltoget; + len = totaltoget; + } + + /* The buffer is complete, parse it as usual. */ + if (parse_message(&(tcpsock->req), buf + 4, len - 4)) { + goto exit; + } else { + goto error_exit; + } + + +exit: + if (tcpsock->excess) { + /* If there are buffer leftovers (because there was more than + * one message on a recv()), leave the buffer, move the + * leftovers to the beginning, adjust the numbers and parse + * recursively. + * The buffer can be the static one or the one in tcpsock (if + * we had a short recv()); we don't care because we know it + * will be big enough to hold an entire message anyway. */ + memmove(buf, buf + len, tcpsock->excess); + tcpsock->len = tcpsock->excess; + tcpsock->excess = 0; + + /* Build a new req just like when we first recv(). */ + init_req(tcpsock); + process_buf(tcpsock, buf, len); + return; + + } + + if (tcpsock->buf) { + /* We had an incomplete read somewhere along the processing of + * this message, and had to malloc() a temporary space. free() + * it and reset the associated information. */ + free(tcpsock->buf); + tcpsock->buf = NULL; + tcpsock->len = 0; + tcpsock->pktsize = 0; + tcpsock->excess = 0; + } + + return; + +error_exit: + close(tcpsock->fd); + event_del(tcpsock->evt); + tcp_socket_free(tcpsock); + return; + +} + + diff --git a/nmdb/tcp.h b/nmdb/tcp.h new file mode 100644 index 0000000..924231d --- /dev/null +++ b/nmdb/tcp.h @@ -0,0 +1,10 @@ + +#ifndef _TCP_H +#define _TCP_H + +int tcp_init(void); +void tcp_close(int fd); +void tcp_newconnection(int fd, short event, void *arg); + +#endif + diff --git a/nmdb/tipc-stub.c b/nmdb/tipc-stub.c new file mode 100644 index 0000000..a717389 --- /dev/null +++ b/nmdb/tipc-stub.c @@ -0,0 +1,18 @@ + +/* TIPC stub file, used when TIPC is not compiled in. */ + +int tipc_init(void) +{ + return -1; +} + +void tipc_close(int fd) +{ + return; +} + +void tipc_recv(int fd, short event, void *arg) +{ + return; +} + diff --git a/nmdb/tipc.c b/nmdb/tipc.c index 3a216bd..eb73bc4 100644 --- a/nmdb/tipc.c +++ b/nmdb/tipc.c @@ -7,18 +7,22 @@ #include /* uint32_t and friends */ #include /* htonls() and friends */ #include /* memcpy() */ +#include /* close() */ #include "tipc.h" #include "common.h" -#include "queue.h" #include "net-const.h" +#include "req.h" +#include "parse.h" -static void parse_msg(struct req_info *req, unsigned char *buf, - size_t bsize); -static void parse_get(struct req_info *req, int impact_db); -static void parse_set(struct req_info *req, int impact_db, int async); -static void parse_del(struct req_info *req, int impact_db, int async); +static void tipc_mini_reply(struct req_info *req, uint32_t reply); +static void tipc_reply_err(struct req_info *req, uint32_t reply); +static void tipc_reply_get(struct req_info *req, uint32_t reply, + unsigned char *val, size_t vsize); +static void tipc_reply_set(struct req_info *req, uint32_t reply); +static void tipc_reply_del(struct req_info *req, uint32_t reply); +static void tipc_reply_cas(struct req_info *req, uint32_t reply); /* @@ -41,8 +45,7 @@ static void rep_send_error(const struct req_info *req, const unsigned int code) memcpy(minibuf + 8, &c, 4); /* If this send fails, there's nothing to be done */ - r = sendto(req->fd, minibuf, 3 * 4, 0, (struct sockaddr *) req->clisa, - req->clilen); + r = sendto(req->fd, minibuf, 3 * 4, 0, req->clisa, req->clilen); if (r < 0) { perror("rep_send_error() failed"); @@ -58,8 +61,7 @@ static int rep_send(const struct req_info *req, const unsigned char *buf, if (settings.passive) return 1; - rv = sendto(req->fd, buf, size, 0, - (struct sockaddr *) req->clisa, req->clilen); + rv = sendto(req->fd, buf, size, 0, req->clisa, req->clilen); if (rv < 0) { rep_send_error(req, ERR_SEND); return 0; @@ -69,7 +71,7 @@ static int rep_send(const struct req_info *req, const unsigned char *buf, /* Send small replies, consisting in only a value. */ -static void mini_reply(struct req_info *req, uint32_t reply) +static void tipc_mini_reply(struct req_info *req, uint32_t reply) { /* We use a mini buffer to speedup the small replies, to avoid the * malloc() overhead. */ @@ -86,72 +88,6 @@ static void mini_reply(struct req_info *req, uint32_t reply) } -/* Create a queue entry structure based on the parameters passed. Memory - * allocated here will be free()'d in queue_entry_free(). It's not the - * cleanest way, but the alternatives are even messier. */ -static struct queue_entry *make_queue_entry(struct req_info *req, - uint32_t operation, const unsigned char *key, size_t ksize, - const unsigned char *val, size_t vsize) -{ - struct queue_entry *e; - unsigned char *kcopy, *vcopy; - - e = queue_entry_create(); - if (e == NULL) { - return NULL; - } - - kcopy = NULL; - if (key != NULL) { - kcopy = malloc(ksize); - if (kcopy == NULL) { - queue_entry_free(e); - return NULL; - } - memcpy(kcopy, key, ksize); - } - - vcopy = NULL; - if (val != NULL) { - vcopy = malloc(vsize); - if (vcopy == NULL) { - queue_entry_free(e); - if (kcopy != NULL) - free(kcopy); - return NULL; - } - memcpy(vcopy, val, vsize); - } - - e->operation = operation; - e->key = kcopy; - e->ksize = ksize; - e->val = vcopy; - e->vsize = vsize; - - /* Create a copy of req, including clisa */ - e->req = malloc(sizeof(struct req_info)); - if (e->req == NULL) { - queue_entry_free(e); - return NULL; - } - memcpy(e->req, req, sizeof(struct req_info)); - - e->req->clisa = malloc(sizeof(struct sockaddr_tipc)); - if (e->req->clisa == NULL) { - queue_entry_free(e); - return NULL; - } - memcpy(e->req->clisa, req->clisa, sizeof(struct sockaddr_tipc)); - - /* clear out unused fields */ - e->req->payload = NULL; - e->req->psize = 0; - - return e; -} - - /* The tipc_reply_* functions are used by the db code to send the network * replies. */ @@ -165,7 +101,7 @@ void tipc_reply_get(struct req_info *req, uint32_t reply, { if (val == NULL) { /* miss */ - mini_reply(req, reply); + tipc_mini_reply(req, reply); } else { unsigned char *buf; size_t bsize; @@ -199,13 +135,18 @@ void tipc_reply_get(struct req_info *req, uint32_t reply, void tipc_reply_set(struct req_info *req, uint32_t reply) { - mini_reply(req, reply); + tipc_mini_reply(req, reply); } void tipc_reply_del(struct req_info *req, uint32_t reply) { - mini_reply(req, reply); + tipc_mini_reply(req, reply); +} + +void tipc_reply_cas(struct req_info *req, uint32_t reply) +{ + tipc_mini_reply(req, reply); } @@ -216,7 +157,7 @@ void tipc_reply_del(struct req_info *req, uint32_t reply) int tipc_init(void) { int fd, rv; - static struct sockaddr_tipc srvsa; + struct sockaddr_tipc srvsa; srvsa.family = AF_TIPC; if (settings.tipc_lower == settings.tipc_upper) @@ -224,7 +165,7 @@ int tipc_init(void) else srvsa.addrtype = TIPC_ADDR_NAMESEQ; - srvsa.addr.nameseq.type = SERVER_TYPE; + srvsa.addr.nameseq.type = TIPC_SERVER_TYPE; srvsa.addr.nameseq.lower = settings.tipc_lower; srvsa.addr.nameseq.upper = settings.tipc_upper; srvsa.scope = TIPC_CLUSTER_SCOPE; @@ -240,6 +181,21 @@ int tipc_init(void) return fd; } + +void tipc_close(int fd) +{ + close(fd); +} + + +/* Static common buffer to avoid unnecessary allocations. + * Originally, this was malloc()ed, but making it static made it go from 27 + * usec for each set operation, to 23 usec: it made test1 go from 3.213s to + * 2.345s for 37618 operations. + * Allocate enough to hold the max msg length of 64kbytes. */ +#define SBSIZE (68 * 1024) +static unsigned char static_buf[SBSIZE]; + /* Called by libevent for each receive event */ void tipc_recv(int fd, short event, void *arg) { @@ -247,21 +203,10 @@ void tipc_recv(int fd, short event, void *arg) struct req_info req; struct sockaddr_tipc clisa; socklen_t clilen; - size_t bsize; - - /* Allocate enough to hold the max msg length of 66000 bytes. - * Originally, this was malloc()ed, but using the stack made it go - * from 27 usec for each set operation, to 23 usec. While it may sound - * worthless, it made test1 go from 3.213s to 2.345s for 37618 - * operations. - * TODO: check for negative impacts (beside being ugly, obviously) - */ - unsigned char buf[128 * 1024]; - bsize = 128 * 1024; clilen = sizeof(clisa); - rv = recvfrom(fd, buf, bsize, 0, (struct sockaddr *) &clisa, + rv = recvfrom(fd, static_buf, SBSIZE, 0, (struct sockaddr *) &clisa, &clilen); if (rv <= 0) { /* rv == 0 means "return of an undeliverable message", which @@ -275,251 +220,21 @@ void tipc_recv(int fd, short event, void *arg) } req.fd = fd; - req.clisa = &clisa; + req.type = REQTYPE_TIPC; + req.clisa = (struct sockaddr *) &clisa; req.clilen = clilen; + req.mini_reply = tipc_mini_reply; + req.reply_err = tipc_reply_err; + req.reply_get = tipc_reply_get; + req.reply_set = tipc_reply_set; + req.reply_del = tipc_reply_del; + req.reply_cas = tipc_reply_cas; /* parse the message */ - parse_msg(&req, buf, rv); + parse_message(&req, static_buf, rv); exit: return; } -/* Main message parsing and dissecting */ -static void parse_msg(struct req_info *req, unsigned char *buf, size_t bsize) -{ - uint32_t hdr, id, cmd; - uint8_t ver; - size_t psize; - unsigned char *payload; - - hdr = * (uint32_t *) buf; - hdr = htonl(hdr); - - /* FIXME: little endian-only */ - ver = (hdr & 0xF0000000) >> 28; - id = hdr & 0x0FFFFFFF; - - cmd = ntohl(* ((uint32_t *) buf + 1)); - - if (ver != PROTO_VER) { - stats.net_version_mismatch++; - rep_send_error(req, ERR_VER); - return; - } - - /* We define payload as the stuff after buf. But be careful because - * there might be none (if bsize == 1). Doing the pointer arithmetic - * isn't problematic, but accessing the payload should be done only if - * we know we have enough data. */ - payload = buf + 8; - psize = bsize - 8; - - /* Store the id encoded in network byte order, so that we don't have - * to calculate it at send time. */ - req->id = htonl(id); - req->cmd = cmd; - req->payload = payload; - req->psize = psize; - - if (cmd == REQ_CACHE_GET) - parse_get(req, 0); - else if (cmd == REQ_CACHE_SET) - parse_set(req, 0, 0); - else if (cmd == REQ_CACHE_DEL) - parse_del(req, 0, 0); - else if (cmd == REQ_GET) - parse_get(req, 1); - else if (cmd == REQ_SET_SYNC) - parse_set(req, 1, 0); - else if (cmd == REQ_DEL_SYNC) - parse_del(req, 1, 0); - else if (cmd == REQ_SET_ASYNC) - parse_set(req, 1, 1); - else if (cmd == REQ_DEL_ASYNC) - parse_del(req, 1, 1); - else { - stats.net_unk_req++; - rep_send_error(req, ERR_UNKREQ); - return; - } - - return; -} - - -static void parse_get(struct req_info *req, int impact_db) -{ - int hit; - unsigned char *key; - uint32_t ksize; - unsigned char *val = NULL; - size_t vsize = 0; - - if (settings.passive) - return; - - ksize = * (uint32_t *) req->payload; - ksize = ntohl(ksize); - if (req->psize < ksize) { - stats.net_broken_req++; - rep_send_error(req, ERR_BROKEN); - return; - } - - key = req->payload + sizeof(uint32_t); - - hit = cache_get(cache_table, key, ksize, &val, &vsize); - - if (!hit && !impact_db) { - mini_reply(req, REP_CACHE_MISS); - return; - } else if (!hit && impact_db) { - struct queue_entry *e; - e = make_queue_entry(req, REQ_GET, key, ksize, NULL, 0); - if (e == NULL) { - rep_send_error(req, ERR_MEM); - return; - } - queue_lock(op_queue); - queue_put(op_queue, e); - queue_unlock(op_queue); - queue_signal(op_queue); - return; - } else { - tipc_reply_get(req, REP_CACHE_HIT, val, vsize); - return; - } -} - - -static void parse_set(struct req_info *req, int impact_db, int async) -{ - int rv; - unsigned char *key, *val; - uint32_t ksize, vsize; - const int max = 65536; - - /* Request format: - * 4 ksize - * 4 vsize - * ksize key - * vsize val - */ - ksize = * (uint32_t *) req->payload; - ksize = ntohl(ksize); - vsize = * ( ((uint32_t *) req->payload) + 1), - vsize = ntohl(vsize); - - /* Sanity check on sizes: - * - ksize and vsize must both be < req->psize - * - ksize and vsize must both be < 2^16 = 64k - * - ksize + vsize < 2^16 = 64k - */ - if ( (req->psize < ksize) || (req->psize < vsize) || - (ksize > max) || (vsize > max) || - ( (ksize + vsize) > max) ) { - stats.net_broken_req++; - rep_send_error(req, ERR_BROKEN); - return; - } - - key = req->payload + sizeof(uint32_t) * 2; - val = key + ksize; - - rv = cache_set(cache_table, key, ksize, val, vsize); - if (!rv) { - rep_send_error(req, ERR_MEM); - return; - } - - if (impact_db) { - struct queue_entry *e; - uint32_t request; - - request = REQ_SET_SYNC; - if (async) - request = REQ_SET_ASYNC; - - e = make_queue_entry(req, request, key, ksize, val, vsize); - if (e == NULL) { - rep_send_error(req, ERR_MEM); - return; - } - queue_lock(op_queue); - queue_put(op_queue, e); - queue_unlock(op_queue); - - if (async) { - mini_reply(req, REP_OK); - } else { - /* Signal the DB thread it has work only if it's a - * synchronous operation, asynchronous don't mind - * waiting. It does have a measurable impact on - * performance (2083847usec vs 2804973usec for sets on - * "test2d 100000 10 10"). */ - queue_signal(op_queue); - } - return; - } else { - mini_reply(req, REP_OK); - } - - return; -} - - -static void parse_del(struct req_info *req, int impact_db, int async) -{ - int hit; - unsigned char *key; - uint32_t ksize; - - ksize = * (uint32_t *) req->payload; - ksize = ntohl(ksize); - if (req->psize < ksize) { - stats.net_broken_req++; - rep_send_error(req, ERR_BROKEN); - return; - } - - key = req->payload + sizeof(uint32_t); - - hit = cache_del(cache_table, key, ksize); - - if (!impact_db && hit) { - mini_reply(req, REP_OK); - } else if (!impact_db && !hit) { - mini_reply(req, REP_NOTIN); - } else if (impact_db) { - struct queue_entry *e; - uint32_t request; - - request = REQ_DEL_SYNC; - if (async) - request = REQ_DEL_ASYNC; - - e = make_queue_entry(req, request, key, ksize, NULL, 0); - if (e == NULL) { - rep_send_error(req, ERR_MEM); - return; - } - queue_lock(op_queue); - queue_put(op_queue, e); - queue_unlock(op_queue); - - if (async) { - mini_reply(req, REP_OK); - } else { - /* See comment on parse_set(). */ - queue_signal(op_queue); - } - - return; - } - - return; -} - - diff --git a/nmdb/tipc.h b/nmdb/tipc.h index a717836..895296e 100644 --- a/nmdb/tipc.h +++ b/nmdb/tipc.h @@ -2,33 +2,9 @@ #ifndef _MYTIPC_H #define _MYTIPC_H -#include /* uint32_t */ -#include /* size_t */ -#include /* socklen_t */ -#include /* sockaddr_tipc */ - -struct req_info { - /* network information */ - int fd; - struct sockaddr_tipc *clisa; - socklen_t clilen; - - /* operation information */ - uint32_t id; - uint32_t cmd; - unsigned char *payload; - size_t psize; -}; - - int tipc_init(void); +void tipc_close(int fd); void tipc_recv(int fd, short event, void *arg); -void tipc_reply_err(struct req_info *req, uint32_t reply); -void tipc_reply_get(struct req_info *req, uint32_t reply, - unsigned char *val, size_t vsize); -void tipc_reply_set(struct req_info *req, uint32_t reply); -void tipc_reply_del(struct req_info *req, uint32_t reply); - #endif diff --git a/nmdb/udp-stub.c b/nmdb/udp-stub.c new file mode 100644 index 0000000..1fbc770 --- /dev/null +++ b/nmdb/udp-stub.c @@ -0,0 +1,18 @@ + +/* UDP stub file, used when UDP is not compiled in. */ + +int udp_init(void) +{ + return -1; +} + +void udp_close(int fd) +{ + return; +} + +void udp_recv(int fd, short event, void *arg) +{ + return; +} + diff --git a/nmdb/udp.c b/nmdb/udp.c new file mode 100644 index 0000000..29cb789 --- /dev/null +++ b/nmdb/udp.c @@ -0,0 +1,240 @@ + +#include /* socket defines */ +#include /* socket functions */ +#include /* malloc() */ +#include /* perror() */ +#include /* uint32_t and friends */ +#include /* htonls() and friends */ +#include /* INET stuff */ +#include /* UDP stuff */ +#include /* memcpy() */ +#include /* close() */ + +#include "udp.h" +#include "common.h" +#include "net-const.h" +#include "req.h" +#include "parse.h" + + +static void udp_mini_reply(struct req_info *req, uint32_t reply); +static void udp_reply_err(struct req_info *req, uint32_t reply); +static void udp_reply_get(struct req_info *req, uint32_t reply, + unsigned char *val, size_t vsize); +static void udp_reply_set(struct req_info *req, uint32_t reply); +static void udp_reply_del(struct req_info *req, uint32_t reply); +static void udp_reply_cas(struct req_info *req, uint32_t reply); + + +/* + * Miscelaneous helper functions + */ + +static void rep_send_error(const struct req_info *req, const unsigned int code) +{ + int r, c; + unsigned char minibuf[3 * 4]; + + if (settings.passive) + return; + + /* Network format: ID (4), REP_ERR (4), error code (4) */ + r = htonl(REP_ERR); + c = htonl(code); + memcpy(minibuf, &(req->id), 4); + memcpy(minibuf + 4, &r, 4); + memcpy(minibuf + 8, &c, 4); + + /* If this send fails, there's nothing to be done */ + r = sendto(req->fd, minibuf, 3 * 4, 0, req->clisa, req->clilen); + + if (r < 0) { + perror("rep_send_error() failed"); + } +} + + +static int rep_send(const struct req_info *req, const unsigned char *buf, + const size_t size) +{ + int rv; + + if (settings.passive) + return 1; + + rv = sendto(req->fd, buf, size, 0, req->clisa, req->clilen); + if (rv < 0) { + rep_send_error(req, ERR_SEND); + return 0; + } + return 1; +} + + +/* Send small replies, consisting in only a value. */ +static void udp_mini_reply(struct req_info *req, uint32_t reply) +{ + /* We use a mini buffer to speedup the small replies, to avoid the + * malloc() overhead. */ + unsigned char minibuf[8]; + + if (settings.passive) + return; + + reply = htonl(reply); + memcpy(minibuf, &(req->id), 4); + memcpy(minibuf + 4, &reply, 4); + rep_send(req, minibuf, 8); + return; +} + + +/* The udp_reply_* functions are used by the db code to send the network + * replies. */ + +void udp_reply_err(struct req_info *req, uint32_t reply) +{ + rep_send_error(req, reply); +} + +void udp_reply_get(struct req_info *req, uint32_t reply, + unsigned char *val, size_t vsize) +{ + if (val == NULL) { + /* miss */ + udp_mini_reply(req, reply); + } else { + unsigned char *buf; + size_t bsize; + uint32_t t; + + reply = htonl(reply); + + /* The reply length is: + * 4 id + * 4 reply code + * 4 vsize + * vsize val + */ + bsize = 4 + 4 + 4 + vsize; + buf = malloc(bsize); + + t = htonl(vsize); + + memcpy(buf, &(req->id), 4); + memcpy(buf + 4, &reply, 4); + memcpy(buf + 8, &t, 4); + memcpy(buf + 12, val, vsize); + + rep_send(req, buf, bsize); + free(buf); + } + return; + +} + + +void udp_reply_set(struct req_info *req, uint32_t reply) +{ + udp_mini_reply(req, reply); +} + + +void udp_reply_del(struct req_info *req, uint32_t reply) +{ + udp_mini_reply(req, reply); +} + +void udp_reply_cas(struct req_info *req, uint32_t reply) +{ + udp_mini_reply(req, reply); +} + + +/* + * Main functions for receiving and parsing + */ + +int udp_init(void) +{ + int fd, rv; + struct sockaddr_in srvsa; + struct in_addr ia; + + rv = inet_pton(AF_INET, settings.udp_addr, &ia); + if (rv <= 0) + return -1; + + srvsa.sin_family = AF_INET; + srvsa.sin_addr.s_addr = ia.s_addr; + srvsa.sin_port = htons(settings.udp_port); + + fd = socket(AF_INET, SOCK_DGRAM, 0); + if (fd < 0) + return -1; + + rv = 1; + if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &rv, sizeof(rv)) < 0 ) { + close(fd); + return -1; + } + + rv = bind(fd, (struct sockaddr *) &srvsa, sizeof(srvsa)); + if (rv < 0) + return -1; + + return fd; +} + + +void udp_close(int fd) +{ + close(fd); +} + + +/* Static common buffer to avoid unnecessary allocations. See the comments on + * this same variable in tipc.c. */ +#define SBSIZE (68 * 1024) +static unsigned char static_buf[SBSIZE]; + +/* Called by libevent for each receive event */ +void udp_recv(int fd, short event, void *arg) +{ + int rv; + struct req_info req; + struct sockaddr_in clisa; + socklen_t clilen; + + clilen = sizeof(clisa); + + rv = recvfrom(fd, static_buf, SBSIZE, 0, (struct sockaddr *) &clisa, + &clilen); + if (rv < 0) { + goto exit; + } + + if (rv < 2) { + stats.net_broken_req++; + goto exit; + } + + req.fd = fd; + req.type = REQTYPE_UDP; + req.clisa = (struct sockaddr *) &clisa; + req.clilen = clilen; + req.mini_reply = udp_mini_reply; + req.reply_err = udp_reply_err; + req.reply_get = udp_reply_get; + req.reply_set = udp_reply_set; + req.reply_del = udp_reply_del; + req.reply_cas = udp_reply_cas; + + /* parse the message */ + parse_message(&req, static_buf, rv); + +exit: + return; +} + + diff --git a/nmdb/udp.h b/nmdb/udp.h new file mode 100644 index 0000000..ea473cf --- /dev/null +++ b/nmdb/udp.h @@ -0,0 +1,10 @@ + +#ifndef _UDP_H +#define _UDP_H + +int udp_init(void); +void udp_close(int fd); +void udp_recv(int fd, short event, void *arg); + +#endif + diff --git a/libnmdb/test1c.c b/tests/c/1.c similarity index 84% rename from libnmdb/test1c.c rename to tests/c/1.c index f4ea111..518313f 100644 --- a/libnmdb/test1c.c +++ b/tests/c/1.c @@ -5,8 +5,9 @@ #include #include -#include "nmdb.h" +#include #include "timer.h" +#include "prototypes.h" int main(int argc, char **argv) @@ -18,7 +19,7 @@ int main(int argc, char **argv) nmdb_t *db; if (argc != 4) { - printf("Usage: test1 TIMES KEY VAL\n"); + printf("Usage: 1-* TIMES KEY VAL\n"); return 1; } @@ -28,16 +29,18 @@ int main(int argc, char **argv) val = (unsigned char *) argv[3]; vsize = strlen((char *) val); - db = nmdb_init(-1); + db = nmdb_init(); if (db == NULL) { perror("nmdb_init() failed"); return 1; } + NADDSRV(db); + printf("set... "); timer_start(); for (i = 0; i < times; i++) { - r = nmdb_cache_set(db, key, ksize, val, vsize); + r = NSET(db, key, ksize, val, vsize); if (r < 0) { perror("Set"); return 1; @@ -50,7 +53,7 @@ int main(int argc, char **argv) printf("get... "); timer_start(); for (i = 0; i < times; i++) { - r = nmdb_cache_get(db, key, ksize, gval, vsize); + r = NGET(db, key, ksize, gval, vsize); if (r < 0) { perror("Get"); return 1; @@ -67,7 +70,7 @@ int main(int argc, char **argv) printf("del... "); timer_start(); for (i = 0; i < times; i++) { - r = nmdb_cache_del(db, key, ksize); + r = NDEL(db, key, ksize); if (r < 0) { perror("Del"); return 1; diff --git a/libnmdb/test2c.c b/tests/c/2.c similarity index 87% rename from libnmdb/test2c.c rename to tests/c/2.c index 1422db9..79a0de1 100644 --- a/libnmdb/test2c.c +++ b/tests/c/2.c @@ -5,8 +5,9 @@ #include #include -#include "nmdb.h" +#include #include "timer.h" +#include "prototypes.h" int main(int argc, char **argv) @@ -18,7 +19,7 @@ int main(int argc, char **argv) nmdb_t *db; if (argc != 4) { - printf("Usage: test2 TIMES KSIZE VSIZE\n"); + printf("Usage: 2-* TIMES KSIZE VSIZE\n"); return 1; } @@ -44,17 +45,19 @@ int main(int argc, char **argv) return 1; } - db = nmdb_init(-1); + db = nmdb_init(); if (db == NULL) { perror("nmdb_init() failed"); return 1; } + NADDSRV(db); + timer_start(); for (i = 0; i < times; i++) { * (int *) key = i; * (int *) val = i; - r = nmdb_cache_set(db, key, ksize, val, vsize); + r = NSET(db, key, ksize, val, vsize); if (r < 0) { perror("Set"); return 1; @@ -68,7 +71,7 @@ int main(int argc, char **argv) timer_start(); for (i = 0; i < times; i++) { * (int *) key = i; - r = nmdb_cache_get(db, key, ksize, val, vsize); + r = NGET(db, key, ksize, val, vsize); if (r < 0) { perror("Get"); return 1; @@ -82,7 +85,7 @@ int main(int argc, char **argv) timer_start(); for (i = 0; i < times; i++) { * (int *) key = i; - r = nmdb_cache_del(db, key, ksize); + r = NDEL(db, key, ksize); if (r < 0) { perror("Del"); return 1; diff --git a/libnmdb/test2d.c b/tests/c/3.c similarity index 64% copy from libnmdb/test2d.c copy to tests/c/3.c index d24b748..b629254 100644 --- a/libnmdb/test2d.c +++ b/tests/c/3.c @@ -5,20 +5,21 @@ #include #include -#include "nmdb.h" +#include #include "timer.h" +#include "prototypes.h" int main(int argc, char **argv) { int i, r, times; unsigned char *key, *val; - size_t ksize, vsize; - unsigned long s_elapsed, g_elapsed, d_elapsed, misses = 0; + size_t ksize, vsize, bsize; + unsigned long elapsed, misses = 0; nmdb_t *db; if (argc != 4) { - printf("Usage: test2 TIMES KSIZE VSIZE\n"); + printf("Usage: 3-* TIMES KSIZE VSIZE\n"); return 1; } @@ -36,60 +37,51 @@ int main(int argc, char **argv) key = malloc(ksize); memset(key, 0, ksize); - val = malloc(vsize); - memset(val, 0, vsize); + val = malloc(70 * 1024); + bsize = 70 * 1024; + memset(val, 0, bsize); if (key == NULL || val == NULL) { perror("Error: malloc()"); return 1; } - db = nmdb_init(-1); + db = nmdb_init(); if (db == NULL) { perror("nmdb_init() failed"); return 1; } + NADDSRV(db); + timer_start(); for (i = 0; i < times; i++) { * (int *) key = i; * (int *) val = i; - r = nmdb_set(db, key, ksize, val, vsize); + r = NSET(db, key, ksize, val, vsize); if (r < 0) { perror("Set"); return 1; } - } - s_elapsed = timer_stop(); - memset(key, 0, ksize); - free(val); - val = malloc(70 * 1024); - timer_start(); - for (i = 0; i < times; i++) { * (int *) key = i; - r = nmdb_get(db, key, ksize, val, vsize); + r = NGET(db, key, ksize, val, bsize); if (r < 0) { perror("Get"); return 1; } else if (r == 0) { misses++; } - } - g_elapsed = timer_stop(); - free(val); - timer_start(); - for (i = 0; i < times; i++) { * (int *) key = i; - r = nmdb_del(db, key, ksize); + r = NDEL(db, key, ksize); if (r < 0) { perror("Del"); return 1; } } - d_elapsed = timer_stop(); - printf("%lu %lu %lu %lu\n", s_elapsed, g_elapsed, d_elapsed, misses); + elapsed = timer_stop(); + printf("%lu %lu\n", elapsed, misses); free(key); nmdb_free(db); diff --git a/tests/c/del.c b/tests/c/del.c new file mode 100644 index 0000000..6bd5607 --- /dev/null +++ b/tests/c/del.c @@ -0,0 +1,70 @@ + +#include +#include +#include +#include +#include + +#include +#include "timer.h" +#include "prototypes.h" + + +int main(int argc, char **argv) +{ + int i, r, times; + unsigned char *key; + size_t ksize; + unsigned long d_elapsed; + nmdb_t *db; + + if (argc != 3) { + printf("Usage: del-* TIMES KSIZE\n"); + return 1; + } + + times = atoi(argv[1]); + ksize = atoi(argv[2]); + if (times < 1) { + printf("Error: TIMES must be >= 1\n"); + return 1; + } + if (ksize < sizeof(int)) { + printf("Error: KSIZE must be >= sizeof(int)\n"); + return 1; + } + + key = malloc(ksize); + memset(key, 0, ksize); + + if (key == NULL) { + perror("Error: malloc()"); + return 1; + } + + db = nmdb_init(); + if (db == NULL) { + perror("nmdb_init() failed"); + return 1; + } + + NADDSRV(db); + + timer_start(); + for (i = 0; i < times; i++) { + * (int *) key = i; + r = NDEL(db, key, ksize); + if (r < 0) { + perror("Del"); + return 1; + } + } + d_elapsed = timer_stop(); + printf("%lu\n", d_elapsed); + + free(key); + nmdb_free(db); + + return 0; +} + diff --git a/libnmdb/test2d.c b/tests/c/get.c similarity index 50% copy from libnmdb/test2d.c copy to tests/c/get.c index d24b748..829e0d3 100644 --- a/libnmdb/test2d.c +++ b/tests/c/get.c @@ -5,8 +5,9 @@ #include #include -#include "nmdb.h" +#include #include "timer.h" +#include "prototypes.h" int main(int argc, char **argv) @@ -14,28 +15,28 @@ int main(int argc, char **argv) int i, r, times; unsigned char *key, *val; size_t ksize, vsize; - unsigned long s_elapsed, g_elapsed, d_elapsed, misses = 0; + unsigned long g_elapsed, misses = 0; nmdb_t *db; - if (argc != 4) { - printf("Usage: test2 TIMES KSIZE VSIZE\n"); + if (argc != 3) { + printf("Usage: get-* TIMES KSIZE\n"); return 1; } times = atoi(argv[1]); ksize = atoi(argv[2]); - vsize = atoi(argv[3]); if (times < 1) { printf("Error: TIMES must be >= 1\n"); return 1; } - if (ksize < sizeof(int) || vsize < sizeof(int)) { - printf("Error: KSIZE and VSIZE must be >= sizeof(int)\n"); + if (ksize < sizeof(int)) { + printf("Error: KSIZE must be >= sizeof(int)\n"); return 1; } key = malloc(ksize); memset(key, 0, ksize); + vsize = 70 * 1024; val = malloc(vsize); memset(val, 0, vsize); @@ -44,31 +45,18 @@ int main(int argc, char **argv) return 1; } - db = nmdb_init(-1); + db = nmdb_init(); if (db == NULL) { perror("nmdb_init() failed"); return 1; } - timer_start(); - for (i = 0; i < times; i++) { - * (int *) key = i; - * (int *) val = i; - r = nmdb_set(db, key, ksize, val, vsize); - if (r < 0) { - perror("Set"); - return 1; - } - } - s_elapsed = timer_stop(); + NADDSRV(db); - memset(key, 0, ksize); - free(val); - val = malloc(70 * 1024); timer_start(); for (i = 0; i < times; i++) { * (int *) key = i; - r = nmdb_get(db, key, ksize, val, vsize); + r = NGET(db, key, ksize, val, vsize); if (r < 0) { perror("Get"); return 1; @@ -77,21 +65,11 @@ int main(int argc, char **argv) } } g_elapsed = timer_stop(); - free(val); - timer_start(); - for (i = 0; i < times; i++) { - * (int *) key = i; - r = nmdb_del(db, key, ksize); - if (r < 0) { - perror("Del"); - return 1; - } - } - d_elapsed = timer_stop(); - printf("%lu %lu %lu %lu\n", s_elapsed, g_elapsed, d_elapsed, misses); + printf("%lu m:%lu\n", g_elapsed, misses); free(key); + free(val); nmdb_free(db); return 0; diff --git a/tests/c/make.sh b/tests/c/make.sh new file mode 100755 index 0000000..a38525c --- /dev/null +++ b/tests/c/make.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +set -e + +USAGE="\ +Use: build.sh [build|debug_build|strict_build|profile_build|clean] +" + + +ALLCF="-D_XOPEN_SOURCE=500 -fPIC -std=c99 -Wall -O3" + +case "$1" in + "build" ) + # defaults are just fine for build + ;; + "debug_build" ) + ALLCF="$ALLCF -g" + ;; + "strict_build" ) + ALLCF="$ALLCF -ansi -pedantic" + ;; + "profile_build" ) + ALLCF="$ALLCF -g -pg -fprofile-arcs -ftest-coverage" + ;; + "clean" ) + CLEAN=1 + ;; + "help" | "--help" | "-h" | "" ) + echo $USAGE + exit 1 + ;; +esac; + + +for p in TIPC TCP UDP MULT; do + for v in NORMAL CACHE SYNC; do + OP=`echo $p-$v | tr '[A-Z]' '[a-z]'` + TF="-DUSE_$p=1 -DUSE_$v=1" + + echo " * $OP" + for t in 1 2 3 "set" "get" "del"; do + if [ "$CLEAN" == 1 ]; then + rm -vf $t-$OP + else + cc -lnmdb $ALLCF $TF -o $t-$OP $t.c + fi + done + done +done + diff --git a/tests/c/prototypes.h b/tests/c/prototypes.h new file mode 100644 index 0000000..e0efcc2 --- /dev/null +++ b/tests/c/prototypes.h @@ -0,0 +1,41 @@ + +#ifndef _TEST_PROTOTYPES_H +#define _TEST_PROTOTYPES_H + + +#if USE_NORMAL + #define NGET(...) nmdb_get(__VA_ARGS__) + #define NSET(...) nmdb_set(__VA_ARGS__) + #define NDEL(...) nmdb_del(__VA_ARGS__) + #define NCAS(...) nmdb_cas(__VA_ARGS__) +#elif USE_CACHE + #define NGET(...) nmdb_cache_get(__VA_ARGS__) + #define NSET(...) nmdb_cache_set(__VA_ARGS__) + #define NDEL(...) nmdb_cache_del(__VA_ARGS__) + #define NCAS(...) nmdb_cache_cas(__VA_ARGS__) +#elif USE_SYNC + #define NGET(...) nmdb_get(__VA_ARGS__) + #define NSET(...) nmdb_set_sync(__VA_ARGS__) + #define NDEL(...) nmdb_del_sync(__VA_ARGS__) + #define NCAS(...) nmdb_cas(__VA_ARGS__) +#endif + + +#if USE_TCP + #define NADDSRV(db) nmdb_add_tcp_server(db, "localhost", -1) +#elif USE_UDP + #define NADDSRV(db) nmdb_add_udp_server(db, "localhost", -1) +#elif USE_TIPC + #define NADDSRV(db) nmdb_add_tipc_server(db, -1) +#elif USE_MULT + #define NADDSRV(db) \ + do { \ + nmdb_add_tipc_server(db, -1); \ + nmdb_add_tcp_server(db, "localhost", -1); \ + nmdb_add_udp_server(db, "localhost", -1); \ + } while (0) +#endif + + +#endif + diff --git a/libnmdb/test2d.c b/tests/c/set.c similarity index 58% rename from libnmdb/test2d.c rename to tests/c/set.c index d24b748..7e556c8 100644 --- a/libnmdb/test2d.c +++ b/tests/c/set.c @@ -5,8 +5,9 @@ #include #include -#include "nmdb.h" +#include #include "timer.h" +#include "prototypes.h" int main(int argc, char **argv) @@ -14,11 +15,11 @@ int main(int argc, char **argv) int i, r, times; unsigned char *key, *val; size_t ksize, vsize; - unsigned long s_elapsed, g_elapsed, d_elapsed, misses = 0; + unsigned long s_elapsed; nmdb_t *db; if (argc != 4) { - printf("Usage: test2 TIMES KSIZE VSIZE\n"); + printf("Usage: set-* TIMES KSIZE VSIZE\n"); return 1; } @@ -44,17 +45,19 @@ int main(int argc, char **argv) return 1; } - db = nmdb_init(-1); + db = nmdb_init(); if (db == NULL) { perror("nmdb_init() failed"); return 1; } + NADDSRV(db); + timer_start(); for (i = 0; i < times; i++) { * (int *) key = i; * (int *) val = i; - r = nmdb_set(db, key, ksize, val, vsize); + r = NSET(db, key, ksize, val, vsize); if (r < 0) { perror("Set"); return 1; @@ -62,36 +65,10 @@ int main(int argc, char **argv) } s_elapsed = timer_stop(); - memset(key, 0, ksize); - free(val); - val = malloc(70 * 1024); - timer_start(); - for (i = 0; i < times; i++) { - * (int *) key = i; - r = nmdb_get(db, key, ksize, val, vsize); - if (r < 0) { - perror("Get"); - return 1; - } else if (r == 0) { - misses++; - } - } - g_elapsed = timer_stop(); - free(val); - - timer_start(); - for (i = 0; i < times; i++) { - * (int *) key = i; - r = nmdb_del(db, key, ksize); - if (r < 0) { - perror("Del"); - return 1; - } - } - d_elapsed = timer_stop(); - printf("%lu %lu %lu %lu\n", s_elapsed, g_elapsed, d_elapsed, misses); + printf("%lu\n", s_elapsed); free(key); + free(val); nmdb_free(db); return 0; diff --git a/libnmdb/timer.h b/tests/c/timer.h similarity index 90% rename from libnmdb/timer.h rename to tests/c/timer.h index 75315c5..a0a8191 100644 --- a/libnmdb/timer.h +++ b/tests/c/timer.h @@ -25,11 +25,11 @@ static struct timeval tv_s, tv_e; -static void timer_start() { +static void timer_start(void) { gettimeofday(&tv_s, NULL); } -static unsigned long timer_stop() { +static unsigned long timer_stop(void) { gettimeofday(&tv_e, NULL); return (tv_e.tv_sec - tv_s.tv_sec) * 1000000ul + (tv_e.tv_usec - tv_s.tv_usec); diff --git a/tests/python/random1.py b/tests/python/random1.py new file mode 100755 index 0000000..0c53d28 --- /dev/null +++ b/tests/python/random1.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python + +import sys +import nmdb +from random import randint, choice + + +class Mismatch (Exception): + pass + + +# network db +ndb = nmdb.DB() +ndb.add_tipc_server() +ndb.add_tcp_server('localhost') +ndb.add_udp_server('localhost') + +# local db +ldb = {} + +# history of each key +history = {} + +# check decorator +def checked(f): + def newf(k, *args, **kwargs): + try: + return f(k, *args, **kwargs) + except: + print history[k] + raise + newf.__name__ = f.__name__ + return newf + + +# operations +@checked +def set(k, v): + ndb[k] = v + ldb[k] = v + if k not in history: + history[k] = [] + history[k].append((set, k, v)) + +@checked +def get(k): + n = ndb[k] + l = ldb[k] + if l != n: + raise Mismatch, (n, l) + history[k].append((get, k)) + return n + +@checked +def delete(k): + del ndb[k] + del ldb[k] + history[k].append((delete, k)) + +@checked +def cas(k, ov, nv): + prel = ldb[k] + pren = ndb[k] + n = ndb.cas(k, ov, nv) + if not ldb.has_key(k): + l = 0 + elif ldb[k] == ov: + ldb[k] = nv + l = 2 + else: + l = 1 + if n != l: + print k, ldb[k], ndb[k] + print prel, pren + print history[k] + raise Mismatch, (n, l) + history[k].append((cas, k, ov, nv)) + return n + + +def check(): + for k in ldb.keys(): + try: + n = ndb[k] + l = ldb[k] + except: + print history[k] + raise Mismatch, (n, l) + + if n != n: + print history[k] + raise Mismatch, (n, l) + + +# Use integers because the normal random() generates floating point numbers, +# and they can mess up comparisons because of architecture details. +def getrand(): + return randint(0, 1000000000000000000) + + +if __name__ == '__main__': + if len(sys.argv) != 2: + print 'Use: random1.py number_of_keys' + sys.exit(1) + + nkeys = int(sys.argv[1]) + + # fill all the keys + print 'populate' + for i in xrange(nkeys): + set(getrand(), getrand()) + + lkeys = ldb.keys() + + # operate on them a bit + print 'random operations' + operations = ('set', 'delete', 'cas0', 'cas1') + for i in xrange(nkeys / 2): + op = choice(operations) + k = choice(lkeys) + if op == 'set': + set(k, getrand()) + elif op == 'delete': + delete(k) + lkeys.remove(k) + elif op == 'cas0': + # unsucessful cas + cas(k, -1, -1) + elif op == 'cas1': + # successful cas + cas(k, ldb[k], getrand()) + + print 'check' + check() + + print 'delete' + for k in lkeys: + delete(k) + + print 'check' + check() + + sys.exit(0) + + + +