git » nmdb » commit d0a45d9

nmdb: Add LevelDB support

author Alberto Bertogli
2012-09-12 22:05:30 UTC
committer Alberto Bertogli
2012-09-12 23:35:23 UTC
parent 5c3206a98cc033881e8513220bfc56f95eeea58d

nmdb: Add LevelDB support

LevelDB is a new key-value storage from Google, see
https://code.google.com/p/leveldb/ for more details.

This patch adds support for using it as a backend.

Signed-off-by: Alberto Bertogli <albertito@blitiri.com.ar>

nmdb/Makefile +7 -1
nmdb/be-leveldb.c +215 -0
nmdb/be.c +7 -0
nmdb/be.h +12 -1
tests/coverage/coverage +4 -4

diff --git a/nmdb/Makefile b/nmdb/Makefile
index 4f7f870..49736c3 100644
--- a/nmdb/Makefile
+++ b/nmdb/Makefile
@@ -16,6 +16,8 @@ BE_ENABLE_TC := $(shell if `pkg-config --exists tokyocabinet`; \
 	then echo 1; else echo 0; fi)
 BE_ENABLE_TDB := $(shell if `pkg-config --exists tdb`; \
 	then echo 1; else echo 0; fi)
+BE_ENABLE_LEVELDB := $(shell if echo "\#include <leveldb/c.h>" | \
+		$(CPP) - > /dev/null 2>&1; then echo 1; else echo 0; fi)
 BE_ENABLE_NULL := 1
 
 
@@ -29,6 +31,7 @@ ALL_CFLAGS += -DENABLE_TIPC=$(ENABLE_TIPC) \
 		-DBE_ENABLE_BDB=$(BE_ENABLE_BDB) \
 		-DBE_ENABLE_TC=$(BE_ENABLE_TC) \
 		-DBE_ENABLE_TDB=$(BE_ENABLE_TDB) \
+		-DBE_ENABLE_LEVELDB=$(BE_ENABLE_LEVELDB) \
 		-DBE_ENABLE_NULL=$(BE_ENABLE_NULL) \
 
 
@@ -50,7 +53,7 @@ PREFIX=/usr/local
 
 
 OBJS = cache.o dbloop.o queue.o log.o net.o netutils.o parse.o stats.o main.o \
-       be.o be-bdb.o be-null.o be-qdbm.o be-tc.o be-tdb.o
+       be.o be-bdb.o be-null.o be-qdbm.o be-tc.o be-tdb.o be-leveldb.o
 LIBS = -levent -lpthread -lrt
 
 
@@ -94,6 +97,9 @@ ifeq ($(BE_ENABLE_TDB), 1)
 	ALL_CFLAGS += `pkg-config tdb --cflags`
 	LIBS += `pkg-config tdb --libs`
 endif
+ifeq ($(BE_ENABLE_LEVELDB), 1)
+	LIBS += -lleveldb
+endif
 ifeq ($(BE_ENABLE_NULL), 1)
 endif
 
diff --git a/nmdb/be-leveldb.c b/nmdb/be-leveldb.c
new file mode 100644
index 0000000..aad0f88
--- /dev/null
+++ b/nmdb/be-leveldb.c
@@ -0,0 +1,215 @@
+
+#if BE_ENABLE_LEVELDB
+
+#include <leveldb/c.h>	/* LevelDB C API */
+#include <string.h>	/* memcpy() */
+#include <stdlib.h>
+
+#include "be.h"
+
+
+int xleveldb_set(struct db_conn *db, const unsigned char *key, size_t ksize,
+		unsigned char *val, size_t vsize);
+int xleveldb_get(struct db_conn *db, const unsigned char *key, size_t ksize,
+		unsigned char *val, size_t *vsize);
+int xleveldb_del(struct db_conn *db, const unsigned char *key, size_t ksize);
+int xleveldb_close(struct db_conn *db);
+int xleveldb_firstkey(struct db_conn *db, unsigned char *key, size_t *ksize);
+int xleveldb_nextkey(struct db_conn *db,
+		const unsigned char *key, size_t ksize,
+		unsigned char *nextkey, size_t *nksize);
+
+struct db_conn *xleveldb_open(const char *name, int flags)
+{
+	struct db_conn *db;
+	leveldb_options_t *options;
+	leveldb_t *level_db;
+
+	options = leveldb_options_create();
+	if (options == NULL)
+		return NULL;
+
+	leveldb_options_set_create_if_missing(options, 1);
+
+	level_db = leveldb_open(options, name, NULL);
+
+	leveldb_options_destroy(options);
+
+	if (level_db == NULL)
+		return NULL;
+
+	db = malloc(sizeof(struct db_conn));
+	if (db == NULL) {
+		leveldb_close(level_db);
+		return NULL;
+	}
+
+	db->conn = level_db;
+	db->set = xleveldb_set;
+	db->get = xleveldb_get;
+	db->del = xleveldb_del;
+	db->firstkey = xleveldb_firstkey;
+	db->nextkey = xleveldb_nextkey;
+	db->close = xleveldb_close;
+
+	return db;
+}
+
+
+int xleveldb_close(struct db_conn *db)
+{
+	leveldb_close(db->conn);
+	free(db);
+	return 1;
+}
+
+
+int xleveldb_set(struct db_conn *db, const unsigned char *key, size_t ksize,
+		unsigned char *val, size_t vsize)
+{
+	leveldb_writeoptions_t *options = leveldb_writeoptions_create();
+	char *err, *origerr;
+
+	err = origerr = malloc(1);
+	leveldb_put(db->conn, options,
+			(const char *) key, ksize,
+			(const char *) val, vsize, &err);
+	free(err);
+
+	leveldb_writeoptions_destroy(options);
+
+	return err == origerr;
+}
+
+
+int xleveldb_get(struct db_conn *db, const unsigned char *key, size_t ksize,
+		unsigned char *val, size_t *vsize)
+{
+	int rv;
+	char *db_val = NULL;
+	size_t db_vsize;
+	leveldb_readoptions_t *options = leveldb_readoptions_create();
+	char *err, *origerr;
+
+	err = origerr = malloc(1);
+	db_val = leveldb_get(db->conn, options,
+			(const char *) key, ksize, &db_vsize, &err);
+
+	free(err);
+	leveldb_readoptions_destroy(options);
+
+	if (err != origerr || db_val == NULL || db_vsize > *vsize) {
+		rv = 0;
+		goto exit;
+	}
+
+	*vsize = db_vsize;
+	memcpy(val, db_val, db_vsize);
+	rv = 1;
+
+exit:
+	free(db_val);
+	return rv;
+}
+
+
+int xleveldb_del(struct db_conn *db, const unsigned char *key, size_t ksize)
+{
+	leveldb_writeoptions_t *options = leveldb_writeoptions_create();
+	char *err, *origerr;
+
+	err = origerr = malloc(1);
+	leveldb_delete(db->conn, options,
+			(const char *) key, ksize, &err);
+	free(err);
+
+	leveldb_writeoptions_destroy(options);
+
+	return err == origerr;
+}
+
+
+int xleveldb_firstkey(struct db_conn *db, unsigned char *key, size_t *ksize)
+{
+	int rv = 0;
+	const char *db_key;
+	size_t db_ksize;
+
+	leveldb_readoptions_t *options = leveldb_readoptions_create();
+	leveldb_iterator_t *it = leveldb_create_iterator(db->conn, options);
+
+	leveldb_iter_seek_to_first(it);
+	if (! leveldb_iter_valid(it)) {
+		rv = 0;
+		goto exit;
+	}
+
+	db_key = leveldb_iter_key(it, &db_ksize);
+
+	if (db_key == NULL || db_ksize > *ksize) {
+		rv = 0;
+		goto exit;
+	}
+
+	*ksize = db_ksize;
+	memcpy(key, db_key, db_ksize);
+	rv = 1;
+
+exit:
+	leveldb_readoptions_destroy(options);
+	leveldb_iter_destroy(it);
+	return rv;
+}
+
+
+int xleveldb_nextkey(struct db_conn *db,
+		const unsigned char *key, size_t ksize,
+		unsigned char *nextkey, size_t *nksize)
+{
+	int rv;
+	const char *db_nextkey;
+	size_t db_nksize = 0;
+
+	leveldb_readoptions_t *options = leveldb_readoptions_create();
+	leveldb_iterator_t *it = leveldb_create_iterator(db->conn, options);
+
+	leveldb_iter_seek(it, (const char *) key, ksize);
+	if (! leveldb_iter_valid(it)) {
+		rv = 0;
+		goto exit;
+	}
+
+	leveldb_iter_next(it);
+	if (! leveldb_iter_valid(it)) {
+		rv = 0;
+		goto exit;
+	}
+
+	db_nextkey = leveldb_iter_key(it, &db_nksize);
+
+	if (db_nextkey == NULL || db_nksize > *nksize) {
+		rv = 0;
+		goto exit;
+	}
+
+	*nksize = db_nksize;
+	memcpy(nextkey, db_nextkey, db_nksize);
+	rv = 1;
+
+exit:
+	leveldb_iter_destroy(it);
+	leveldb_readoptions_destroy(options);
+	return rv;
+}
+
+#else
+
+#include <stddef.h>	/* NULL */
+
+struct db_conn *leveldb_open(const char *name, int flags)
+{
+	return NULL;
+}
+
+#endif
+
diff --git a/nmdb/be.c b/nmdb/be.c
index d1349e4..59a56c4 100644
--- a/nmdb/be.c
+++ b/nmdb/be.c
@@ -7,6 +7,7 @@ struct db_conn *qdbm_open(const char *name, int flags);
 struct db_conn *bdb_open(const char *name, int flags);
 struct db_conn *tc_open(const char *name, int flags);
 struct db_conn *xtdb_open(const char *name, int flags);
+struct db_conn *xleveldb_open(const char *name, int flags);
 struct db_conn *null_open(const char *name, int flags);
 
 
@@ -21,6 +22,8 @@ struct db_conn *db_open(enum backend_type type, const char *name, int flags)
 		return tc_open(name, flags);
 	case BE_TDB:
 		return xtdb_open(name, flags);
+	case BE_LEVELDB:
+		return xleveldb_open(name, flags);
 	case BE_NULL:
 		return null_open(name, flags);
 	default:
@@ -38,6 +41,8 @@ enum backend_type be_type_from_str(const char *name)
 		return BE_ENABLE_TC ? BE_TC : BE_UNSUPPORTED;
 	if (strcmp(name, "tdb") == 0)
 		return BE_ENABLE_TDB ? BE_TDB : BE_UNSUPPORTED;
+	if (strcmp(name, "leveldb") == 0)
+		return BE_ENABLE_LEVELDB ? BE_LEVELDB : BE_UNSUPPORTED;
 	if (strcmp(name, "null") == 0)
 		return BE_ENABLE_NULL ? BE_NULL : BE_UNSUPPORTED;
 	return BE_UNKNOWN;
@@ -54,6 +59,8 @@ const char *be_str_from_type(enum backend_type type)
 		return "tc";
 	if (type == BE_TDB)
 		return "tdb";
+	if (type == BE_LEVELDB)
+		return "leveldb";
 	if (type == BE_NULL)
 		return "null";
 	return "unknown";
diff --git a/nmdb/be.h b/nmdb/be.h
index 37a8210..802b9bc 100644
--- a/nmdb/be.h
+++ b/nmdb/be.h
@@ -31,6 +31,7 @@ enum backend_type {
 	BE_BDB,
 	BE_TC,
 	BE_TDB,
+	BE_LEVELDB,
 	BE_NULL,
 };
 
@@ -68,13 +69,20 @@ const char *be_str_from_type(enum backend_type type);
   #define _TDB_SUPP ""
 #endif
 
+#if BE_ENABLE_LEVELDB
+  #define _LEVELDB_SUPP "leveldb "
+#else
+  #define _LEVELDB_SUPP ""
+#endif
+
 #if BE_ENABLE_NULL
   #define _NULL_SUPP "null "
 #else
   #define _NULL_SUPP ""
 #endif
 
-#define SUPPORTED_BE _QDBM_SUPP _BDB_SUPP _TC_SUPP _TDB_SUPP _NULL_SUPP
+#define SUPPORTED_BE \
+	_QDBM_SUPP _BDB_SUPP _TC_SUPP _TDB_SUPP _LEVELDB_SUPP _NULL_SUPP
 
 
 /* Default backend */
@@ -90,6 +98,9 @@ const char *be_str_from_type(enum backend_type type);
 #elif BE_ENABLE_BDB
   #define DEFAULT_BE BE_BDB
   #define DEFAULT_BE_NAME "bdb"
+#elif BE_ENABLE_LEVELDB
+  #define DEFAULT_BE BE_LEVELDB
+  #define DEFAULT_BE_NAME "leveldb"
 #elif BE_ENABLE_NULL
   #warning "using null backend as the default"
   #define DEFAULT_BE BE_NULL
diff --git a/tests/coverage/coverage b/tests/coverage/coverage
index 56c8405..8178db2 100755
--- a/tests/coverage/coverage
+++ b/tests/coverage/coverage
@@ -36,7 +36,7 @@ function run() {
 
 function nmdb() {
 	log "-- nmdb starting:" "$@"
-	rm -f $DB
+	rm -rf $DB
 	./nmdb/nmdb -f -d $DB "$@" >> $LOG 2>> $LOG &
 	sleep 0.2
 }
@@ -49,7 +49,7 @@ function kill_nmdb() {
 
 function nmdb_and_kill() {
 	log "-- nmdb_and_kill starting:" "$@"
-	rm -f $DB
+	rm -rf $DB
 	./nmdb/nmdb -f -d $DB "$@" >> $LOG 2>> $LOG &
 	sleep 0.2
 	killall nmdb >> $LOG 2>> $LOG
@@ -103,9 +103,9 @@ run killall -TERM nmdb
 wait `pidof nmdb`
 
 
-for be in bdb tc qdbm tdb; do
+for be in bdb tc qdbm tdb leveldb; do
 	out "+ backend $be"
-	run rm -f $DB
+	run rm -rf $DB
 	nmdb -b $be
 	if ! pidof nmdb > /dev/null; then
 		out "  unsupported"