Kyoto Cabinet: a straightforward implementation of DBM

Introduction

Kyoto Cabinet is a library of routines for managing a database. The database is a simple data file containing records, each is a pair of a key and a value. Every key and value is serial bytes with variable length. Both binary data and character string can be used as a key and a value. Each key must be unique within a database. There is neither concept of data tables nor data types. Records are organized in hash table or B+ tree.

The following access methods are provided to the database: storing a record with a key and a value, deleting a record by a key, retrieving a record by a key. Moreover, traversal access to every key are provided. These access methods are similar to ones of DBM (or its followers: NDBM and GDBM) library defined in the UNIX standard. Kyoto Cabinet is an alternative for DBM because of its higher performance.

Each operation of hash database has the time complexity of O(1). So, in theory, the performance is consant regardless of the scale of the database. In practice, the performance is determined by the speed of the main memory or the storage device. If the size of the database is less than the capacity of the main memory, the performance will seem on-memory speed which is fastar than std::map of STL. Of cource, the database size can be greater than the capacity of the main memory and the upper limit is 8 exabytes. Even in that case, each operation needs only one or two seeking of the storage device.

Each operation of B+ tree has the time complexity of O(log N). So, in theory, the performance is logarithmic about the scale of the database. Although the performance of random access of B+ tree is slower than that of hash database, B+ tree supports sequential access in order of the keys, which realizes forward matching search for strings and range search for integers. The performance of sequential access is much faster than that of random access.

As the API is based on object-oriented design, hash database and B+ tree database have same methods which inherited from the upper abstract class. Prototype database by containers of STL and cache database with LRU deletion algorithm are also provided under the same base class. All databases have practical utility methods related to transaction and cursor. Programs for command line interface are also included in the package.

The following classes are most important.

See the project homepage ( http://1978th.net/kyotocabinet/ ) for detail.

Example

The following code is an example to use a file hash database.

#include <kchashdb.h>

using namespace std;
using namespace kyotocabinet;

// main routine
int main(int argc, char** argv) {

  // create the database object
  HashDB db;

  // open the database
  if (!db.open("casket.kch", HashDB::OWRITER | HashDB::OCREATE)) {
    cerr << "open error: " << db.error().name() << endl;
  }

  // store records
  if (!db.set("foo", "hop") ||
      !db.set("bar", "step") ||
      !db.set("baz", "jump")) {
    cerr << "set error: " << db.error().name() << endl;
  }

  // retrieve a record
  string* value = db.get("foo");
  if (value) {
    cout << *value << endl;
    delete value;
  } else {
    cerr << "get error: " << db.error().name() << endl;
  }

  // traverse records
  DB::Cursor* cur = db.cursor();
  cur->jump();
  pair<string, string>* rec;
  while ((rec = cur->get_pair(true)) != NULL) {
    cout << rec->first << ":" << rec->second << endl;
    delete rec;
  }
  delete cur;

  // close the database
  if (!db.close()) {
    cerr << "close error: " << db.error().name() << endl;
  }

  return 0;
}

The following code is a more complex example, which uses the Visitor pattern.

#include <kchashdb.h>

using namespace std;
using namespace kyotocabinet;

// main routine
int main(int argc, char** argv) {

  // create the database object
  HashDB db;

  // open the database
  if (!db.open("casket.kch", HashDB::OREADER)) {
    cerr << "open error: " << db.error().name() << endl;
  }

  // define the visitor
  class VisitorImpl : public DB::Visitor {
    // call back function for an existing record
    const char* visit_full(const char* kbuf, size_t ksiz,
                           const char* vbuf, size_t vsiz, size_t *sp) {
      cout << string(kbuf, ksiz) << ":" << string(vbuf, vsiz) << endl;
      return NOP;
    }
    // call back function for an empty record space
    const char* visit_empty(const char* kbuf, size_t ksiz, size_t *sp) {
      cerr << string(kbuf, ksiz) << " is missing" << endl;
      return NOP;
    }
  } visitor;

  // retrieve a record with visitor
  if (!db.accept("foo", 3, &visitor, false) ||
      !db.accept("dummy", 5, &visitor, false)) {
    cerr << "accept error: " << db.error().name() << endl;
  }

  // traverse records with visitor
  if (!db.iterate(&visitor, false)) {
    cerr << "iterate error: " << db.error().name() << endl;
  }

  // close the database
  if (!db.close()) {
    cerr << "close error: " << db.error().name() << endl;
  }

  return 0;
}

The C language binding is also provided as a wrapper of the polymorphic database API. The following code is an example.

#include <kclangc.h>

/* call back function for an existing record */
const char* visitfull(const char* kbuf, size_t ksiz,
                      const char* vbuf, size_t vsiz, size_t *sp, void* opq) {
  fwrite(kbuf, 1, ksiz, stdout);
  printf(":");
  fwrite(vbuf, 1, vsiz, stdout);
  printf("\n");
  return KCVISNOP;
}

/* call back function for an empty record space */
const char* visitempty(const char* kbuf, size_t ksiz, size_t *sp, void* opq) {
  fwrite(kbuf, 1, ksiz, stdout);
  printf(" is missing\n");
  return KCVISNOP;
}

/* main routine */
int main(int argc, char** argv) {
  KCDB* db;
  KCCUR* cur;
  char *kbuf, *vbuf;
  size_t ksiz, vsiz;
  const char *cvbuf;

  /* create the database object */
  db = kcdbnew();

  /* open the database */
  if (!kcdbopen(db, "casket.kch", KCOWRITER | KCOCREATE)) {
    fprintf(stderr, "open error: %s\n", kcecodename(kcdbecode(db)));
  }

  /* store records */
  if (!kcdbset(db, "foo", 3, "hop", 3) ||
      !kcdbset(db, "bar", 3, "step", 4) ||
      !kcdbset(db, "baz", 3, "jump", 4)) {
    fprintf(stderr, "set error: %s\n", kcecodename(kcdbecode(db)));
  }

  /* retrieve a record */
  vbuf = kcdbget(db, "foo", 3, &vsiz);
  if (vbuf) {
    printf("%s\n", vbuf);
    kcfree(vbuf);
  } else {
    fprintf(stderr, "get error: %s\n", kcecodename(kcdbecode(db)));
  }

  /* traverse records */
  cur = kcdbcursor(db);
  kccurjump(cur);
  while ((kbuf = kccurget(cur, &ksiz, &cvbuf, &vsiz, 1)) != NULL) {
    printf("%s:%s\n", kbuf, cvbuf);
    kcfree(kbuf);
  }
  kccurdel(cur);

  /* retrieve a record with visitor */
  if (!kcdbaccept(db, "foo", 3, visitfull, visitempty, NULL, 0) ||
      !kcdbaccept(db, "dummy", 5, visitfull, visitempty, NULL, 0)) {
    fprintf(stderr, "accept error: %s\n", kcecodename(kcdbecode(db)));
  }

  /* traverse records with visitor */
  if (!kcdbiterate(db, visitfull, NULL, 0)) {
    fprintf(stderr, "iterate error: %s\n", kcecodename(kcdbecode(db)));
  }

  /* close the database */
  if (!kcdbclose(db)) {
    fprintf(stderr, "close error: %s\n", kcecodename(kcdbecode(db)));
  }

  /* delete the database object */
  kcdbdel(db);

  return 0;
}
Generated on Tue May 4 10:08:02 2010 for Kyoto Cabinet by  doxygen 1.6.3