From c24de6da5773da5ae431ce9554bde28eaf15a3df Mon Sep 17 00:00:00 2001 From: leitner Date: Fri, 25 Nov 2005 15:02:31 +0000 Subject: [PATCH] add cdb --- CHANGES | 1 + GNUmakefile | 10 ++- Makefile | 11 +++- cdb.h | 37 +++++++++++ cdb/cdb.c | 128 ++++++++++++++++++++++++++++++++++++++ cdb/cdb_datalen.3 | 17 +++++ cdb/cdb_datapos.3 | 17 +++++ cdb/cdb_find.3 | 42 +++++++++++++ cdb/cdb_firstkey.3 | 19 ++++++ cdb/cdb_free.3 | 17 +++++ cdb/cdb_hash.c | 19 ++++++ cdb/cdb_init.3 | 20 ++++++ cdb/cdb_keylen.3 | 16 +++++ cdb/cdb_keypos.3 | 16 +++++ cdb/cdb_make.c | 148 ++++++++++++++++++++++++++++++++++++++++++++ cdb/cdb_nextkey.3 | 19 ++++++ cdb/cdb_read.3 | 20 ++++++ cdb/cdb_successor.3 | 21 +++++++ cdb/cdb_traverse.c | 40 ++++++++++++ cdb_make.h | 39 ++++++++++++ 20 files changed, 652 insertions(+), 5 deletions(-) create mode 100644 cdb.h create mode 100644 cdb/cdb.c create mode 100644 cdb/cdb_datalen.3 create mode 100644 cdb/cdb_datapos.3 create mode 100644 cdb/cdb_find.3 create mode 100644 cdb/cdb_firstkey.3 create mode 100644 cdb/cdb_free.3 create mode 100644 cdb/cdb_hash.c create mode 100644 cdb/cdb_init.3 create mode 100644 cdb/cdb_keylen.3 create mode 100644 cdb/cdb_keypos.3 create mode 100644 cdb/cdb_make.c create mode 100644 cdb/cdb_nextkey.3 create mode 100644 cdb/cdb_read.3 create mode 100644 cdb/cdb_successor.3 create mode 100644 cdb/cdb_traverse.c create mode 100644 cdb_make.h diff --git a/CHANGES b/CHANGES index ba07a6b..f08bfea 100644 --- a/CHANGES +++ b/CHANGES @@ -9,6 +9,7 @@ that. implement Nikola's idea to remove limit number of strings in errmsg add taia_half + add cdb 0.24: fix scan_to_sa (Tim Lorenz) diff --git a/GNUmakefile b/GNUmakefile index 712e5a9..3c5a82f 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -10,7 +10,8 @@ INCLUDEDIR=${prefix}/include MAN3DIR=${prefix}/man/man3 LIBS=byte.a fmt.a scan.a str.a uint.a open.a stralloc.a unix.a socket.a \ -buffer.a mmap.a taia.a tai.a dns.a case.a mult.a array.a io.a textcode.a +buffer.a mmap.a taia.a tai.a dns.a case.a mult.a array.a io.a \ +textcode.a cdb.a all: t $(LIBS) libowfat.a libsocket @@ -53,7 +54,7 @@ endif # to build without diet libc support, use $ make DIET= # see http://www.fefe.de/dietlibc/ for details about the diet libc -VPATH=str:byte:fmt:scan:uint:open:stralloc:unix:socket:buffer:mmap:textcode:taia:tai:dns:case:array:mult:io +VPATH=str:byte:fmt:scan:uint:open:stralloc:unix:socket:buffer:mmap:textcode:taia:tai:dns:case:array:mult:io:cdb BYTE_OBJS=$(patsubst byte/%.c,%.o,$(wildcard byte/*.c)) FMT_OBJS=$(patsubst fmt/%.c,%.o,$(wildcard fmt/*.c)) @@ -74,6 +75,7 @@ CASE_OBJS=$(patsubst case/%.c,%.o,$(wildcard case/*.c)) ARRAY_OBJS=$(patsubst array/%.c,%.o,$(wildcard array/*.c)) MULT_OBJS=$(patsubst mult/%.c,%.o,$(wildcard mult/*.c)) IO_OBJS=$(patsubst io/%.c,%.o,$(wildcard io/*.c)) +CDB_OBJS=$(patsubst cdb/%.c,%.o,$(wildcard cdb/*.c)) $(BYTE_OBJS): byte.h $(FMT_OBJS): fmt.h @@ -92,6 +94,7 @@ $(CASE_OBJS): case.h $(ARRAY_OBJS): uint64.h array.h $(MULT_OBJS): uint64.h uint32.h uint16.h safemult.h $(IO_OBJS): uint64.h array.h io.h io_internal.h taia.h tai.h haveepoll.h havekqueue.h havesigio.h havebsdsf.h havedevpoll.h havesendfile.h +$(CDB_OBJS): cdb.h uint32.h iob_addbuf.o iob_addfile.o iob_new.o iob_reset.o iob_send.o: iob_internal.h iob.h @@ -120,6 +123,7 @@ case.a: $(CASE_OBJS) array.a: $(ARRAY_OBJS) mult.a: $(MULT_OBJS) io.a: $(IO_OBJS) +cdb.a: $(CDB_OBJS) ALL_OBJS=$(DNS_OBJS) $(BYTE_OBJS) $(FMT_OBJS) $(SCAN_OBJS) \ $(STR_OBJS) $(UINT_OBJS) $(OPEN_OBJS) $(STRALLOC_OBJS) $(UNIX_OBJS) \ @@ -155,7 +159,7 @@ dep libsocket havealloca.h INCLUDES=buffer.h byte.h fmt.h ip4.h ip6.h mmap.h scan.h socket.h str.h stralloc.h \ uint16.h uint32.h uint64.h open.h textcode.h tai.h taia.h dns.h iopause.h case.h \ openreadclose.h readclose.h ndelay.h array.h io.h safemult.h iob.h havealloca.h \ -errmsg.h +errmsg.h cdb.h install: libowfat.a install -d $(INCLUDEDIR) $(MAN3DIR) $(LIBDIR) diff --git a/Makefile b/Makefile index a3be572..214cb34 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,8 @@ INCLUDEDIR=${prefix}/include MAN3DIR=${prefix}/man/man3 LIBS=byte.a fmt.a scan.a str.a uint.a open.a stralloc.a unix.a socket.a \ -buffer.a mmap.a taia.a tai.a dns.a case.a mult.a array.a io.a textcode.a +buffer.a mmap.a taia.a tai.a dns.a case.a mult.a array.a io.a \ +textcode.a cdb.a all: t $(LIBS) libowfat.a libsocket @@ -116,6 +117,10 @@ case_diffs.o: case/case_diffs.c case.h case_lowerb.o: case/case_lowerb.c case.h case_lowers.o: case/case_lowers.c case.h case_starts.o: case/case_starts.c case.h +cdb.o: cdb/cdb.c byte.h cdb.h uint32.h +cdb_hash.o: cdb/cdb_hash.c cdb.h uint32.h +cdb_make.o: cdb/cdb_make.c cdb.h uint32.h cdb_make.h buffer.h +cdb_traverse.o: cdb/cdb_traverse.c cdb.h uint32.h dns_dfd.o: dns/dns_dfd.c byte.h dns.h stralloc.h iopause.h taia.h tai.h \ uint64.h dns_domain.o: dns/dns_domain.c case.h byte.h dns.h stralloc.h iopause.h \ @@ -523,6 +528,7 @@ MULT_OBJS=imult16.o imult32.o imult64.o umult16.o umult32.o umult64.o ARRAY_OBJS=array_allocate.o array_bytes.o array_cat.o array_cat0.o array_catb.o array_cate.o array_cats.o array_cats0.o array_equal.o array_fail.o array_get.o array_length.o array_reset.o array_start.o array_trunc.o array_truncate.o IO_OBJS=io_appendfile.o io_canread.o io_canwrite.o io_check.o io_close.o io_closeonexec.o io_createfile.o io_dontwantread.o io_dontwantwrite.o io_eagain.o io_fd.o io_finishandshutdown.o io_getcookie.o io_mmapwritefile.o io_nonblock.o io_passfd.o io_pipe.o io_readfile.o io_readwritefile.o io_receivefd.o io_sendfile.o io_setcookie.o io_sigpipe.o io_socketpair.o io_timeout.o io_timeouted.o io_tryread.o io_tryreadtimeout.o io_trywrite.o io_trywritetimeout.o io_wait.o io_waitread.o io_waituntil.o io_waituntil2.o io_waitwrite.o io_wantread.o io_wantwrite.o iob_addbuf.o iob_addbuf_free.o iob_addbuf_internal.o iob_addfile.o iob_addfile_close.o iob_adds.o iob_adds_free.o iob_free.o iob_new.o iob_prefetch.o iob_reset.o iob_send.o iob_write.o TEXTCODE_OBJS=base64.o fmt_base64.o fmt_cescape.o fmt_foldwhitespace.o fmt_hexdump.o fmt_html.o fmt_quotedprintable.o fmt_to_array.o fmt_to_sa.o fmt_tofrom_array.o fmt_urlencoded.o fmt_uuencoded.o fmt_yenc.o scan_base64.o scan_cescape.o scan_hexdump.o scan_html.o scan_quotedprintable.o scan_to_array.o scan_to_sa.o scan_tofrom_array.o scan_urlencoded.o scan_uuencoded.o scan_yenc.o +CDB_OBJS=cdb.o cdb_hash.o cdb_make.o cdb_traverse.o byte.a: $(BYTE_OBJS) fmt.a: $(FMT_OBJS) @@ -543,6 +549,7 @@ case.a: $(CASE_OBJS) array.a: $(ARRAY_OBJS) mult.a: $(MULT_OBJS) io.a: $(IO_OBJS) +cdb.a: $(CDB_OBJS) ALL_OBJS=$(DNS_OBJS) $(BYTE_OBJS) $(FMT_OBJS) $(SCAN_OBJS) \ $(STR_OBJS) $(UINT_OBJS) $(OPEN_OBJS) $(STRALLOC_OBJS) $(UNIX_OBJS) \ @@ -578,7 +585,7 @@ dep libsocket havealloca.h INCLUDES=buffer.h byte.h fmt.h ip4.h ip6.h mmap.h scan.h socket.h str.h stralloc.h \ uint16.h uint32.h uint64.h open.h textcode.h tai.h taia.h dns.h iopause.h case.h \ openreadclose.h readclose.h ndelay.h array.h io.h safemult.h iob.h havealloca.h \ -errmsg.h +errmsg.h cdb.h install: libowfat.a install -d $(INCLUDEDIR) $(MAN3DIR) $(LIBDIR) diff --git a/cdb.h b/cdb.h new file mode 100644 index 0000000..9080bad --- /dev/null +++ b/cdb.h @@ -0,0 +1,37 @@ +/* Public domain. */ + +#ifndef CDB_H +#define CDB_H + +#include "uint32.h" + +#define CDB_HASHSTART 5381 +extern uint32 cdb_hashadd(uint32 h,unsigned char c); +extern uint32 cdb_hash(const unsigned char *buf,unsigned long int len); + +struct cdb { + char *map; /* 0 if no map is available */ + int fd; + uint32 size; /* initialized if map is nonzero */ + uint32 loop; /* number of hash slots searched under this key */ + uint32 khash; /* initialized if loop is nonzero */ + uint32 kpos; /* initialized if loop is nonzero */ + uint32 hpos; /* initialized if loop is nonzero */ + uint32 hslots; /* initialized if loop is nonzero */ + uint32 dpos; /* initialized if cdb_findnext() returns 1 */ + uint32 dlen; /* initialized if cdb_findnext() returns 1 */ +} ; + +extern void cdb_free(struct cdb *); +extern void cdb_init(struct cdb *,int fd); + +extern int cdb_read(struct cdb *,unsigned char *,unsigned long int,uint32); + +extern void cdb_findstart(struct cdb *); +extern int cdb_findnext(struct cdb *,const unsigned char *,unsigned long int); +extern int cdb_find(struct cdb *,const unsigned char *,unsigned long int); + +#define cdb_datapos(c) ((c)->dpos) +#define cdb_datalen(c) ((c)->dlen) + +#endif diff --git a/cdb/cdb.c b/cdb/cdb.c new file mode 100644 index 0000000..fb5740c --- /dev/null +++ b/cdb/cdb.c @@ -0,0 +1,128 @@ +#define _FILE_OFFSET_BITS 64 + +#include +#include +#include +#include +#include +#include "byte.h" +#include "cdb.h" + +void cdb_free(struct cdb *c) { + if (c->map) { + munmap(c->map,c->size); + c->map = 0; + } +} + +void cdb_findstart(struct cdb *c) { + c->loop = 0; +} + +void cdb_init(struct cdb *c,int fd) { + struct stat st; + char *x; + + cdb_free(c); + cdb_findstart(c); + c->fd = fd; + + if (fstat(fd,&st) == 0) + if (st.st_size <= 0xffffffff) { + x = mmap(0,st.st_size,PROT_READ,MAP_SHARED,fd,0); + if (x + 1) { + c->size = st.st_size; + c->map = x; + } + } +} + +int cdb_read(struct cdb *c,unsigned char *buf,unsigned long len,uint32 pos) { + if (c->map) { + if ((pos > c->size) || (c->size - pos < len)) goto FORMAT; + byte_copy(buf,len,c->map + pos); + } + else { + if (lseek(c->fd,pos,SEEK_SET) == -1) return -1; + while (len > 0) { + int r; + do + r = read(c->fd,buf,len); + while ((r == -1) && (errno == EINTR)); + if (r == -1) return -1; + if (r == 0) goto FORMAT; + buf += r; + len -= r; + } + } + return 0; + + FORMAT: + errno = EPROTO; + return -1; +} + +static int match(struct cdb *c,const unsigned char *key,unsigned long int len,uint32 pos) { + unsigned char buf[32]; + unsigned long n; + + while (len > 0) { + n = sizeof buf; + if (n > len) n = len; + if (cdb_read(c,buf,n,pos) == -1) return -1; + if (byte_diff(buf,n,key)) return 0; + pos += n; + key += n; + len -= n; + } + return 1; +} + +int cdb_findnext(struct cdb *c,const unsigned char *key,unsigned long int len) { + unsigned char buf[8]; + uint32 pos; + uint32 u; + + if (!c->loop) { + u = cdb_hash(key,len); + if (cdb_read(c,buf,8,(u << 3) & 2047) == -1) return -1; + uint32_unpack(buf + 4,&c->hslots); + if (!c->hslots) return 0; + uint32_unpack(buf,&c->hpos); + c->khash = u; + u >>= 8; + u %= c->hslots; + u <<= 3; + c->kpos = c->hpos + u; + } + + while (c->loop < c->hslots) { + if (cdb_read(c,buf,8,c->kpos) == -1) return -1; + uint32_unpack(buf + 4,&pos); + if (!pos) return 0; + c->loop += 1; + c->kpos += 8; + if (c->kpos == c->hpos + (c->hslots << 3)) c->kpos = c->hpos; + uint32_unpack(buf,&u); + if (u == c->khash) { + if (cdb_read(c,buf,8,pos) == -1) return -1; + uint32_unpack(buf,&u); + if (u == len) + switch(match(c,key,len,pos + 8)) { + case -1: + return -1; + case 1: + uint32_unpack(buf + 4,&c->dlen); + c->dpos = pos + 8 + len; + return 1; + } + } + } + + return 0; +} + +int cdb_find(struct cdb *c,const unsigned char *key,unsigned long int len) { + cdb_findstart(c); + return cdb_findnext(c,key,len); +} diff --git a/cdb/cdb_datalen.3 b/cdb/cdb_datalen.3 new file mode 100644 index 0000000..3b0f0f7 --- /dev/null +++ b/cdb/cdb_datalen.3 @@ -0,0 +1,17 @@ +.TH cdb_datalen 3 +.SH NAME +cdb_datalen \- get length of data +.SH SYNTAX +.B #include + +unsigned int cdb_datalen(struct cdb *\fIc\fR); + +.SH DESCRIPTION +.B cdb_datalen +returns the length of the data associated with the last key you looked +up with cdb_find or cdb_findnext. + +Use cdb_datapos to get the position of the data. + +.SH "SEE ALSO" +cdb_datapos(3), cdb_keylen(3), cdb_keypos(3) diff --git a/cdb/cdb_datapos.3 b/cdb/cdb_datapos.3 new file mode 100644 index 0000000..e679795 --- /dev/null +++ b/cdb/cdb_datapos.3 @@ -0,0 +1,17 @@ +.TH cdb_datapos 3 +.SH NAME +cdb_datapos \- get position of data +.SH SYNTAX +.B #include + +uint32 cdb_datapos(struct cdb *\fIc\fR); + +.SH DESCRIPTION +.B cdb_datapos +returns the position of the data associated with the last key you looked +up using cdb_find or cdb_findnext. + +Use cdb_datalen to get the length of the data. + +.SH "SEE ALSO" +cdb_datalen(3), cdb_keypos(3), cdb_keylen(3) diff --git a/cdb/cdb_find.3 b/cdb/cdb_find.3 new file mode 100644 index 0000000..5e9dfe6 --- /dev/null +++ b/cdb/cdb_find.3 @@ -0,0 +1,42 @@ +.TH cdb_find 3 +.SH NAME +cdb_find \- look up a key in a constant database +.SH SYNTAX +.B #include +.br +.B #include + +int cdb_find(struct cdb *\fIc\fR,char *\fIkey\fR,unsigned int \fIkeylen\fR); +.br +int cdb_datalen(struct cdb *\fIc\fR); +.br +int cdb_datapos(struct cdb *\fIc\fR); +.br +int cdb_findnext(struct cdb *\fIc\fR,char *\fIkey\fR,unsigned int \fIkeylen\fR); + +.SH DESCRIPTION +\fBcdb_find\fR looks for \fIkey\fR. If cdb_find returns 0, the database +does not contain that key; stop. If cdb_find returns -1, there was a +read error; abort. + +\fBcdb_datalen\fR returns the length of the data associated with the +\fIkey\fR. Use it to allocate a pointer \fIp\fR with enough space to +hold the data. + +\fBcdb_datapos\fR returns the position of the data inside the file. Use +it as argument to cdb_read to retrieve the data. + +There may be several records under a single key. You can use +\fBcdb_findnext\fR to find the next record under this key. + +.SH EXAMPLE +static struct cdb; + +if (cdb_find(&c,key,strlen(key)>0) { + char *buf=alloca(cdb_datalen(&c)); + cdb_read(&c,buf,cdb_datalen(&c),cdb_datapos(&c)); + write(1,buf,cdb_datalen(&c)); +.br +} +.SH "SEE ALSO" +cdb_read(3), cdb_init(3), cdb_free(3), cdbmake(1) diff --git a/cdb/cdb_firstkey.3 b/cdb/cdb_firstkey.3 new file mode 100644 index 0000000..6f34234 --- /dev/null +++ b/cdb/cdb_firstkey.3 @@ -0,0 +1,19 @@ +.TH cdb_firstkey 3 +.SH NAME +cdb_firstkey \- find first physical record in constant database +.SH SYNTAX +.B #include +.br +.B #include + +int cdb_firstkey(struct cdb *\fIc\fR,uint32 *\fIkpos\fR); + +.SH DESCRIPTION +\fBcdb_firstkey\fR finds the physically first record in the constant +database. + +You can use cdb_datapos or cdb_keypos to get \fIposition\fR and +cdb_datalen and cdb_keylen to get \fIlen\fR. + +.SH "SEE ALSO" +cdb_nextkey(3), cdb_successor(3) diff --git a/cdb/cdb_free.3 b/cdb/cdb_free.3 new file mode 100644 index 0000000..f901b77 --- /dev/null +++ b/cdb/cdb_free.3 @@ -0,0 +1,17 @@ +.TH cdb_free 3 +.SH NAME +cdb_free \- close a constant databased +.SH SYNTAX +.B #include + +int cdb_free(struct cdb *\fIc\fR); + +.SH DESCRIPTION +.B cdb_free +removes any memory mapping that cdb_init might have established. + +It will not close the file descriptor that you passed to cdb_init or +attempt to call free() on the struct cdb pointer. + +.SH "SEE ALSO" +cdb_free(3), cdb_read(3), cdb_find(3), cdbmake(1) diff --git a/cdb/cdb_hash.c b/cdb/cdb_hash.c new file mode 100644 index 0000000..4211486 --- /dev/null +++ b/cdb/cdb_hash.c @@ -0,0 +1,19 @@ + +#include "cdb.h" + +uint32 cdb_hashadd(uint32 h,unsigned char c) { + h += (h << 5); + return h ^ c; +} + +uint32 cdb_hash(const unsigned char *buf,unsigned long int len) { + uint32 h; + + h = CDB_HASHSTART; + while (len) { + h = cdb_hashadd(h,*buf); + ++buf; + --len; + } + return h; +} diff --git a/cdb/cdb_init.3 b/cdb/cdb_init.3 new file mode 100644 index 0000000..69a8850 --- /dev/null +++ b/cdb/cdb_init.3 @@ -0,0 +1,20 @@ +.TH cdb_init 3 +.SH NAME +cdb_init \- open a constant database +.SH SYNTAX +.B #include + +int cdb_init(struct cdb *\fIc\fR,int \fIfd\fR); + +.SH DESCRIPTION +.B cdb_init +places information about \fIfd\fR into a struct cdb variable \fIc\fR. +\fIfd\fR has to be a seekable file previously opened for reading. + +On systems that support mmap, cdb_init will try to map the whole +constant database into memory. + +The inverse operation to cdb_init is cdb_free. + +.SH "SEE ALSO" +cdb_free(3), cdb_read(3), cdb_find(3), cdbmake(1) diff --git a/cdb/cdb_keylen.3 b/cdb/cdb_keylen.3 new file mode 100644 index 0000000..203af9e --- /dev/null +++ b/cdb/cdb_keylen.3 @@ -0,0 +1,16 @@ +.TH cdb_keylen 3 +.SH NAME +cdb_keylen \- get length of key +.SH SYNTAX +.B #include + +unsigned int cdb_keylen(struct cdb *\fIc\fR); + +.SH DESCRIPTION +.B cdb_keylen +returns the length of the key cdb_firstkey or cdb_nextkey found. + +Use cdb_keypos to get the position of the key. + +.SH "SEE ALSO" +cdb_keypos(3), cdb_datalen(3), cdb_datapos(3) diff --git a/cdb/cdb_keypos.3 b/cdb/cdb_keypos.3 new file mode 100644 index 0000000..06f4c34 --- /dev/null +++ b/cdb/cdb_keypos.3 @@ -0,0 +1,16 @@ +.TH cdb_keypos 3 +.SH NAME +cdb_keypos \- get position of key +.SH SYNTAX +.B #include + +uint32 cdb_keypos(struct cdb *\fIc\fR); + +.SH DESCRIPTION +.B cdb_keypos +returns the position of the key cdb_firstkey or cdb_nextkey found. + +Use cdb_keylen to get the length of the key. + +.SH "SEE ALSO" +cdb_keylen(3), cdb_datapos(3), cdb_datalen(3) diff --git a/cdb/cdb_make.c b/cdb/cdb_make.c new file mode 100644 index 0000000..fe94af1 --- /dev/null +++ b/cdb/cdb_make.c @@ -0,0 +1,148 @@ +#define _FILE_OFFSET_BITS 64 + +#include +#include +#include +#include "cdb.h" +#include "cdb_make.h" + +int cdb_make_start(struct cdb_make *c,int fd) { + c->head = 0; + c->split = 0; + c->hash = 0; + c->numentries = 0; + c->fd = fd; + c->pos = sizeof c->final; + buffer_init(&c->b,(void*)write,fd,c->bspace,sizeof c->bspace); + return lseek(fd,c->pos,SEEK_SET); +} + +static int posplus(struct cdb_make *c,uint32 len) { + uint32 newpos = c->pos + len; + if (newpos < len) { errno = ENOMEM; return -1; } + c->pos = newpos; + return 0; +} + +int cdb_make_addend(struct cdb_make *c,unsigned long int keylen,unsigned long int datalen,uint32 h) { + struct cdb_hplist *head; + + head = c->head; + if (!head || (head->num >= CDB_HPLIST)) { + head = (struct cdb_hplist *) malloc(sizeof(struct cdb_hplist)); + if (!head) return -1; + head->num = 0; + head->next = c->head; + c->head = head; + } + head->hp[head->num].h = h; + head->hp[head->num].p = c->pos; + ++head->num; + ++c->numentries; + if (posplus(c,8) == -1) return -1; + if (posplus(c,keylen) == -1) return -1; + if (posplus(c,datalen) == -1) return -1; + return 0; +} + +int cdb_make_addbegin(struct cdb_make *c,unsigned long int keylen,unsigned long int datalen) { + char buf[8]; + + if (keylen > 0xffffffff) { errno = ENOMEM; return -1; } + if (datalen > 0xffffffff) { errno = ENOMEM; return -1; } + + uint32_pack(buf,keylen); + uint32_pack(buf + 4,datalen); + if (buffer_putalign(&c->b,buf,8) == -1) return -1; + return 0; +} + +int cdb_make_add(struct cdb_make *c,const unsigned char *key,unsigned long int keylen,const unsigned char *data,unsigned long int datalen) +{ + if (cdb_make_addbegin(c,keylen,datalen) == -1) return -1; + if (buffer_putalign(&c->b,(char*)key,keylen) == -1) return -1; + if (buffer_putalign(&c->b,(char*)data,datalen) == -1) return -1; + return cdb_make_addend(c,keylen,datalen,cdb_hash(key,keylen)); +} + +int cdb_make_finish(struct cdb_make *c) +{ + char buf[8]; + int i; + uint32 len; + uint32 u; + uint32 memsize; + uint32 count; + uint32 where; + struct cdb_hplist *x; + struct cdb_hp *hp; + + for (i = 0;i < 256;++i) + c->count[i] = 0; + + for (x = c->head;x;x = x->next) { + i = x->num; + while (i--) + ++c->count[255 & x->hp[i].h]; + } + + memsize = 1; + for (i = 0;i < 256;++i) { + u = c->count[i] * 2; + if (u > memsize) + memsize = u; + } + + memsize += c->numentries; /* no overflow possible up to now */ + u = (uint32) 0 - (uint32) 1; + u /= sizeof(struct cdb_hp); + if (memsize > u) { errno = ENOMEM; return -1; } + + c->split = (struct cdb_hp *) malloc(memsize * sizeof(struct cdb_hp)); + if (!c->split) return -1; + + c->hash = c->split + c->numentries; + + u = 0; + for (i = 0;i < 256;++i) { + u += c->count[i]; /* bounded by numentries, so no overflow */ + c->start[i] = u; + } + + for (x = c->head;x;x = x->next) { + i = x->num; + while (i--) + c->split[--c->start[255 & x->hp[i].h]] = x->hp[i]; + } + + for (i = 0;i < 256;++i) { + count = c->count[i]; + + len = count + count; /* no overflow possible */ + uint32_pack(c->final + 8 * i,c->pos); + uint32_pack(c->final + 8 * i + 4,len); + + for (u = 0;u < len;++u) + c->hash[u].h = c->hash[u].p = 0; + + hp = c->split + c->start[i]; + for (u = 0;u < count;++u) { + where = (hp->h >> 8) % len; + while (c->hash[where].p) + if (++where == len) + where = 0; + c->hash[where] = *hp++; + } + + for (u = 0;u < len;++u) { + uint32_pack(buf,c->hash[u].h); + uint32_pack(buf + 4,c->hash[u].p); + if (buffer_putalign(&c->b,buf,8) == -1) return -1; + if (posplus(c,8) == -1) return -1; + } + } + + if (buffer_flush(&c->b) == -1) return -1; + if (lseek(c->fd,0,SEEK_SET) == -1) return -1; + return buffer_putflush(&c->b,c->final,sizeof c->final); +} diff --git a/cdb/cdb_nextkey.3 b/cdb/cdb_nextkey.3 new file mode 100644 index 0000000..7dbf846 --- /dev/null +++ b/cdb/cdb_nextkey.3 @@ -0,0 +1,19 @@ +.TH cdb_nextkey 3 +.SH NAME +cdb_nextkey \- find next physical record in constant database +.SH SYNTAX +.B #include +.br +.B #include + +int cdb_nextkey(struct cdb *\fIc\fR,uint32 *\fIkpos\fR); + +.SH DESCRIPTION +\fBcdb_nextkey\fR finds the next physical record in the constant +database. + +You can use cdb_datapos or cdb_keypos to get \fIposition\fR and +cdb_datalen and cdb_keylen to get \fIlen\fR. + +.SH "SEE ALSO" +cdb_firstkey(3), cdb_successor(3) diff --git a/cdb/cdb_read.3 b/cdb/cdb_read.3 new file mode 100644 index 0000000..2dd8451 --- /dev/null +++ b/cdb/cdb_read.3 @@ -0,0 +1,20 @@ +.TH cdb_read 3 +.SH NAME +cdb_read \- read bytes from a constant database +.SH SYNTAX +.B #include +.br +.B #include + +int cdb_read(struct cdb *\fIc\fR,char *\fIbuf\fR,unsigned int \fIlen\fR,uint32 \fIposition\fR); + +.SH DESCRIPTION +\fBcdb_read\fR reads \fIlen\fR bytes starting at \fIposition\fR from +\fIc\fR to buf. You can use cdb_datapos or cdb_keypos to get +\fIposition\fR and cdb_datalen and cdb_keylen to get \fIlen\fR. + +\fIbuf\fR needs to point to a memory region large enough to hold +\fIlen\fR bytes. + +.SH "SEE ALSO" +cdb_find(3), cdb_init(3), cdb_free(3), cdbmake(1) diff --git a/cdb/cdb_successor.3 b/cdb/cdb_successor.3 new file mode 100644 index 0000000..28c9626 --- /dev/null +++ b/cdb/cdb_successor.3 @@ -0,0 +1,21 @@ +.TH cdb_successor 3 +.SH NAME +cdb_successor \- find next record +.SH SYNTAX +.B #include + +int cdb_successor(struct cdb *\fIc\fR,char *\fIkey\fR,unsigned int \fIklen\fR); + +.SH DESCRIPTION +\fBcdb_successor\fR finds the record that follows \fIkey\fR. If +\fIkey\fR is NULL, cdb_successor finds the first record. + +\fBNOTE!\fR The database must not contain keys with more than one +associated record or this API will lead to infinite loops! Use +cdb_firstkey and cdb_nextkey instead. + +You can use cdb_datapos or cdb_keypos to get \fIposition\fR and +cdb_datalen and cdb_keylen to get \fIlen\fR. + +.SH "SEE ALSO" +cdb_firstkey(3), cdb_nextkey(3) diff --git a/cdb/cdb_traverse.c b/cdb/cdb_traverse.c new file mode 100644 index 0000000..81bfd50 --- /dev/null +++ b/cdb/cdb_traverse.c @@ -0,0 +1,40 @@ +#include "cdb.h" + +static int doit(struct cdb *c,uint32 *kpos) { + unsigned char buf[8]; + uint32 eod,klen; + if (cdb_read(c,buf,4,0)) return -1; + uint32_unpack(buf,&eod); + if (eod<8 || eod-8<*kpos) return 0; + c->kpos=*kpos+8; + if (c->kpos<*kpos) return -1; /* wraparound */ + cdb_findstart(c); + c->hslots=1; + if (cdb_read(c,buf,8,*kpos) == -1) return -1; + uint32_unpack(buf,&klen); + uint32_unpack(buf+4,&c->dlen); + c->dpos=c->kpos+klen; + *kpos+=8+klen+c->dlen; + return 1; +} + +int cdb_firstkey(struct cdb *c,uint32 *kpos) { + *kpos=2048; + return doit(c,kpos); +} + +int cdb_nextkey(struct cdb *c,uint32 *kpos) { + return doit(c,kpos); +} + +int cdb_successor(struct cdb *c,unsigned char *key,unsigned long int klen) { + int r; + uint32 kpos; + if (key) { + r=cdb_find(c,key,klen); + if (r<1) return r; + kpos=c->dpos+c->dlen; + } else + kpos=2048; + return doit(c,&kpos); +} diff --git a/cdb_make.h b/cdb_make.h new file mode 100644 index 0000000..961b10b --- /dev/null +++ b/cdb_make.h @@ -0,0 +1,39 @@ +/* Public domain. */ + +#ifndef CDB_MAKE_H +#define CDB_MAKE_H + +#include "buffer.h" +#include "uint32.h" + +#define CDB_HPLIST 1000 + +struct cdb_hp { uint32 h; uint32 p; } ; + +struct cdb_hplist { + struct cdb_hp hp[CDB_HPLIST]; + struct cdb_hplist *next; + int num; +} ; + +struct cdb_make { + char bspace[8192]; + char final[2048]; + uint32 count[256]; + uint32 start[256]; + struct cdb_hplist *head; + struct cdb_hp *split; /* includes space for hash */ + struct cdb_hp *hash; + uint32 numentries; + buffer b; + uint32 pos; + int fd; +} ; + +extern int cdb_make_start(struct cdb_make *,int); +extern int cdb_make_addbegin(struct cdb_make *,unsigned long int,unsigned long int); +extern int cdb_make_addend(struct cdb_make *,unsigned long int,unsigned long int,uint32); +extern int cdb_make_add(struct cdb_make *,const unsigned char *,unsigned long int,const unsigned char *,unsigned long int); +extern int cdb_make_finish(struct cdb_make *); + +#endif