diff --git a/CHANGES b/CHANGES index 70008e2..14f37ee 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,7 @@ 0.29: save 8 bytes in taia.h for 64-bit systems add buffer_tosa (buffer writing to auto-growing stralloc) + add iarray 0.28: add uint64 pack and unpack routines diff --git a/GNUmakefile b/GNUmakefile index d9069c4..c909403 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -147,7 +147,7 @@ CFLAGS+=-I. t.o: iopause.h t: t.o libowfat.a libsocket - $(DIET) $(CC) -g -o $@ t.o libowfat.a `cat libsocket` + $(DIET) $(CC) -g -o $@ t.o libowfat.a `cat libsocket` -lpthread .PHONY: all clean tar install rename clean: @@ -159,7 +159,7 @@ dep libsocket havealloca.h INCLUDES=buffer.h byte.h fmt.h ip4.h ip6.h mmap.h scan.h socket.h str.h stralloc.h \ uint16.h uint32.h uint64.h open.h textcode.h tai.h taia.h dns.h iopause.h case.h \ openreadclose.h readclose.h ndelay.h array.h io.h safemult.h iob.h havealloca.h \ -errmsg.h cdb.h cdb_make.h rangecheck.h +errmsg.h cdb.h cdb_make.h rangecheck.h iarray.h install: libowfat.a install -d $(INCLUDEDIR) $(MAN3DIR) $(LIBDIR) diff --git a/array/array_allocate.c b/array/array_allocate.c index 681b6d4..a9c1c4a 100644 --- a/array/array_allocate.c +++ b/array/array_allocate.c @@ -1,9 +1,4 @@ -#ifdef __dietlibc__ -#include -#else -#define __likely(x) x -#define __unlikely(x) x -#endif +#include "likely.h" #include #include #include "safemult.h" diff --git a/array/array_get.c b/array/array_get.c index 2bd97b0..dce1c63 100644 --- a/array/array_get.c +++ b/array/array_get.c @@ -1,9 +1,4 @@ -#ifdef __dietlibc__ -#include -#else -#define __likely(x) x -#define __unlikely(x) x -#endif +#include "likely.h" #include "safemult.h" #include "array.h" diff --git a/array/array_truncate.c b/array/array_truncate.c index 512a5cf..8a3e25d 100644 --- a/array/array_truncate.c +++ b/array/array_truncate.c @@ -1,9 +1,4 @@ -#ifdef __dietlibc__ -#include -#else -#define __likely(x) x -#define __unlikely(x) x -#endif +#include "likely.h" #include "safemult.h" #include "array.h" diff --git a/array/iarray_allocate.3 b/array/iarray_allocate.3 new file mode 100644 index 0000000..14939c8 --- /dev/null +++ b/array/iarray_allocate.3 @@ -0,0 +1,31 @@ +.TH iarray_allocate 3 +.SH NAME +iarray_allocate \- get pointer to nth element in iarray +.SH SYNTAX +.B #include + +void* \fBiarray_allocate\fP(iarray* \fIx\fR, size_t \fIpos\fR); + + iarray \fIx\fR; + size_t \fIpos\fR; + \fIt\fR* p = iarray_allocate(&\fIx\fR,\fIpos\fR); + +.SH DESCRIPTION +iarray_allocate is similar to iarray_get, but if the requested element +is not in the array, the array will be resized. If the resize fails, +iarray_allocate returns NULL and leaves the array untouched. + +This function is safe to use in environments with multiple threads, but +it can block for indeterminate time if other threads are reallocating +the array at the same time. + +Note that it is safe to use iarray_allocate where you would otherwise +use iarray_get. The only reason to use iarray_get over iarray_allocate +would be optimization. + +.SH "RETURN VALUE" +Return a pointer to the requested element. If there was a memory +allocation failure, returns NULL. + +.SH "SEE ALSO" +iarray_init(3), iarray_get(3), iarray_free(3) diff --git a/array/iarray_allocate.c b/array/iarray_allocate.c new file mode 100644 index 0000000..095b5b2 --- /dev/null +++ b/array/iarray_allocate.c @@ -0,0 +1,47 @@ +#include "likely.h" +#include +#include "iarray.h" + +void* iarray_allocate(iarray* ia,size_t pos) { + size_t y; + /* first the easy case without locking */ + if (__likely((y=pos/ia->elemperpage) < ia->pagefence && ia->pages[y])) + return ia->pages[y]+(pos%ia->elemperpage)*ia->elemsize; + /* the case where ia->pages == NULL is implicit */ + + pthread_mutex_lock(&ia->m); + + if (__unlikely(y >= ia->pagefence)) { + char** np; + /* The data structure is an array of pointer to pages. + * Each page holds at least one element of the array. + * Here we realloc the array of pointers. Each element in this + * array is only 4 or 8 bytes, so we should allocate a few more than + * we need to cut down on future reallocs. */ + size_t z=(y+512)&-512; /* round up to multiple of 512 */ + /* It may seem as if there can be no integer overflow in the + * indirect index, because then the array would not fit into the + * address space in the first place, but remember that this is a + * sparse array. Someone might just pass in an unreasonable large + * index and have large elements, too */ + if (z==0) goto unlockandfail; /* integer overflow */ + np=realloc(ia->pages,z*ia->bytesperpage); + if (!np) goto unlockandfail; + ia->pagefence=z; + ia->pages=np; + } + + /* at this point we know the slot exists */ + /* through a race between the early-out above and the + * pthread_mutex_lock, the page pointer to it could be non-NULL, + * however */ + if (__unlikely(ia->pages[y]==0 && (ia->pages[y]=malloc(ia->bytesperpage))==0)) { +unlockandfail: + pthread_mutex_unlock(&ia->m); + return 0; + } + + pthread_mutex_unlock(&ia->m); + + return ia->pages[y] + (pos%ia->elemperpage)*ia->elemsize; +} diff --git a/array/iarray_free.3 b/array/iarray_free.3 new file mode 100644 index 0000000..32465c8 --- /dev/null +++ b/array/iarray_free.3 @@ -0,0 +1,16 @@ +.TH iarray_free 3 +.SH NAME +iarray_free \- free iarray data structure +.SH SYNTAX +.B #include + +void \fBiarray_free\fP(iarray* \fIx\fR); + +.SH DESCRIPTION +iarray_free frees the iarray and all elements in it. + +Using the array during or after iarray_free results in undefined +behavior. + +.SH "SEE ALSO" +iarray_allocate(3), iarray_get(3), iarray_allocate(3) diff --git a/array/iarray_free.c b/array/iarray_free.c new file mode 100644 index 0000000..7ea1393 --- /dev/null +++ b/array/iarray_free.c @@ -0,0 +1,9 @@ +#include +#include "iarray.h" + +void iarray_free(iarray* ia) { + size_t i; + for (i=0; ipagefence; ++i) + if (ia->pages[i]) free(ia->pages[i]); + free(ia->pages); +} diff --git a/array/iarray_get.3 b/array/iarray_get.3 new file mode 100644 index 0000000..7fa7ac0 --- /dev/null +++ b/array/iarray_get.3 @@ -0,0 +1,25 @@ +.TH iarray_get 3 +.SH NAME +iarray_get \- get pointer to nth element in iarray +.SH SYNTAX +.B #include + +void* \fBiarray_get\fP(iarray* \fIx\fR, size_t \fIpos\fR); + + iarray \fIx\fR; + size_t \fIpos\fR; + \fIt\fR* p = iarray_get(&\fIx\fR,\fIpos\fR); + +.SH DESCRIPTION +iarray_get is similar to iarray_allocate, but it only works if the +element has previously been allocated. If the element in the iarray +is not there, this function will fail instead of manipulating the +iarray. This also guarantees that there will be no locks, so this +function returns in a deterministic time. + +.SH "RETURN VALUE" +Return a pointer to the requested element. If there is no such element +in the array, returns NULL. + +.SH "SEE ALSO" +iarray_init(3), iarray_allocate(3), iarray_free(3) diff --git a/array/iarray_get.c b/array/iarray_get.c new file mode 100644 index 0000000..d578856 --- /dev/null +++ b/array/iarray_get.c @@ -0,0 +1,12 @@ +#include "iarray.h" + +void* iarray_get(iarray* ia,size_t pos) { + char* x; + size_t y; + if (!ia->pages) return 0; + y=pos/ia->elemperpage; + if (y>=ia->pagefence) return 0; + x=ia->pages[y]; + if (!x) return 0; + return x+(pos%ia->elemperpage)*ia->elemsize; +} diff --git a/array/iarray_init.3 b/array/iarray_init.3 new file mode 100644 index 0000000..768a999 --- /dev/null +++ b/array/iarray_init.3 @@ -0,0 +1,19 @@ +.TH iarray_init 3 +.SH NAME +iarray_init \- initialize iarray data structure +.SH SYNTAX +.B #include + +void \fBiarray_init\fP(array* \fIx\fR, size_t \fIelemsize\fR); + + iarray \fIx\fR; + int64 \fIpos\fR; + \fIt\fR* p = iarray_init(&\fIx\fR,sizeof(\fIelement\fR)); + +.SH DESCRIPTION +iarray_init initializes an iarray so that it can hold elements of size +\fIelemsize\fR. iarray_init does not actually allocate anything, so it +can not fail. + +.SH "SEE ALSO" +iarray_allocate(3), iarray_get(3), iarray_free(3) diff --git a/array/iarray_init.c b/array/iarray_init.c new file mode 100644 index 0000000..f273cf5 --- /dev/null +++ b/array/iarray_init.c @@ -0,0 +1,16 @@ +#include "iarray.h" + +void iarray_init(iarray* ia,size_t elemsize) { + ia->elemsize=elemsize; + ia->pages=0; + ia->pagefence=0; + if (elemsize<1024) + ia->bytesperpage=4096; + else if (elemsize<8192) + ia->bytesperpage=65536; + else + ia->bytesperpage=elemsize; + ia->elemperpage=ia->bytesperpage/elemsize; + pthread_mutex_init(&ia->m,NULL); +} + diff --git a/iarray.h b/iarray.h new file mode 100644 index 0000000..6840080 --- /dev/null +++ b/iarray.h @@ -0,0 +1,29 @@ +#ifndef IARRAY_H +#define IARRAY_H + +#include "uint64.h" +#include +#include + +/* this is an indirect array; it only reallocs the indirect index, not + * the whole array. The actual data does not move. So there is no need + * to lock the array for read accesses. */ + +typedef struct { + char** pages; + size_t elemsize,pagefence,elemperpage,bytesperpage; + /* pagefence is the number of pages + 1, + * i.e. the first out of bounds index in "pages" */ + pthread_mutex_t m; +} iarray; + +void iarray_init(iarray* ia,size_t elemsize); +void* iarray_get(iarray* ia,size_t pos); +void* iarray_allocate(iarray* ia,size_t pos); + +/* WARNING: do not use the array during or after iarray_free, make sure + * no threads are potentially doing anything with the iarray while it is + * being freed! */ +void iarray_free(iarray* ia); + +#endif diff --git a/likely.h b/likely.h new file mode 100644 index 0000000..5d01ca5 --- /dev/null +++ b/likely.h @@ -0,0 +1,13 @@ +#ifdef __dietlibc__ +#include +#else + +#if __GNUC__ < 3 +#define __expect(foo,bar) (foo) +#else +#define __expect(foo,bar) __builtin_expect((long)(foo),bar) +#endif +#define __likely(foo) __expect((foo),1) +#define __unlikely(foo) __expect((foo),0) + +#endif diff --git a/t.c b/t.c index 4db485a..c958b04 100644 --- a/t.c +++ b/t.c @@ -22,6 +22,9 @@ #include "errmsg.h" #include "iob.h" #include "safemult.h" +#include "iarray.h" + +#include "io_internal.h" #define rdtscl(low) \ __asm__ __volatile__ ("rdtsc" : "=a" (low) : : "edx") @@ -41,25 +44,13 @@ int64 writecb(int64 fd,const void* buf,uint64 n) { } int main(int argc,char* argv[]) { - stralloc a; - buffer b; - int i; - stralloc_init(&a); - buffer_tosa(&b,&a); - - for (i=0; i<100; ++i) - buffer_puts(&b,"foo bar baz!\n"); - buffer_flush(&b); - buffer_putsa(buffer_1,&a); - buffer_flush(buffer_1); -#if 0 - char* c=fmt_strm_alloca("foo"," bar","\n"); - - write(1,c,strlen(c)); - - (void)argc; - (void)argv; -#endif + iarray a; + char* c; + iarray_init(&a,sizeof(io_entry)); + printf("15 -> %p\n",c=iarray_allocate(&a,15)); + printf("23 -> %p\n",c=iarray_allocate(&a,23)); + printf("1234567 -> %p\n",c=iarray_allocate(&a,1234567)); + printf("23 -> %p\n",iarray_get(&a,23)); #if 0 io_batch* b=iob_new(1234); int64 fd=open("t.c",0); diff --git a/test/buffer_tosa.c b/test/buffer_tosa.c new file mode 100644 index 0000000..2624053 --- /dev/null +++ b/test/buffer_tosa.c @@ -0,0 +1,20 @@ +#include "byte.h" +#include "stralloc.h" +#include "buffer.h" +#include + +int main() { + stralloc a; + buffer b; + int i; + stralloc_init(&a); + buffer_tosa(&b,&a); + + for (i=0; i<100; ++i) + buffer_puts(&b,"foo bar baz!\n"); + buffer_flush(&b); + assert(a.len==100*sizeof("foo bar baz!")); + for (i=0; i<100; ++i) + assert(byte_equal(a.s+i*sizeof("foo bar baz!"),sizeof("foo bar baz!"),"foo bar baz!\n")); + return 0; +} diff --git a/test/fmt_strm_alloca.c b/test/fmt_strm_alloca.c new file mode 100644 index 0000000..328493c --- /dev/null +++ b/test/fmt_strm_alloca.c @@ -0,0 +1,9 @@ +#include +#include "fmt.h" +#include "byte.h" +#include + +int main() { + char* c=fmt_strm_alloca("foo"," bar","\n"); + assert(byte_equal(c,sizeof("foo bar\n"),"foo bar\n")); +}