add iarray

CHANGES

@@ -1,6 +1,7 @@
 0.29:
   save 8 bytes in taia.h for 64-bit systems
   add buffer_tosa (buffer writing to auto-growing stralloc)
+  add iarray
 
 0.28:
   add uint64 pack and unpack routines

GNUmakefile

@@ -147,7 +147,7 @@ CFLAGS+=-I.
 t.o: iopause.h
 
 t: t.o libowfat.a libsocket
-	$(DIET) $(CC) -g -o $@ t.o libowfat.a `cat libsocket`
+	$(DIET) $(CC) -g -o $@ t.o libowfat.a `cat libsocket` -lpthread
 
 .PHONY: all clean tar install rename
 clean:
@@ -159,7 +159,7 @@ dep libsocket havealloca.h
 INCLUDES=buffer.h byte.h fmt.h ip4.h ip6.h mmap.h scan.h socket.h str.h stralloc.h \
 uint16.h uint32.h uint64.h open.h textcode.h tai.h taia.h dns.h iopause.h case.h \
 openreadclose.h readclose.h ndelay.h array.h io.h safemult.h iob.h havealloca.h \
-errmsg.h cdb.h cdb_make.h rangecheck.h
+errmsg.h cdb.h cdb_make.h rangecheck.h iarray.h
 
 install: libowfat.a
 	install -d $(INCLUDEDIR) $(MAN3DIR) $(LIBDIR)

array/array_allocate.c

@@ -1,9 +1,4 @@
-#ifdef __dietlibc__
+#include "likely.h"
-#include <sys/cdefs.h>
-#else
-#define __likely(x) x
-#define __unlikely(x) x
-#endif
 #include <sys/types.h>
 #include <stdlib.h>
 #include "safemult.h"

array/array_get.c

@@ -1,9 +1,4 @@
-#ifdef __dietlibc__
+#include "likely.h"
-#include <sys/cdefs.h>
-#else
-#define __likely(x) x
-#define __unlikely(x) x
-#endif
 #include "safemult.h"
 #include "array.h"
 

array/array_truncate.c

@@ -1,9 +1,4 @@
-#ifdef __dietlibc__
+#include "likely.h"
-#include <sys/cdefs.h>
-#else
-#define __likely(x) x
-#define __unlikely(x) x
-#endif
 #include "safemult.h"
 #include "array.h"
 

array/iarray_allocate.3

@@ -0,0 +1,31 @@
+.TH iarray_allocate 3
+.SH NAME
+iarray_allocate \- get pointer to nth element in iarray
+.SH SYNTAX
+.B #include <iarray.h>
+
+void* \fBiarray_allocate\fP(iarray* \fIx\fR, size_t \fIpos\fR);
+
+  iarray \fIx\fR;
+  size_t \fIpos\fR;
+  \fIt\fR* p = iarray_allocate(&\fIx\fR,\fIpos\fR);
+
+.SH DESCRIPTION
+iarray_allocate is similar to iarray_get, but if the requested element
+is not in the array, the array will be resized.  If the resize fails,
+iarray_allocate returns NULL and leaves the array untouched.
+
+This function is safe to use in environments with multiple threads, but
+it can block for indeterminate time if other threads are reallocating
+the array at the same time.
+
+Note that it is safe to use iarray_allocate where you would otherwise
+use iarray_get.  The only reason to use iarray_get over iarray_allocate
+would be optimization.
+
+.SH "RETURN VALUE"
+Return a pointer to the requested element.  If there was a memory
+allocation failure, returns NULL.
+
+.SH "SEE ALSO"
+iarray_init(3), iarray_get(3), iarray_free(3)

array/iarray_allocate.c

@@ -0,0 +1,47 @@
+#include "likely.h"
+#include <stdlib.h>
+#include "iarray.h"
+
+void* iarray_allocate(iarray* ia,size_t pos) {
+  size_t y;
+  /* first the easy case without locking */
+  if (__likely((y=pos/ia->elemperpage) < ia->pagefence && ia->pages[y]))
+    return ia->pages[y]+(pos%ia->elemperpage)*ia->elemsize;
+  /* the case where ia->pages == NULL is implicit */
+
+  pthread_mutex_lock(&ia->m);
+
+  if (__unlikely(y >= ia->pagefence)) {
+    char** np;
+    /* The data structure is an array of pointer to pages.
+     * Each page holds at least one element of the array.
+     * Here we realloc the array of pointers.  Each element in this
+     * array is only 4 or 8 bytes, so we should allocate a few more than
+     * we need to cut down on future reallocs. */
+    size_t z=(y+512)&-512;		/* round up to multiple of 512 */
+    /* It may seem as if there can be no integer overflow in the
+     * indirect index, because then the array would not fit into the
+     * address space in the first place, but remember that this is a
+     * sparse array.  Someone might just pass in an unreasonable large
+     * index and have large elements, too */
+    if (z==0) goto unlockandfail;	/* integer overflow */
+    np=realloc(ia->pages,z*ia->bytesperpage);
+    if (!np) goto unlockandfail;
+    ia->pagefence=z;
+    ia->pages=np;
+  }
+
+  /* at this point we know the slot exists */
+  /* through a race between the early-out above and the
+   * pthread_mutex_lock, the page pointer to it could be non-NULL,
+   * however */
+  if (__unlikely(ia->pages[y]==0 && (ia->pages[y]=malloc(ia->bytesperpage))==0)) {
+unlockandfail:
+    pthread_mutex_unlock(&ia->m);
+    return 0;
+  }
+
+  pthread_mutex_unlock(&ia->m);
+
+  return ia->pages[y] + (pos%ia->elemperpage)*ia->elemsize;
+}

array/iarray_free.3

@@ -0,0 +1,16 @@
+.TH iarray_free 3
+.SH NAME
+iarray_free \- free iarray data structure
+.SH SYNTAX
+.B #include <iarray.h>
+
+void \fBiarray_free\fP(iarray* \fIx\fR);
+
+.SH DESCRIPTION
+iarray_free frees the iarray and all elements in it.
+
+Using the array during or after iarray_free results in undefined
+behavior.
+
+.SH "SEE ALSO"
+iarray_allocate(3), iarray_get(3), iarray_allocate(3)

array/iarray_free.c

@@ -0,0 +1,9 @@
+#include <stdlib.h>
+#include "iarray.h"
+
+void iarray_free(iarray* ia) {
+  size_t i;
+  for (i=0; i<ia->pagefence; ++i)
+    if (ia->pages[i]) free(ia->pages[i]);
+  free(ia->pages);
+}

array/iarray_get.3

@@ -0,0 +1,25 @@
+.TH iarray_get 3
+.SH NAME
+iarray_get \- get pointer to nth element in iarray
+.SH SYNTAX
+.B #include <iarray.h>
+
+void* \fBiarray_get\fP(iarray* \fIx\fR, size_t \fIpos\fR);
+
+  iarray \fIx\fR;
+  size_t \fIpos\fR;
+  \fIt\fR* p = iarray_get(&\fIx\fR,\fIpos\fR);
+
+.SH DESCRIPTION
+iarray_get is similar to iarray_allocate, but it only works if the
+element has previously been allocated.  If the element in the iarray
+is not there, this function will fail instead of manipulating the
+iarray.  This also guarantees that there will be no locks, so this
+function returns in a deterministic time.
+
+.SH "RETURN VALUE"
+Return a pointer to the requested element.  If there is no such element
+in the array, returns NULL.
+
+.SH "SEE ALSO"
+iarray_init(3), iarray_allocate(3), iarray_free(3)

array/iarray_get.c

@@ -0,0 +1,12 @@
+#include "iarray.h"
+
+void* iarray_get(iarray* ia,size_t pos) {
+  char* x;
+  size_t y;
+  if (!ia->pages) return 0;
+  y=pos/ia->elemperpage;
+  if (y>=ia->pagefence) return 0;
+  x=ia->pages[y];
+  if (!x) return 0;
+  return x+(pos%ia->elemperpage)*ia->elemsize;
+}

array/iarray_init.3

@@ -0,0 +1,19 @@
+.TH iarray_init 3
+.SH NAME
+iarray_init \- initialize iarray data structure
+.SH SYNTAX
+.B #include <iarray.h>
+
+void \fBiarray_init\fP(array* \fIx\fR, size_t \fIelemsize\fR);
+
+  iarray \fIx\fR;
+  int64 \fIpos\fR;
+  \fIt\fR* p = iarray_init(&\fIx\fR,sizeof(\fIelement\fR));
+
+.SH DESCRIPTION
+iarray_init initializes an iarray so that it can hold elements of size
+\fIelemsize\fR.  iarray_init does not actually allocate anything, so it
+can not fail.
+
+.SH "SEE ALSO"
+iarray_allocate(3), iarray_get(3), iarray_free(3)

array/iarray_init.c

@@ -0,0 +1,16 @@
+#include "iarray.h"
+
+void iarray_init(iarray* ia,size_t elemsize) {
+  ia->elemsize=elemsize;
+  ia->pages=0;
+  ia->pagefence=0;
+  if (elemsize<1024)
+    ia->bytesperpage=4096;
+  else if (elemsize<8192)
+    ia->bytesperpage=65536;
+  else
+    ia->bytesperpage=elemsize;
+  ia->elemperpage=ia->bytesperpage/elemsize;
+  pthread_mutex_init(&ia->m,NULL);
+}
+

iarray.h

@@ -0,0 +1,29 @@
+#ifndef IARRAY_H
+#define IARRAY_H
+
+#include "uint64.h"
+#include <stddef.h>
+#include <pthread.h>
+
+/* this is an indirect array; it only reallocs the indirect index, not
+ * the whole array.  The actual data does not move.  So there is no need
+ * to lock the array for read accesses. */
+
+typedef struct {
+  char** pages;
+  size_t elemsize,pagefence,elemperpage,bytesperpage;
+  /* pagefence is the number of pages + 1,
+   * i.e. the first out of bounds index in "pages" */
+  pthread_mutex_t m;
+} iarray;
+
+void iarray_init(iarray* ia,size_t elemsize);
+void* iarray_get(iarray* ia,size_t pos);
+void* iarray_allocate(iarray* ia,size_t pos);
+
+/* WARNING: do not use the array during or after iarray_free, make sure
+ * no threads are potentially doing anything with the iarray while it is
+ * being freed! */
+void iarray_free(iarray* ia);
+
+#endif

likely.h

@@ -0,0 +1,13 @@
+#ifdef __dietlibc__
+#include <sys/cdefs.h>
+#else
+
+#if __GNUC__ < 3
+#define __expect(foo,bar) (foo)
+#else
+#define __expect(foo,bar) __builtin_expect((long)(foo),bar)
+#endif
+#define __likely(foo) __expect((foo),1)
+#define __unlikely(foo) __expect((foo),0)
+
+#endif

t.c

@@ -22,6 +22,9 @@
 #include "errmsg.h"
 #include "iob.h"
 #include "safemult.h"
+#include "iarray.h"
+
+#include "io_internal.h"
 
 #define rdtscl(low) \
      __asm__ __volatile__ ("rdtsc" : "=a" (low) : : "edx")
@@ -41,25 +44,13 @@ int64 writecb(int64 fd,const void* buf,uint64 n) {
 }
 
 int main(int argc,char* argv[]) {
-  stralloc a;
+  iarray a;
-  buffer b;
+  char* c;
-  int i;
+  iarray_init(&a,sizeof(io_entry));
-  stralloc_init(&a);
+  printf("15 -> %p\n",c=iarray_allocate(&a,15));
-  buffer_tosa(&b,&a);
+  printf("23 -> %p\n",c=iarray_allocate(&a,23));
-
+  printf("1234567 -> %p\n",c=iarray_allocate(&a,1234567));
-  for (i=0; i<100; ++i)
+  printf("23 -> %p\n",iarray_get(&a,23));
-    buffer_puts(&b,"foo bar baz!\n");
-  buffer_flush(&b);
-  buffer_putsa(buffer_1,&a);
-  buffer_flush(buffer_1);
-#if 0
-  char* c=fmt_strm_alloca("foo"," bar","\n");
-
-  write(1,c,strlen(c));
-
-  (void)argc;
-  (void)argv;
-#endif
 #if 0
   io_batch* b=iob_new(1234);
   int64 fd=open("t.c",0);

test/buffer_tosa.c

@@ -0,0 +1,20 @@
+#include "byte.h"
+#include "stralloc.h"
+#include "buffer.h"
+#include <assert.h>
+
+int main() {
+  stralloc a;
+  buffer b;
+  int i;
+  stralloc_init(&a);
+  buffer_tosa(&b,&a);
+
+  for (i=0; i<100; ++i)
+    buffer_puts(&b,"foo bar baz!\n");
+  buffer_flush(&b);
+  assert(a.len==100*sizeof("foo bar baz!"));
+  for (i=0; i<100; ++i)
+    assert(byte_equal(a.s+i*sizeof("foo bar baz!"),sizeof("foo bar baz!"),"foo bar baz!\n"));
+  return 0;
+}

test/fmt_strm_alloca.c

@@ -0,0 +1,9 @@
+#include <stdlib.h>
+#include "fmt.h"
+#include "byte.h"
+#include <assert.h>
+
+int main() {
+  char* c=fmt_strm_alloca("foo"," bar","\n");
+  assert(byte_equal(c,sizeof("foo bar\n"),"foo bar\n"));
+}