add a few helpers for parsing binary data

buffer.h

@@ -41,16 +41,51 @@ typedef struct buffer {
 #define BUFFER_INSIZE 8192
 #define BUFFER_OUTSIZE 8192
 
+/* Initialize a buffer with an existing memory area, which the buffer
+ * will NOT take ownership of (i.e. won't free the memory when it's done */
 __writememsz__(4,5)
 void buffer_init(buffer* b,ssize_t (*op)(),int fd,char* y,size_t ylen);
+
+/* Initialize a buffer with an existing memory area, which the buffer
+ * WILL take ownership of (it will call free() on it when it's done) */
 __writememsz__(4,5)
 void buffer_init_free(buffer* b,ssize_t (*op)(),int fd,char* y,size_t ylen);
+
+/* Initialize a buffer without actual I/O.
+ * You give it a pre-existing memory area.
+ * When reading from this buffer, it will simply return the data from
+ * that memory area. If it reaches the end, it will signal EOF and never
+ * actually attempt to read from any actual file.
+ * Does not take ownership. Useful for testing. */
+void buffer_init_staticcontents(buffer* b,char* y,size_t ylen);
+
+/* Same but the buffer takes ownership of the static buffer and frees it
+ * in buffer_close. */
+void buffer_init_staticcontents_free(buffer* b,char* y,size_t ylen);
+
+
+/* Set buffer->deinit to this if you want buffer_close() to call free() on
+ * the associated memory buffer */
 void buffer_free(void* buf);
+
+/* Set buffer->deinit to this if you want buffer_close() to call munmap() on
+ * the associated memory buffer */
 void buffer_munmap(void* buf);
+
+/* Initialize a buffer so it will read from this file by memory mapping
+ * the whole thing. */
 int buffer_mmapread(buffer* b,const char* filename);
+
+/* Indicate you are done with a buffer.
+ * If the buffer has an associated memory map or buffer it owns, it will
+ * free that memory. This will NOT call free() on b itself! */
 void buffer_close(buffer* b);
 
+/* Flush the buffer. This is only meaningful for write buffers and will
+ * cause a write() syscall (or whatever you set as buffer->op) for all
+ * the data in the buffer. */
 int buffer_flush(buffer* b);
+
 __readmemsz__(2,3)
 int buffer_put(buffer* b,const char* x,size_t len);
 __readmemsz__(2,3)

buffer/bs_capacitycheck.c

@@ -0,0 +1,12 @@
+#include "parse.h"
+
+// This function is supposed to tell the caller if there is more data to
+// read. However, we have several limits we could run into. We have our
+// own limit, which we check first, but then, if the bytestream is bound
+// to an I/O stream we should also try to find out if the I/O stream has
+// hit EOF.
+int bs_capacitycheck(struct bytestream* bs,size_t capacity) {
+  if (bs->cur>=bs->max) return 0;	// if EOF or error, return 0
+  if (bs->max - bs->cur < capacity) return 0;	// not EOF but less than that many bytes left
+  return 1;
+}

buffer/bs_get.c

@@ -0,0 +1,112 @@
+#include "parse.h"
+
+unsigned char bs_get(struct bytestream* bs) {
+  unsigned char r;
+  char c;
+  if (bs->cur>=bs->max) {	// EOF or already error state?
+    bs->max=0;	// signal error
+    bs->cur=1;
+    return 0;	// return 0
+  }
+  switch (bs->type) {
+
+  case MEMBUF:
+    r=bs->u.base[bs->cur];
+    break;
+
+  case IOBUF:
+    {
+      int ret=buffer_getc(bs->u.b, &c);
+      if (ret==1) {
+	r=c;
+      } else {
+	bs->max=0;
+	bs->cur=1;
+	return 0;
+      }
+    }
+    break;
+
+  case BSTREAM:
+    r=bs_get(bs->u.bs);
+    break;
+
+  default:
+    r=0;	// cannot happen
+  }
+  ++bs->cur;
+  return r;
+}
+
+int bs_err(struct bytestream* bs) {
+  return (bs->cur > bs->max);
+}
+
+#ifdef UNITTEST
+#include <assert.h>
+
+int main() {
+  struct bytestream bs = BS_FROM_MEMBUF("fnord\nx", 6);
+  int i;
+  char buf[7];
+
+  /* first test: membuf.
+   * See if we get all the bytes we put in and then error is signaled */
+  for (i=0; i<6; ++i) {
+    buf[i] = bs_get(&bs);
+    assert(buf[i] == "fnord\n"[i]);
+    assert(!bs_err(&bs));
+  }
+  buf[6] = bs_get(&bs);
+  /* We put an x there in memory.
+   * If the bytestream range check failed, we'll get 'x', otherwise 0. */
+  assert(buf[6] == 0);
+  assert(bs_err(&bs));
+
+  /* second test: iobuf with no limit. Otherwise the same. */
+
+  struct buffer b;
+  buffer_init_staticcontents(&b, "fnord\nx", 6);	// this will let us read 6 bytes
+  bs_init_iobuf(&bs, &b);
+  for (i=0; i<6; ++i) {
+    buf[i] = bs_get(&bs);
+    assert(buf[i] == "fnord\n"[i]);
+    assert(!bs_err(&bs));
+  }
+  buf[6] = bs_get(&bs);
+  /* We put an x there in memory.
+   * If the bytestream range check failed, we'll get 'x', otherwise 0. */
+  assert(buf[6] == 0);
+  assert(bs_err(&bs));
+
+  /* third test: iobuf with limit. Otherwise the same. */
+  buffer_init_staticcontents(&b, "fnord\nx", 7);	// this will let us read 7 bytes
+
+  bs_init_iobuf_size(&bs, &b, 6);	// but we tell bytestream the limit is 6
+  for (i=0; i<6; ++i) {
+    buf[i] = bs_get(&bs);
+    assert(buf[i] == "fnord\n"[i]);
+    assert(!bs_err(&bs));
+  }
+  buf[6] = bs_get(&bs);
+  /* We put an x there in the backing buffer.
+   * If the bytestream range check failed, we'll get 'x', otherwise 0. */
+  assert(buf[6] == 0);
+  assert(bs_err(&bs));
+
+  /* fourth test: iobuf with EOF */
+  buffer_init_staticcontents(&b, "fnord\nx", 6);
+  bs_init_iobuf(&bs, &b);	// bytestream has no limit but will hit EOF in backing buffer
+  for (i=0; i<6; ++i) {
+    buf[i] = bs_get(&bs);
+    assert(buf[i] == "fnord\n"[i]);
+    assert(!bs_err(&bs));
+  }
+  buf[6] = bs_get(&bs);
+  /* We did not give the bytestream a limit, but the buffer should
+   * refuse to return more. */
+  assert(buf[6] == 0);
+  assert(bs_err(&bs));
+}
+
+#endif

buffer/bs_init_bstream_size.c

@@ -0,0 +1,15 @@
+#include "parse.h"
+
+void bs_init_bstream_size(struct bytestream* bs,struct bytestream* other,size_t maxlen) {
+  bs->type = BSTREAM;
+  // check if we have enough capacity in the parent bytestream
+  if (bs_capacitycheck(other, maxlen)) {
+    bs->cur = 0;
+    bs->max = maxlen;
+  } else {
+    // nope, so set the new stream to error state right out of the box
+    bs->cur = 1;
+    bs->max = 0;
+  }
+  bs->u.bs=other;
+}

buffer/bs_init_iobuf.c

@@ -0,0 +1,8 @@
+#include "parse.h"
+
+void bs_init_iobuf(struct bytestream* bs,struct buffer* b) {
+  bs->type = IOBUF;
+  bs->cur = 0;
+  bs->max = (size_t)-1;
+  bs->u.b=b;
+}

buffer/bs_init_iobuf_size.c

@@ -0,0 +1,8 @@
+#include "parse.h"
+
+void bs_init_iobuf_size(struct bytestream* bs,struct buffer* b,size_t maxlen) {
+  bs->type = IOBUF;
+  bs->cur = 0;
+  bs->max = maxlen;
+  bs->u.b=b;
+}

buffer/bs_init_membuf.c

@@ -0,0 +1,21 @@
+#include "parse.h"
+
+void bs_init_membuf(struct bytestream* bs,const unsigned char* membuf,size_t len) {
+  bs->type = MEMBUF;
+  bs->cur = 0;
+  bs->max = len;
+  bs->u.base=membuf;
+}
+
+#ifdef UNITTEST
+#include <assert.h>
+int main() {
+  static struct bytestream bs;
+  bs_init_membuf(&bs, "fnord\n", 6);
+  char buf[7];
+  int i;
+  for (i=0; i<7; ++i) buf[i]=bs_get(&bs);
+  assert(!memcmp(buf,"fnord\n",7));	// we should have gotten everything and then a 0 byte
+  assert(bs_err(&bs));		// that should have set the error flag
+}
+#endif

buffer/buffer_init_staticcontents.c

@@ -0,0 +1,29 @@
+#include "buffer.h"
+#include <mmap.h>
+
+static ssize_t op() {
+  return 0;
+}
+
+void buffer_init_staticcontents(buffer* b, char* y, size_t len) {
+  b->x=y;
+  b->p=0; b->a=b->n=len;
+  b->fd=-1;
+  b->op=op;
+  b->deinit=0;
+}
+
+#ifdef UNITTEST
+#include <assert.h>
+
+int main() {
+  buffer b;
+  buffer_init_staticcontents(&b, "fnord", 5);
+  char tmp[6];
+  assert(buffer_get(&b, tmp, 6) == 5);
+  assert(!memcmp(tmp,"fnord",5));
+  buffer_init_staticcontents(&b, tmp, sizeof tmp);
+  buffer_puts(&b, "foo\n");
+  assert(!memcmp(tmp, "foo\n", 4));
+}
+#endif

buffer/buffer_init_staticcontents_free.c

@@ -0,0 +1,7 @@
+#include "buffer.h"
+#include <mmap.h>
+
+void buffer_init_staticcontents_free(buffer* b, char* y, size_t len) {
+  buffer_init_staticcontents(b, y, len);
+  b->deinit=buffer_free;
+}

buffer/prs_u16.c

@@ -0,0 +1,17 @@
+#include "parse.h"
+
+uint16_t prs_u16(struct bytestream* bs) {
+  return bs_get(bs) | (bs_get(bs) << 8);
+}
+
+#ifdef UNITTEST
+#include <assert.h>
+
+int main() {
+  struct bytestream bs = BS_FROM_MEMBUF("\x34\x12",2);
+  assert(prs_u16(&bs) == 0x1234);
+  assert(bs_err(&bs) == 0);
+  assert(prs_u16(&bs) == 0);
+  assert(bs_err(&bs));
+}
+#endif

buffer/prs_u16_big.c

@@ -0,0 +1,17 @@
+#include "parse.h"
+
+uint16_t prs_u16_big(struct bytestream* bs) {
+  return (bs_get(bs) << 8) | bs_get(bs);
+}
+
+#ifdef UNITTEST
+#include <assert.h>
+
+int main() {
+  struct bytestream bs = BS_FROM_MEMBUF("\x12\x34",2);
+  assert(prs_u16_big(&bs) == 0x1234);
+  assert(bs_err(&bs) == 0);
+  assert(prs_u16_big(&bs) == 0);
+  assert(bs_err(&bs));
+}
+#endif

buffer/prs_u32.c

@@ -0,0 +1,17 @@
+#include "parse.h"
+
+uint32_t prs_u32(struct bytestream* bs) {
+  return bs_get(bs) | (bs_get(bs) << 8) | (bs_get(bs) << 16) | (bs_get(bs) << 24);
+}
+
+#ifdef UNITTEST
+#include <assert.h>
+
+int main() {
+  struct bytestream bs = BS_FROM_MEMBUF("\x78\x56\x34\x12",4);
+  assert(prs_u32(&bs) == 0x12345678);
+  assert(bs_err(&bs) == 0);
+  assert(prs_u32(&bs) == 0);
+  assert(bs_err(&bs));
+}
+#endif

buffer/prs_u32_big.c

@@ -0,0 +1,17 @@
+#include "parse.h"
+
+uint32_t prs_u32_big(struct bytestream* bs) {
+  return (bs_get(bs) << 24) | (bs_get(bs) << 16) | (bs_get(bs) << 8) | bs_get(bs);
+}
+
+#ifdef UNITTEST
+#include <assert.h>
+
+int main() {
+  struct bytestream bs = BS_FROM_MEMBUF("\x12\x34\x56\x78",4);
+  assert(prs_u32_big(&bs) == 0x12345678);
+  assert(bs_err(&bs) == 0);
+  assert(prs_u32_big(&bs) == 0);
+  assert(bs_err(&bs));
+}
+#endif

buffer/prs_u64.c

@@ -0,0 +1,21 @@
+#include "parse.h"
+
+uint64_t prs_u64(struct bytestream* bs) {
+  unsigned int i;
+  uint64_t x = bs_get(bs);
+  for (i=1; i<8; ++i)
+    x |= ((uint64_t)bs_get(bs) << (i*8));
+  return x;
+}
+
+#ifdef UNITTEST
+#include <assert.h>
+
+int main() {
+  struct bytestream bs = BS_FROM_MEMBUF("\x78\x56\x34\x12\xef\xbe\xad\xde",8);
+  assert(prs_u64(&bs) == 0xdeadbeef12345678);
+  assert(bs_err(&bs) == 0);
+  assert(prs_u64(&bs) == 0);
+  assert(bs_err(&bs));
+}
+#endif

buffer/prs_u64_big.c

@@ -0,0 +1,21 @@
+#include "parse.h"
+
+uint64_t prs_u64_big(struct bytestream* bs) {
+  unsigned int i;
+  uint64_t x = bs_get(bs);
+  for (i=1; i<8; ++i)
+    x = (x << 8) | bs_get(bs);
+  return x;
+}
+
+#ifdef UNITTEST
+#include <assert.h>
+
+int main() {
+  struct bytestream bs = BS_FROM_MEMBUF("\xde\xad\xbe\xef\x12\x34\x56\x78",8);
+  assert(prs_u64_big(&bs) == 0xdeadbeef12345678);
+  assert(bs_err(&bs) == 0);
+  assert(prs_u64_big(&bs) == 0);
+  assert(bs_err(&bs));
+}
+#endif

scan.h

@@ -25,7 +25,7 @@ extern "C" {
 #define __readmemsz__(a,b)
 #endif
 
-/* This file declared functions used to decode / scan / unmarshal
+/* This file declares functions used to decode / scan / unmarshal
  * integer or string values from a buffer.
  * The first argument is always the source buffer, the second argument
  * is a pointer to the destination (where to store the result). The