$ make WERROR=-Werror now builds with -Werror

add some single char escaping routines to fmt.h
pull in html5 entities from w3c and use those to do a proper scan_html decoding
fix an off-by-one in fmt_to_array
add a ton of unit tests for the fmt routines
master
leitner 11 years ago
parent dd4f86fbfa
commit 35942878c2

@ -22,3 +22,4 @@ trysendfile
havealloca.h
uudecode
haveuint128.h
ent

@ -13,7 +13,7 @@ LIBS=byte.a fmt.a scan.a str.a uint.a open.a stralloc.a unix.a socket.a \
buffer.a mmap.a taia.a tai.a dns.a case.a mult.a array.a io.a \
textcode.a cdb.a
all: $(LIBS) libowfat.a libsocket t
all: ent $(LIBS) libowfat.a libsocket t
CROSS=
#CROSS=i686-mingw-
@ -21,9 +21,13 @@ CC=$(CROSS)gcc
CFLAGS=-pipe -W -Wall -O2 -fomit-frame-pointer
#CFLAGS=-pipe -Os -march=pentiumpro -mcpu=pentiumpro -fomit-frame-pointer -fschedule-insns2 -Wall
ent: ent.c
gcc -g -o ent ent.c
# CFLAGS += -fstrict-aliasing -Wstrict-aliasing=2
CFLAGS += -D_REENTRANT
WERROR=
CFLAGS += -D_REENTRANT $(WERROR)
# startrip
ifneq ($(DEBUG),)
@ -326,3 +330,11 @@ windoze:
windoze64:
$(MAKE) DIET= CROSS=x86_64-mingw32-
update:
dl -n http://www.w3.org/TR/html5/entities.json
entities.h: entities.json ent
./ent
scan_html.o: entities.h

@ -836,3 +836,8 @@ windoze:
windoze64:
$(MAKE) DIET= CROSS=x86_64-mingw32-
entities.h: entities.json ent
./ent
scan_html.o: entities.h

228
ent.c

@ -0,0 +1,228 @@
#include <stdio.h>
#include <ctype.h>
#include <string.h>
#include <stdlib.h>
#include "scan.h"
#include <assert.h>
#include "scan/scan_ulong.c"
#include "scan/scan_ulongn.c"
#include "fmt/fmt_utf8.c"
#include "fmt/fmt_escapecharc.c"
char tmp[20];
char tmp2[20];
size_t n,m;
unsigned long l;
struct entity {
const char* entity;
char utf8[10];
struct entity* next;
}* root,** cur=&root;
struct letter {
char c;
struct letters* weiter;
uint32_t marshaled; // lower 8 bits: char. rest: ofs from start of marshaled blob
};
struct letters {
size_t n;
struct letter liste[256];
};
struct letters* d;
size_t nodes,datasize;
void addword(struct letters** s,const char* t, void* pointer) {
size_t i;
if (!*s) {
*s=malloc(sizeof(**s));
memset(*s,0,sizeof(**s));
(*s)->liste[0].c='?';
}
i=(unsigned char)*t;
if ((*s)->liste[i].c==*t) {
if (!*t) {
datasize+=strlen((char*)pointer)+1;
(*s)->liste[i].weiter=pointer;
} else
addword(&(*s)->liste[i].weiter,t+1,pointer);
return;
}
++nodes;
(*s)->n++;
(*s)->liste[i].c=*t;
if (!*t) {
datasize+=strlen((char*)pointer)+1;
(*s)->liste[i].weiter=pointer;
} else {
(*s)->liste[i].weiter=0;
addword(&(*s)->liste[i].weiter,t+1,pointer);
}
}
void dump(struct letters* s,size_t depth) {
size_t i,j;
if (!s) return;
for (i=0; i<256; ++i) {
if (s->liste[i].c!=i) continue;
for (j=0; j<depth; ++j) printf(" ");
printf("'%c' -> {\n",s->liste[i].c);
if (s->liste[i].c)
dump(s->liste[i].weiter,depth+1);
for (j=0; j<depth; ++j) printf(" ");
printf("}\n");
}
}
size_t used;
size_t useddata;
char* heap;
uint32_t* marshaled;
char* data;
size_t marshalhelper(struct letters* s) {
size_t i;
uint32_t myindex=used;
if (!s) return 0;
used+=s->n;
assert(used<nodes+2);
for (i=1; i!=0; ++i) { // start at 1, go to 256, then access modulo 256; effect: sort but put 0 last
uint32_t x;
i&=0xff;
// printf("%c ",i);
if (s->liste[i].c!=i) {
if (i==0) return;
continue;
}
// printf("marshalhelper: %c\n",i);
x=(unsigned char)s->liste[i].c;
if (!x) {
size_t l=strlen((char*)s->liste[i].weiter)+1;
// puts((char*)s->liste[i].weiter);
x|=useddata<<8;
assert(useddata+l<=datasize);
memcpy(data+useddata,s->liste[i].weiter,l);
useddata+=l;
marshaled[++myindex]=x;
return;
} else {
x|=(used+1)<<8;
marshalhelper(s->liste[i].weiter);
}
marshaled[++myindex]=x;
}
// printf("return\n");
}
void marshal(struct letters* s) {
fprintf(stderr,"nodes=%lu, datasize=%lu\n",nodes,datasize);
heap=malloc((nodes+1)*sizeof(uint32_t)+datasize);
if (!heap) return;
marshaled=(uint32_t*)heap;
marshaled[0]=nodes+1;
data=heap+(nodes+1)*sizeof(uint32_t);
marshalhelper(s);
fprintf(stderr,"actually used: %lu nodes, %lu bytes data\n",used,useddata);
}
char* lookup(char* ds,size_t ofs,const char* t) {
uint32_t* tab=(uint32_t*)ds;
if (ofs>tab[0]) return 0;
while (ofs<tab[0]) {
unsigned char ch=tab[ofs]&0xff;
if (ch==(unsigned char)*t) {
if (!ch)
return ds+tab[0]*sizeof(uint32_t)+(tab[ofs]>>8);
else
return lookup(ds,tab[ofs]>>8,t+1);
} else
++ofs;
if (!ch) break;
}
return NULL;
}
int main() {
FILE* f=fopen("entities.json","r");
char buf[256];
if (!f) return 1;
#if 0
puts("struct { const char* entity; const char* utf8; } codepoints[] = {");
#endif
while (fgets(buf,sizeof(buf),f)) {
char* s,* entity;
size_t ul;
if (!isspace(buf[0])) continue;
for (s=buf; *s && *s!='"'; ++s) ; // skip whitespace
if (!*s=='"') continue;
++s;
entity=s;
if (*entity!='&') continue; ++entity; ++s;
for (; *s && *s!='"'; ++s) ; // skip to end of entity
if (!*s=='"') continue;
if (s[-1]!=';') continue;
s[-1]=0; ++s;
s=strchr(s,'[');
if (!s) continue;
n=0;
#if 0
printf(" { \"%s\", \"",entity);
#endif
++s;
*cur=malloc(sizeof(**cur));
(*cur)->next=0;
if (!((*cur)->entity=strdup(entity))) return 1;
ul=0;
do {
while (isspace(*s)) ++s;
m=scan_ulong(s,&l);
if (!m) return 2;
s+=n;
n=fmt_utf8(tmp,l);
if (ul+n>sizeof((*cur)->utf8)) return 3;
memcpy((*cur)->utf8+ul,tmp,n);
ul+=n;
#if 0
{
size_t i;
for (i=0; i<n; ++i) {
fwrite(tmp2,fmt_escapecharc(tmp2,(unsigned char)tmp[i]),1,stdout);
}
}
#endif
if (*s==']') break;
} while (*s==',');
#if 0
puts("\" },");
#endif
addword(&d,(*cur)->entity,(*cur)->utf8);
}
fclose(f);
// dump(d,0);
marshal(d);
{
FILE* f=fopen("entities.h","w");
size_t i;
fprintf(f,"struct {\n uint32_t tab[%lu];\n char data[%lu];\n} entities = {\n {",marshaled[0],datasize);
for (i=0; i<marshaled[0]; ++i) {
if (i%8 == 0) fprintf(f,"\n ");
fprintf(f,"0x%x,",marshaled[i]);
}
fprintf(f,"\n } , {");
for (i=0; i<datasize; ++i) {
if (i%16 == 0) fprintf(f,"\n ");
fprintf(f,"0x%x,",data[i]&0xff);
}
fprintf(f,"\n }\n};");
fclose(f);
}
// puts(lookup(heap,1,"zwnj"));
#if 0
puts("};");
#endif
return 0;
}

File diff suppressed because it is too large Load Diff

31
fmt.h

@ -145,12 +145,31 @@ size_t fmt_utf8(char* dest,uint32_t n); /* can store 0-0x7fffffff */
size_t fmt_asn1derlength(char* dest,unsigned long long l); /* 0-0x7f: 1 byte, above that 1+bytes_needed bytes */
size_t fmt_asn1dertag(char* dest,unsigned long long l); /* 1 byte for each 7 bits; upper bit = more bytes coming */
/* '&' -> '&amp;', '<' -> '&lt;'
* 0 is rejected
* control characters except \t \r \n are written in escaped form
* characters from d780 to dfff and fffe and ffff and everything above 10ffff are also rejected.
* everything else is passed through encoded as UTF-8 */
size_t fmt_xmlescape(char* dest,uint32_t ch);
/* Marshaling helper functions.
* Escape one character, no matter if it needs escaping or not.
* The functions will reject characters that cannot be represented but
* not characters that the standard says should never occur. The idea
* is to make these functions useful for creating bad encoding for
* penetration testing.
* Depending on the escaping method, the input character (uint32_t, a
* unicode codepoint) may be limited to 0x7f, 0xff or 0x10ffff. */
/* XML escaping: '&' -> '&amp;', '<' -> '&lt;', 'ö' -> '&#xf6;' */
size_t fmt_escapecharxml(char* dest,uint32_t ch);
/* HTML escaping is the same as XML escaping. */
size_t fmt_escapecharhtml(char* dest,uint32_t ch);
/* JSON escaping: '\' -> '\\', '"' -> '\"', 'ö' -> '\u00f6' */
size_t fmt_escapecharjson(char* dest,uint32_t ch);
/* MIME quoted-printable escaping: 'ö' -> '=f6', characters > 0xff not supported */
size_t fmt_escapecharquotedprintable(char* dest,uint32_t ch);
/* MIME quoted-printable escaping with UTF-8: 'ö' -> '=c3=b6', characters > 0x7fffffff not supported */
size_t fmt_escapecharquotedprintableutf8(char* dest,uint32_t ch);
/* C escaping: '\' -> '\\', newline -> '\n', 0xc2 -> '\302' */
size_t fmt_escapecharc(char* dest,uint32_t ch);
/* internal functions, may be independently useful */
char fmt_tohex(char c);

@ -0,0 +1,36 @@
#include "fmt.h"
static void fmt_oct3(char* dest,uint8_t w) {
dest[2]=(w&7)+'0'; w>>=3;
dest[1]=(w&7)+'0'; w>>=3;
dest[0]=(w&7)+'0';
}
size_t fmt_escapecharc(char* dest,uint32_t ch) {
char c;
if (ch>0xff) return 0;
switch (ch) {
case '\a': c='a'; goto doescape;
case '\b': c='b'; goto doescape;
case 0x1b: c='e'; goto doescape;
case '\f': c='f'; goto doescape;
case '\n': c='n'; goto doescape;
case '\r': c='r'; goto doescape;
case '\t': c='t'; goto doescape;
case '\v': c='v'; goto doescape;
case '\\':
c='\\';
doescape:
if (dest) {
dest[0]='\\';
dest[1]=c;
}
return 2;
default:
if (dest) {
dest[0]='\\';
fmt_oct3(dest+1,ch&0xff);
}
return 4;
}
}

@ -0,0 +1,9 @@
#ifndef __GNUC__
#include "fmt.h"
size_t fmt_escapecharhtml(char* dest,uint32_t ch) {
return fmt_escapecharxml(dest,ch);
}
#endif

@ -0,0 +1,47 @@
#include "fmt.h"
static void fmt_hex4(char* dest,uint16_t w) {
dest[3]=fmt_tohex(w&0xf); w>>=4;
dest[2]=fmt_tohex(w&0xf); w>>=4;
dest[1]=fmt_tohex(w&0xf); w>>=4;
dest[0]=fmt_tohex(w&0xf);
}
size_t fmt_escapecharjson(char* dest,uint32_t ch) {
size_t n;
switch (ch) {
case '\b':
ch='b'; goto simple;
case '\n':
ch='n'; goto simple;
case '\r':
ch='r'; goto simple;
case '"':
case '\\':
case '/':
simple:
if (dest) {
dest[0]='\\';
dest[1]=ch;
}
return 2;
}
if (ch>0xffff) {
if (ch>0x10ffff) return 0; // highest representable unicode codepoint
if (dest) {
dest[0]='\\';
dest[1]='u';
fmt_hex4(dest+2,0xd800 | (((ch-0x10000)>>10)&0x3ff));
dest+=6;
}
ch=(ch&0x3ff)|0xdc00;
n=6;
} else
n=0;
if (dest) {
dest[0]='\\';
dest[1]='u';
fmt_hex4(dest+2,ch);
}
return n+6;
}

@ -0,0 +1,11 @@
#include "fmt.h"
size_t fmt_escapecharquotedprintable(char* dest,uint32_t ch) {
if (ch>0xff) return 0;
if (dest) {
dest[0]='=';
dest[2]=fmt_tohex(ch&0xf); ch>>=4;
dest[1]=fmt_tohex(ch&0xf);
}
return 3;
}

@ -0,0 +1,10 @@
#include "fmt.h"
size_t fmt_escapecharquotedprintableutf8(char* dest,uint32_t ch) {
char buf[FMT_UTF8];
size_t i,o,j=fmt_utf8(buf,ch);
if (!dest) return j*3;
for (i=o=0; i<j; ++i)
o+=fmt_escapecharquotedprintable(dest+o,(unsigned char)buf[i]);
return o;
}

@ -0,0 +1,33 @@
#include "fmt.h"
size_t fmt_escapecharxml(char* dest,uint32_t ch) {
char a[FMT_LONG], b[FMT_XLONG];
const char* s;
size_t i,j;
switch (ch) {
case '&': s="&amp;"; goto string;
case '<': s="&lt;"; goto string;
case '>': s="&gt;"; goto string;
case '\'': s="&apos;"; goto string;
case '"': s="&quot;"; goto string;
default:
a[i=fmt_ulong(a,ch)]=0;
b[0]='x';
b[j=fmt_xlong(b+1,ch)+1]=0;
s=a;
if (i>j) { s=b; i=j; }
if (dest) {
dest[0]='&';
dest[1]='#';
byte_copy(dest+2,i,s);
dest[i+2]=';';
}
return i+3;
}
string:
return fmt_str(dest,s);
}
#ifdef __GNUC__
size_t fmt_escapecharhtml(char* dest,uint32_t ch) __attribute__((__alias__("fmt_escapecharxml")));
#endif

@ -104,4 +104,28 @@ int main() {
zap(); assert(fmt_asn1dertag(buf,0xc2)==2 && byte_equal(buf,3,"\x81\x42_"));
zap(); assert(fmt_strm(buf,"hell","o, worl","d!\n")==14 && byte_equal(buf,15,"hello, world!\n_"));
assert(fmt_escapecharxml(NULL,0xc2)==6);
zap(); assert(fmt_escapecharxml(buf,0xc2)==6 && byte_equal(buf,7,"&#194;_"));
assert(fmt_escapecharxml(NULL,0)==4);
zap(); assert(fmt_escapecharxml(buf,0)==4 && byte_equal(buf,5,"&#0;_"));
assert(fmt_escapecharjson(NULL,'\\')==2); // "\\"
zap(); assert(fmt_escapecharjson(buf,'\\')==2 && byte_equal(buf,3,"\\\\_"));
assert(fmt_escapecharjson(NULL,0xc2)==6); // "\u00c2"
zap(); assert(fmt_escapecharjson(buf,0xc2)==6 && byte_equal(buf,7,"\\u00c2_"));
assert(fmt_escapecharjson(NULL,0x1d11e)==12); // "\ud834\xdd1e"
zap(); assert(fmt_escapecharjson(buf,0x1d11e)==12 && byte_equal(buf,13,"\\ud834\\udd1e_"));
assert(fmt_escapecharquotedprintable(NULL,'=')==3); // =3d
zap(); assert(fmt_escapecharquotedprintable(buf,'=')==3 && byte_equal(buf,4,"=3d_"));
assert(fmt_escapecharquotedprintable(NULL,0xf6)==3); // =f6
zap(); assert(fmt_escapecharquotedprintable(buf,0xf6)==3 && byte_equal(buf,4,"=f6_"));
assert(fmt_escapecharquotedprintable(NULL,0x100)==0);
assert(fmt_escapecharquotedprintableutf8(NULL,'=')==3); // =3d
zap(); assert(fmt_escapecharquotedprintableutf8(buf,'=')==3 && byte_equal(buf,4,"=3d_"));
assert(fmt_escapecharquotedprintableutf8(NULL,0xf6)==6); // =c3=b6
zap(); assert(fmt_escapecharquotedprintableutf8(buf,0xf6)==6 && byte_equal(buf,7,"=c3=b6_"));
}

@ -19,5 +19,12 @@ int main() {
assert(!array_failed(&a));
write(1,array_start(&a),array_bytes(&a)); write(1,"\n",1);
array_trunc(&a);
strcpy(buf,"&Auml;cht fn&ouml;rdig.");
size_t n;
scan_html(buf,buf,&n);
buf[n]=0;
puts(buf);
return 0;
}

@ -4,7 +4,7 @@
void fmt_to_array(size_t (*func)(char*,const char*,size_t),
array* a,const char* src,size_t len) {
size_t needed=func(0,src,len);
if (array_bytes(a)+needed>needed &&
if (array_bytes(a)+needed>=needed &&
array_allocate(a,1,array_bytes(a)+needed-1)) {
char* x=((char*)array_start(a))+array_bytes(a)-needed;
func(x,src,len);

@ -1,23 +1,63 @@
#include <inttypes.h>
#include <string.h>
#include "entities.h"
#include "fmt.h"
#include "textcode.h"
#include "haveinline.h"
#include "scan.h"
#include "case.h"
#include "str.h"
static const char* lookup(size_t ofs,const char* t) {
if (ofs>entities.tab[0]) return 0;
while (ofs<entities.tab[0]) {
unsigned char ch=entities.tab[ofs]&0xff;
if (ch==(unsigned char)*t || (!ch && *t==';')) {
if (!ch || *t==';')
return entities.data+(entities.tab[ofs]>>8);
else
return lookup(entities.tab[ofs]>>8,t+1);
} else
++ofs;
if (!ch) break;
}
return NULL;
}
size_t scan_html(const char *src,char *dest,size_t *destlen) {
register const unsigned char* s=(const unsigned char*) src;
size_t written=0,i;
for (i=0; s[i]; ++i) {
if (s[i]=='&') {
if (case_starts((const char*)s+i+1,"amp;")) {
dest[written]='&';
i+=4;
} else if (case_starts((const char*)s+i+1,"lt;")) {
dest[written]='<';
i+=3;
} else if (case_starts((const char*)s+i+1,"gt;")) {
dest[written]='>';
i+=3;
const char* utf8;
if (s[i+1]=='#') {
unsigned long l;
size_t j;
if ((s[i+2]&~32)=='X') {
j=scan_xlong(src+i+3,&l);
if (!j) j+=3;
} else {
j=scan_ulong(src+i+2,&l);
if (!j) j+=3;
}
if (s[i+j]==';') {
i+=j;
written+=fmt_utf8(dest+written,l);
} else {
dest[written++]='&';
}
continue;
}
utf8=lookup(1,src+i+1);
if (utf8) {
size_t l=strlen(utf8);
memcpy(dest+written,utf8,l);
written+=l;
i+=2+str_chr(src+i+2,';');
continue;
} else
dest[written]='&';
} else if (s[i]=='<') {
if (case_starts((const char*)s+i+1,"br>")) {
dest[written]='\n';

@ -5,6 +5,8 @@
#include <malloc.h>
#endif
main() {
int main() {
char* c=alloca(23);
(void)c;
return 0;
}

@ -16,4 +16,6 @@ int main() {
hdr.headers=v; hdr.hdr_cnt=17;
hdr.trailers=v+17; hdr.trl_cnt=23;
r=sendfile(0,1,37,42,&hdr,&sbytes,0);
(void)r;
return 0;
}

@ -1,5 +1,6 @@
#include <inttypes.h>
#include <sys/epoll.h>
#include <stdio.h>
int main() {
int efd=epoll_create(10);
@ -9,10 +10,11 @@ int main() {
x.data.fd=0;
if (epoll_ctl(efd,EPOLL_CTL_ADD,0 /* fd */,&x)==-1) return 111;
{
int i,n;
int n;
struct epoll_event y[100];
if ((n=epoll_wait(efd,y,100,1000))==-1) return 111;
if (n>0)
printf("event %d on fd #%d\n",y[0].events,y[0].data.fd);
}
return 0;
}

@ -2,7 +2,9 @@
#include <sys/socket.h>
#include <netinet/in.h>
main() {
int main() {
struct sockaddr_in6 sa;
sa.sin6_family = PF_INET6;
(void)sa;
return 0;
}

@ -5,4 +5,6 @@
int main() {
static char ifname[IFNAMSIZ];
char *tmp=if_indextoname(0,ifname);
(void)tmp;
return 0;
}

@ -2,8 +2,10 @@
#include <sys/socket.h>
#include <netinet/in.h>
main() {
int main() {
struct sockaddr_in6 sa;
sa.sin6_family = PF_INET6;
sa.sin6_scope_id = 23;
(void)sa;
return 0;
}

@ -55,6 +55,7 @@ int main() {
p.trailer_length=6;
if (send_file(&destfd,&p,0)>=0)
printf("sent %lu bytes.\n",p.bytes_sent);
return 0;
}
#elif defined(__linux__)
@ -77,6 +78,7 @@ int main() {
off_t r=sendfile(1,fd,&o,23);
if (r!=-1)
printf("sent %llu bytes.\n",r);
return 0;
}
#else

@ -3,6 +3,7 @@
#include <sys/poll.h>
#include <signal.h>
#include <fcntl.h>
#include <stdio.h>
int main() {
int signum=SIGRTMIN+1;
@ -36,4 +37,5 @@ int main() {
}
}
}
return 0;
}

@ -7,6 +7,8 @@
#include <netinet/in.h>
#endif
main() {
int main() {
socklen_t t;
(void)t;
return 0;
}

@ -2,6 +2,8 @@
#include <sys/socket.h>
#include <netinet/in.h>
main() {
int main() {
int fd=socket(PF_INET,SOCK_STREAM,IPPROTO_TCP);
(int)fd;
return 0;
}

Loading…
Cancel
Save