|
|
@ -256,6 +256,7 @@ static void httpresponse( const int64 s, char *data, size_t l ) {
|
|
|
|
unsigned short port = htons(6881);
|
|
|
|
unsigned short port = htons(6881);
|
|
|
|
ssize_t len;
|
|
|
|
ssize_t len;
|
|
|
|
size_t reply_size = 0, reply_off;
|
|
|
|
size_t reply_size = 0, reply_off;
|
|
|
|
|
|
|
|
tai6464 t;
|
|
|
|
|
|
|
|
|
|
|
|
/* Touch l and d in case it is unused */
|
|
|
|
/* Touch l and d in case it is unused */
|
|
|
|
l = l; d = d;
|
|
|
|
l = l; d = d;
|
|
|
@ -394,6 +395,9 @@ LOG_TO_STDERR( "sync: %d.%d.%d.%d\n", h->ip[0], h->ip[1], h->ip[2], h->ip[3] );
|
|
|
|
#endif
|
|
|
|
#endif
|
|
|
|
/* Pass this task to the worker thread */
|
|
|
|
/* Pass this task to the worker thread */
|
|
|
|
h->flag |= STRUCT_HTTP_FLAG_WAITINGFORTASK;
|
|
|
|
h->flag |= STRUCT_HTTP_FLAG_WAITINGFORTASK;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/* Clients waiting for us should not easily timeout */
|
|
|
|
|
|
|
|
taia_uint( &t, 0 ); io_timeout( s, t );
|
|
|
|
fullscrape_deliver( s, format );
|
|
|
|
fullscrape_deliver( s, format );
|
|
|
|
io_dontwantread( s );
|
|
|
|
io_dontwantread( s );
|
|
|
|
return;
|
|
|
|
return;
|
|
|
@ -412,20 +416,25 @@ LOG_TO_STDERR( "sync: %d.%d.%d.%d\n", h->ip[0], h->ip[1], h->ip[2], h->ip[3] );
|
|
|
|
|
|
|
|
|
|
|
|
/* Full scrape... you might want to limit that */
|
|
|
|
/* Full scrape... you might want to limit that */
|
|
|
|
if( !byte_diff( data, 12, "scrape HTTP/" ) ) {
|
|
|
|
if( !byte_diff( data, 12, "scrape HTTP/" ) ) {
|
|
|
|
LOG_TO_STDERR( "[%08d] scrp: %d.%d.%d.%d - FULL SCRAPE\n", (unsigned int)(g_now - ot_start_time), h->ip[0], h->ip[1], h->ip[2], h->ip[3] );
|
|
|
|
|
|
|
|
#ifdef _DEBUG_HTTPERROR
|
|
|
|
|
|
|
|
write( 2, debug_request, l );
|
|
|
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
format = 0;
|
|
|
|
format = 0;
|
|
|
|
#ifdef WANT_COMPRESSION_GZIP
|
|
|
|
#ifdef WANT_COMPRESSION_GZIP
|
|
|
|
if( strnstr( d, "gzip", l ) ) {
|
|
|
|
d[l-1] = 0;
|
|
|
|
|
|
|
|
if( strstr( d, "gzip" ) ) {
|
|
|
|
h->flag |= STRUCT_HTTP_FLAG_GZIP;
|
|
|
|
h->flag |= STRUCT_HTTP_FLAG_GZIP;
|
|
|
|
format = TASK_FLAG_GZIP;
|
|
|
|
format = TASK_FLAG_GZIP;
|
|
|
|
}
|
|
|
|
LOG_TO_STDERR( "[%08d] scrp: %d.%d.%d.%d - FULL SCRAPE GZIP\n", (unsigned int)(g_now - ot_start_time), h->ip[0], h->ip[1], h->ip[2], h->ip[3] );
|
|
|
|
|
|
|
|
} else
|
|
|
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
LOG_TO_STDERR( "[%08d] scrp: %d.%d.%d.%d - FULL SCRAPE\n", (unsigned int)(g_now - ot_start_time), h->ip[0], h->ip[1], h->ip[2], h->ip[3] );
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
#ifdef _DEBUG_HTTPERROR
|
|
|
|
|
|
|
|
write( 2, debug_request, l );
|
|
|
|
#endif
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
|
|
/* Pass this task to the worker thread */
|
|
|
|
/* Pass this task to the worker thread */
|
|
|
|
h->flag |= STRUCT_HTTP_FLAG_WAITINGFORTASK;
|
|
|
|
h->flag |= STRUCT_HTTP_FLAG_WAITINGFORTASK;
|
|
|
|
|
|
|
|
/* Clients waiting for us should not easily timeout */
|
|
|
|
|
|
|
|
taia_uint( &t, 0 ); io_timeout( s, t );
|
|
|
|
fullscrape_deliver( s, TASK_FULLSCRAPE | format );
|
|
|
|
fullscrape_deliver( s, TASK_FULLSCRAPE | format );
|
|
|
|
io_dontwantread( s );
|
|
|
|
io_dontwantread( s );
|
|
|
|
return;
|
|
|
|
return;
|
|
|
|