Add a stat option the count bucket stalls

Add an iterator for all torrents. It's slower but for stats it's okay.
Move some stats collection stuff to the new iterator. More to come.
Start a "report all stats" page.
Start fixing the code to identify "busy" networks.
Add the concept of hosts allowed to proxy.
Add a parser for the X-Forwarded-For: HTTP header.
Clean up HTTP Header handling code. (Remove some left overs of now vanished sync code).

opentracker.c

@@ -135,10 +135,8 @@ static ssize_t handle_read( const int64 sock, struct ot_workstruct *ws ) {
 
   array_catb( &cookie->data.request, ws->inbuf, byte_count );
 
-  if( array_failed( &cookie->data.request ) )
+  if( array_failed( &cookie->data.request ) ||
-    return http_issue_error( sock, ws, CODE_HTTPERROR_500 );
+      array_bytes( &cookie->data.request ) > 8192 )
-
-  if( array_bytes( &cookie->data.request ) > 8192 )
     return http_issue_error( sock, ws, CODE_HTTPERROR_500 );
 
   if( !memchr( array_start( &cookie->data.request ), '\n', array_bytes( &cookie->data.request ) ) )
@@ -369,6 +367,11 @@ int parse_configfile( char * config_filename ) {
     } else if(!byte_diff(p, 12, "access.stats" ) && isspace(p[12])) {
       if( !scan_ip6( p+13, tmpip )) goto parse_error;
       accesslist_blessip( tmpip, OT_PERMISSION_MAY_STAT );
+#endif
+#ifdef WANT_IP_FROM_PROXY
+    } else if(!byte_diff(p, 12, "access.proxy" ) && isspace(p[12])) {
+      if( !scan_ip6( p+13, tmpip )) goto parse_error;
+      accesslist_blessip( tmpip, OT_PERMISSION_MAY_PROXY );
 #endif
     } else if(!byte_diff(p, 20, "tracker.redirect_url" ) && isspace(p[20])) {
       set_config_option( &g_redirecturl, p+21 );

ot_accesslist.c

@@ -124,6 +124,7 @@ int accesslist_blessip( ot_ip6 ip, ot_permissions permissions ) {
     if( permissions & OT_PERMISSION_MAY_STAT       ) off += snprintf( _debug+off, 512-off, " may_fetch_stats" );
     if( permissions & OT_PERMISSION_MAY_LIVESYNC   ) off += snprintf( _debug+off, 512-off, " may_sync_live" );
     if( permissions & OT_PERMISSION_MAY_FULLSCRAPE ) off += snprintf( _debug+off, 512-off, " may_fetch_fullscrapes" );
+    if( permissions & OT_PERMISSION_MAY_PROXY      ) off += snprintf( _debug+off, 512-off, " may_proxy" );
     if( !permissions ) off += snprintf( _debug+off, sizeof(_debug)-off, " nothing\n" );
     _debug[off++] = '.';
     write( 2, _debug, off );

ot_accesslist.h

@@ -27,7 +27,8 @@ extern char *g_accesslist_filename;
 typedef enum {
   OT_PERMISSION_MAY_FULLSCRAPE = 0x1,
   OT_PERMISSION_MAY_STAT       = 0x2,
-  OT_PERMISSION_MAY_LIVESYNC   = 0x4
+  OT_PERMISSION_MAY_LIVESYNC   = 0x4,
+  OT_PERMISSION_MAY_PROXY      = 0x8
 } ot_permissions;
 
 int  accesslist_blessip( ot_ip6 ip, ot_permissions permissions );

ot_http.c

@@ -312,6 +312,7 @@ static ssize_t http_handle_announce( const int64 sock, struct ot_workstruct *ws,
   unsigned short    port = htons(6881);
   char             *write_ptr;
   ssize_t           len;
+  struct http_data *cookie = io_getcookie( sock );    
 
   /* This is to hack around stupid clients that send "announce ?info_hash" */
   if( read_ptr[-1] != '?' ) {
@@ -320,7 +321,33 @@ static ssize_t http_handle_announce( const int64 sock, struct ot_workstruct *ws,
     ++read_ptr;
   }
 
-  OT_SETIP( &peer, ((struct http_data*)io_getcookie( sock ) )->ip );
+#ifdef WANT_IP_FROM_PROXY
+  if( accesslist_isblessed( cookie->ip, OT_PERMISSION_MAY_PROXY ) ) {
+    ot_ip6 proxied_ip;
+    char *fwd, *fwd_new = ws->request;
+
+    /* Zero terminate for string routines. Normally we'd only overwrite bollocks */
+    ws->request[ws->request_size-1] = 0;
+
+    /* Find last occurence of the forwarded header */
+    do {
+      fwd = fwd_new;
+      fwd_new = strcasestr( fwd_new, "\nX-Forwarded-For:" );
+    } while( fwd_new );
+
+    /* Skip spaces between : and the ip address */
+    if( fwd ) {
+      fwd += 18; /* sizeof( "\nX-Forwarded-For:" ) */
+      while( *fwd == ' ' ) ++fwd;
+    }
+
+    if( fwd && scan_ip6( fwd, proxied_ip ) )
+      OT_SETIP( &peer, proxied_ip );
+    else
+      OT_SETIP( &peer, cookie->ip );
+  }
+#endif
+  OT_SETIP( &peer, cookie->ip );
   OT_SETPORT( &peer, &port );
   OT_PEERFLAG( &peer ) = 0;
   numwant = 50;

ot_mutex.c

@@ -18,6 +18,7 @@
 /* Opentracker */
 #include "trackerlogic.h"
 #include "ot_mutex.h"
+#include "ot_stats.h"
 
 /* #define MTX_DBG( STRING ) fprintf( stderr, STRING ) */
 #define MTX_DBG( STRING )
@@ -47,8 +48,10 @@ static int bucket_check( int bucket ) {
 
   /* See, if bucket is already locked */
   for( i=0; i<bucket_locklist_count; ++i )
-    if( bucket_locklist[ i ] == bucket )
+    if( bucket_locklist[ i ] == bucket ) {
+      stats_issue_event( EVENT_BUCKET_LOCKED, 0, 0 );
       return -1;
+    }
 
   return 0;
 }

ot_stats.c

@@ -50,12 +50,13 @@ static unsigned long long ot_full_scrape_size = 0;
 static unsigned long long ot_failed_request_counts[CODE_HTTPERROR_COUNT];
 static unsigned long long ot_renewed[OT_PEER_TIMEOUT];
 static unsigned long long ot_overall_sync_count;
+static unsigned long long ot_overall_stall_count;
 
 static time_t ot_start_time;
 
 #ifdef WANT_LOG_NETWORKS
 #define STATS_NETWORK_NODE_BITWIDTH  8
-#define STATS_NETWORK_NODE_MAXDEPTH  3
+#define STATS_NETWORK_NODE_MAXDEPTH  16
 
 #define STATS_NETWORK_NODE_BITMASK ((1<<STATS_NETWORK_NODE_BITWIDTH)-1)
 #define STATS_NETWORK_NODE_COUNT    (1<<STATS_NETWORK_NODE_BITWIDTH)
@@ -68,8 +69,9 @@ union stats_network_node {
 
 static stats_network_node *stats_network_counters_root = NULL;
 
-static int stat_increase_network_count( stats_network_node **node, int depth, uint32_t ip ) {
+static int stat_increase_network_count( stats_network_node **node, int depth, uintptr_t ip ) {
-  int foo = ( ip >> ( 32 - STATS_NETWORK_NODE_BITWIDTH * ( ++depth ) ) ) & STATS_NETWORK_NODE_BITMASK;
+  ot_ip6 *_ip = (ot_ip6*)ip;
+  int foo = (*_ip)[depth];
   
   if( !*node ) {
     *node = malloc( sizeof( stats_network_node ) );
@@ -157,6 +159,20 @@ static size_t stats_return_busy_networks( char * reply ) {
 
 #endif
 
+typedef struct {
+  unsigned long long torrent_count;
+  unsigned long long peer_count;
+  unsigned long long seed_count;
+} torrent_stats;
+
+static int torrent_statter( ot_torrent *torrent, uintptr_t data ) {
+  torrent_stats *stats = (torrent_stats*)data;
+  stats->torrent_count++;
+  stats->peer_count += torrent->peer_list->peer_count;
+  stats->seed_count += torrent->peer_list->seed_count;
+  return 0;
+}
+
 /* Converter function from memory to human readable hex strings */
 static char*to_hex(char*d,uint8_t*s){char*m="0123456789ABCDEF";char *t=d;char*e=d+40;while(d<e){*d++=m[*s>>4];*d++=m[*s++&15];}*d=0;return t;}
 
@@ -208,7 +224,7 @@ size_t stats_top10_txt( char * reply ) {
 
 /* This function collects 4096 /24s in 4096 possible
  malloc blocks
-*/
+ */
 static size_t stats_slash24s_txt( char * reply, size_t amount, uint32_t thresh ) {
   
 #define NUM_TOPBITS 12
@@ -375,24 +391,14 @@ static size_t stats_fullscrapes_mrtg( char * reply ) {
 }
 
 static size_t stats_peers_mrtg( char * reply ) {
-  size_t    torrent_count = 0, peer_count = 0, seed_count = 0, j;
+  torrent_stats stats = {0,0,0};
-  int bucket;
   
-  for( bucket=0; bucket<OT_BUCKET_COUNT; ++bucket ) {
+  iterate_all_torrents( torrent_statter, (uintptr_t)&stats );
-    ot_vector *torrents_list = mutex_bucket_lock( bucket );
+  
-    torrent_count += torrents_list->size;
+  return sprintf( reply, "%llu\n%llu\nopentracker serving %llu torrents\nopentracker",
-    for( j=0; j<torrents_list->size; ++j ) {
+                 stats.peer_count,
-      ot_peerlist *peer_list = ( ((ot_torrent*)(torrents_list->data))[j] ).peer_list;
+                 stats.seed_count,
-      peer_count += peer_list->peer_count; seed_count += peer_list->seed_count;
+                 stats.torrent_count
-    }
-    mutex_bucket_unlock( bucket, 0 );
-    if( !g_opentracker_running )
-      return 0;
-  }
-  return sprintf( reply, "%zd\n%zd\nopentracker serving %zd torrents\nopentracker",
-    peer_count,
-    seed_count,
-    torrent_count
                  );
 }
 
@@ -459,8 +465,7 @@ static size_t stats_return_renew_bucket( char * reply ) {
   return r - reply;
 }
 
-static size_t stats_return_sync_mrtg( char * reply )
+static size_t stats_return_sync_mrtg( char * reply ) {
-{
 	ot_time t = time( NULL ) - ot_start_time;
 	return sprintf( reply,
                  "%llu\n%llu\n%i seconds (%i hours)\nopentracker connections, %lu conns/s :: %lu success/s.",
@@ -473,6 +478,17 @@ static size_t stats_return_sync_mrtg( char * reply )
                  );
 }
 
+static size_t stats_return_everything( char * reply ) {
+  char * r = reply;
+  r += sprintf( r, "<stats>\n" );
+  r += sprintf( r, "  <uptime>%llu</uptime>\n", (unsigned long long)(time( NULL ) - ot_start_time) );
+  r += sprintf( r, "  <torrents>%zd</torrents>\n", mutex_get_torrent_count() );
+  /*  r += sprintf( r, "  <peers>%llu</peers>\n",  ); */
+  
+  r += sprintf( reply, "</stats>" );
+  return r - reply;
+}
+
 extern const char
 *g_version_opentracker_c, *g_version_accesslist_c, *g_version_clean_c, *g_version_fullscrape_c, *g_version_http_c,
 *g_version_iovec_c, *g_version_mutex_c, *g_version_stats_c, *g_version_udp_c, *g_version_vector_c,
@@ -590,6 +606,9 @@ void stats_issue_event( ot_status_event event, PROTO_FLAG proto, uintptr_t event
     case EVENT_SYNC:
       ot_overall_sync_count+=event_data;
 	    break;
+    case EVENT_BUCKET_LOCKED:
+      ot_overall_stall_count++;
+      break;
     default:
       break;
   }

ot_stats.h

@@ -17,7 +17,8 @@ typedef enum {
   EVENT_FULLSCRAPE_REQUEST,
   EVENT_FULLSCRAPE_REQUEST_GZIP,
   EVENT_FULLSCRAPE,   /* TCP only */
-  EVENT_FAILED
+  EVENT_FAILED,
+  EVENT_BUCKET_LOCKED
 } ot_status_event;
 
 enum {

trackerlogic.c

@@ -337,6 +337,23 @@ size_t remove_peer_from_torrent( ot_hash hash, ot_peer *peer, char *reply, PROTO
   return reply_size;
 }
 
+void iterate_all_torrents( int (*for_each)( ot_torrent* torrent, uintptr_t data ), uintptr_t data ) {
+  int bucket;
+  size_t j;
+
+  for( bucket=0; bucket<OT_BUCKET_COUNT; ++bucket ) {
+    ot_vector  *torrents_list = mutex_bucket_lock( bucket );
+    ot_torrent *torrents = (ot_torrent*)(torrents_list->data);
+
+    for( j=0; j<torrents_list->size; ++j )
+      if( for_each( torrents + j, data ) )
+        break;
+
+    mutex_bucket_unlock( bucket, 0 );
+    if( !g_opentracker_running ) return;
+  }
+}
+
 void exerr( char * message ) {
   fprintf( stderr, "%s\n", message );
   exit( 111 );

trackerlogic.h

@@ -148,6 +148,9 @@ size_t  remove_peer_from_torrent( ot_hash hash, ot_peer *peer, char *reply, PROT
 size_t  return_tcp_scrape_for_torrent( ot_hash *hash, int amount, char *reply );
 size_t  return_udp_scrape_for_torrent( ot_hash hash, char *reply );
 
+/* torrent iterator */
+void iterate_all_torrents( int (*for_each)( ot_torrent* torrent, uintptr_t data ), uintptr_t data );
+
 /* Helper, before it moves to its own object */
 void free_peerlist( ot_peerlist *peer_list );