address linter warnings. Thanks to gagath@debian.org

This commit fixes the syntax of the implicit fallthrough comments, in
order to be matched by GCC (and probably other compilers as well) with
the following regular expression:

[ \t.!]*([Ee]lse,? |[Ii]ntentional(ly)? )? fall(s | |-)?thr(ough|u)[
\t.!]*(-[^\n\r]*)?

See: https://gcc.gnu.org/onlinedocs/gcc/Warning-Options.html#index-Wimplicit-fallthrough=

.clang-format

@@ -0,0 +1,246 @@
+---
+Language:        Cpp
+# BasedOnStyle:  LLVM
+AccessModifierOffset: -2
+AlignAfterOpenBracket: Align
+AlignArrayOfStructures: None
+AlignConsecutiveAssignments:
+  Enabled:         true
+  AcrossEmptyLines: true
+  AcrossComments:  true
+  AlignCompound:   true
+  AlignFunctionPointers: false
+  PadOperators:    true
+AlignConsecutiveBitFields:
+  Enabled:         false
+  AcrossEmptyLines: true
+  AcrossComments:  true
+  AlignCompound:   false
+  AlignFunctionPointers: false
+  PadOperators:    false
+AlignConsecutiveDeclarations:
+  Enabled:         true
+  AcrossEmptyLines: true
+  AcrossComments:  true
+  AlignCompound:   true
+  AlignFunctionPointers: false
+  PadOperators:    true
+AlignConsecutiveMacros:
+  Enabled:         true
+  AcrossEmptyLines: true
+  AcrossComments:  true
+  AlignCompound:   true
+  AlignFunctionPointers: false
+  PadOperators:    false
+AlignConsecutiveShortCaseStatements:
+  Enabled:         true
+  AcrossEmptyLines: true
+  AcrossComments:  true
+  AlignCaseColons: false
+AlignEscapedNewlines: Right
+AlignOperands:   Align
+AlignTrailingComments:
+  Kind:            Always
+  OverEmptyLines:  0
+AllowAllArgumentsOnNextLine: true
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowBreakBeforeNoexceptSpecifier: Never
+AllowShortBlocksOnASingleLine: Never
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortCompoundRequirementOnASingleLine: true
+AllowShortEnumsOnASingleLine: true
+AllowShortFunctionsOnASingleLine: All
+AllowShortIfStatementsOnASingleLine: Never
+AllowShortLambdasOnASingleLine: All
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: false
+AlwaysBreakTemplateDeclarations: MultiLine
+AttributeMacros:
+  - __capability
+BinPackArguments: true
+BinPackParameters: true
+BitFieldColonSpacing: Both
+BraceWrapping:
+  AfterCaseLabel:  false
+  AfterClass:      false
+  AfterControlStatement: Never
+  AfterEnum:       false
+  AfterExternBlock: false
+  AfterFunction:   false
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  BeforeCatch:     false
+  BeforeElse:      false
+  BeforeLambdaBody: false
+  BeforeWhile:     false
+  IndentBraces:    false
+  SplitEmptyFunction: true
+  SplitEmptyRecord: true
+  SplitEmptyNamespace: true
+BreakAdjacentStringLiterals: true
+BreakAfterAttributes: Leave
+BreakAfterJavaFieldAnnotations: false
+BreakArrays:     true
+BreakBeforeBinaryOperators: None
+BreakBeforeConceptDeclarations: Always
+BreakBeforeBraces: Attach
+BreakBeforeInlineASMColon: OnlyMultiline
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializers: BeforeColon
+BreakInheritanceList: BeforeColon
+BreakStringLiterals: false
+ColumnLimit:     160
+CommentPragmas:  '^ IWYU pragma:'
+CompactNamespaces: false
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DerivePointerAlignment: false
+DisableFormat:   false
+EmptyLineAfterAccessModifier: Never
+EmptyLineBeforeAccessModifier: LogicalBlock
+ExperimentalAutoDetectBinPacking: false
+FixNamespaceComments: true
+ForEachMacros:
+  - foreach
+  - Q_FOREACH
+  - BOOST_FOREACH
+IfMacros:
+  - KJ_IF_MAYBE
+IncludeBlocks:   Preserve
+IncludeCategories:
+  - Regex:           '^"(llvm|llvm-c|clang|clang-c)/'
+    Priority:        2
+    SortPriority:    0
+    CaseSensitive:   false
+  - Regex:           '^(<|"(gtest|gmock|isl|json)/)'
+    Priority:        3
+    SortPriority:    0
+    CaseSensitive:   false
+  - Regex:           '.*'
+    Priority:        1
+    SortPriority:    0
+    CaseSensitive:   false
+IncludeIsMainRegex: '(Test)?$'
+IncludeIsMainSourceRegex: ''
+IndentAccessModifiers: false
+IndentCaseBlocks: false
+IndentCaseLabels: false
+IndentExternBlock: AfterExternBlock
+IndentGotoLabels: true
+IndentPPDirectives: None
+IndentRequiresClause: true
+IndentWidth:     2
+IndentWrappedFunctionNames: false
+InsertBraces:    false
+InsertNewlineAtEOF: false
+InsertTrailingCommas: None
+IntegerLiteralSeparator:
+  Binary:          0
+  BinaryMinDigits: 0
+  Decimal:         0
+  DecimalMinDigits: 0
+  Hex:             0
+  HexMinDigits:    0
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: true
+KeepEmptyLinesAtEOF: false
+LambdaBodyIndentation: Signature
+LineEnding:      DeriveLF
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBinPackProtocolList: Auto
+ObjCBlockIndentWidth: 2
+ObjCBreakBeforeNestedBlockParam: true
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: true
+PackConstructorInitializers: BinPack
+PenaltyBreakAssignment: 2
+PenaltyBreakBeforeFirstCallParameter: 19
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakOpenParenthesis: 0
+PenaltyBreakScopeResolution: 500
+PenaltyBreakString: 1000
+PenaltyBreakTemplateDeclaration: 10
+PenaltyExcessCharacter: 1000000
+PenaltyIndentedWhitespace: 0
+PenaltyReturnTypeOnItsOwnLine: 60
+PointerAlignment: Right
+PPIndentWidth:   -1
+QualifierAlignment: Leave
+ReferenceAlignment: Pointer
+ReflowComments:  true
+RemoveBracesLLVM: false
+RemoveParentheses: Leave
+RemoveSemicolon: false
+RequiresClausePosition: OwnLine
+RequiresExpressionIndentation: OuterScope
+SeparateDefinitionBlocks: Leave
+ShortNamespaceLines: 1
+SkipMacroDefinitionBody: false
+SortIncludes:    CaseSensitive
+SortJavaStaticImport: Before
+SortUsingDeclarations: LexicographicNumeric
+SpaceAfterCStyleCast: false
+SpaceAfterLogicalNot: false
+SpaceAfterTemplateKeyword: true
+SpaceAroundPointerQualifiers: Default
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeCaseColon: false
+SpaceBeforeCpp11BracedList: false
+SpaceBeforeCtorInitializerColon: true
+SpaceBeforeInheritanceColon: true
+SpaceBeforeJsonColon: false
+SpaceBeforeParens: ControlStatements
+SpaceBeforeParensOptions:
+  AfterControlStatements: true
+  AfterForeachMacros: true
+  AfterFunctionDefinitionName: false
+  AfterFunctionDeclarationName: false
+  AfterIfMacros:   true
+  AfterOverloadedOperator: false
+  AfterPlacementOperator: true
+  AfterRequiresInClause: false
+  AfterRequiresInExpression: false
+  BeforeNonEmptyParentheses: false
+SpaceBeforeRangeBasedForLoopColon: true
+SpaceBeforeSquareBrackets: false
+SpaceInEmptyBlock: false
+SpacesBeforeTrailingComments: 1
+SpacesInAngles:  Never
+SpacesInContainerLiterals: true
+SpacesInLineCommentPrefix:
+  Minimum:         1
+  Maximum:         -1
+SpacesInParens:  Never
+SpacesInParensOptions:
+  InCStyleCasts:   false
+  InConditionalStatements: false
+  InEmptyParentheses: false
+  Other:           false
+SpacesInSquareBrackets: false
+Standard:        Latest
+StatementAttributeLikeMacros:
+  - Q_EMIT
+StatementMacros:
+  - Q_UNUSED
+  - QT_REQUIRE_VERSION
+TabWidth:        8
+UseTab:          Never
+VerilogBreakBetweenInstancePorts: true
+WhitespaceSensitiveMacros:
+  - BOOST_PP_STRINGIZE
+  - CF_SWIFT_NAME
+  - NS_SWIFT_NAME
+  - PP_STRINGIZE
+  - STRINGIZE
+...
+

.gitignore

@@ -0,0 +1,2 @@
+*.o
+

Makefile

@@ -1,7 +1,5 @@
 # $Id$
 
-CC?=gcc
-
 # Linux flavour
 # PREFIX?=/opt/diet
 # LIBOWFAT_HEADERS=$(PREFIX)/include
@@ -18,35 +16,65 @@ LIBOWFAT_HEADERS=$(PREFIX)/libowfat
 LIBOWFAT_LIBRARY=$(PREFIX)/libowfat
 
 BINDIR?=$(PREFIX)/bin
+STRIP?=strip
 
+#FEATURES+=-DWANT_V4_ONLY
 #FEATURES+=-DWANT_ACCESSLIST_BLACK
 #FEATURES+=-DWANT_ACCESSLIST_WHITE
+#FEATURES+=-DWANT_DYNAMIC_ACCESSLIST
 
-#FEATURES+=-DWANT_SYNC_BATCH
 #FEATURES+=-DWANT_SYNC_LIVE
-
-#FEATURES+=-DWANT_UTORRENT1600_WORKAROUND
 #FEATURES+=-DWANT_IP_FROM_QUERY_STRING
-#FEATURES+=-DWANT_COMPRESSION_GZIP 
+
+# If you want gzip support to be compiled in, uncomment the next include.
+# You can further modify the behaviour by setting DWANT_COMPRESSION_GZIP_ALWAYS
+# in Makefile.gzip
+include Makefile.gzip
+
+# If you want zstd support to be compiled in, uncomment the next include.
+# You can further modify the behaviour by setting DWANT_COMPRESSION_ZSTD_ALWAYS
+# in Makefile.zstd
+#include Makefile.zstd
+
 #FEATURES+=-DWANT_LOG_NETWORKS
 #FEATURES+=-DWANT_RESTRICT_STATS
-#FEATURES+=-D_DEBUG_HTTPERROR
-#FEATURES+=-D_DEBUG_VECTOR
-
+#FEATURES+=-DWANT_IP_FROM_PROXY
+#FEATURES+=-DWANT_FULLLOG_NETWORKS
+#FEATURES+=-DWANT_LOG_NUMWANT
+#FEATURES+=-DWANT_MODEST_FULLSCRAPES
+#FEATURES+=-DWANT_SPOT_WOODPECKER
+#FEATURES+=-DWANT_SYSLOGS
+#FEATURES+=-DWANT_DEV_RANDOM
 FEATURES+=-DWANT_FULLSCRAPE
 
-OPTS_debug=-D_DEBUG -g -ggdb #-pg # -fprofile-arcs -ftest-coverage
-OPTS_production=-Os
+# You need libowfat version 0.34 to allow for automatic release of chunks during
+# full scrape transfer, if you rely on an older versions, enable this flag
+#FEATURES+=-DWANT_NO_AUTO_FREE
 
-CFLAGS+=-I$(LIBOWFAT_HEADERS) -Wall -pipe -Wextra #-pedantic -ansi
-LDFLAGS+=-L$(LIBOWFAT_LIBRARY) -lowfat -pthread -lz
+# Is enabled on BSD systems by default in trackerlogic.h
+# on Linux systems the include Makefile adds -lbsd
+#include Makefile.arc4random
+
+#FEATURES+=-D_DEBUG_HTTPERROR
+#FEATURES+=-D_DEBUG_RANDOMTORRENTS
+
+GIT_VERSION=$(shell sh -c 'command -v git >/dev/null && test -d .git && git rev-parse HEAD || echo _git_or_commit_not_found_')
+
+OPTS_debug=-D_DEBUG -g -ggdb # -pg -fprofile-arcs -ftest-coverage
+OPTS_production=-O3
+
+CFLAGS+=-I$(LIBOWFAT_HEADERS) -DGIT_VERSION=$(GIT_VERSION) -Wall -pipe -pthread -Wextra #-ansi -pedantic
+LDFLAGS+=-L$(LIBOWFAT_LIBRARY) -lowfat -pthread
 
 BINARY =opentracker
-HEADERS=trackerlogic.h scan_urlencoded_query.h ot_mutex.h ot_stats.h ot_sync.h ot_vector.h ot_clean.h ot_udp.h ot_iovec.h ot_fullscrape.h ot_accesslist.h ot_http.h ot_livesync.h
-SOURCES=opentracker.c trackerlogic.c scan_urlencoded_query.c ot_mutex.c ot_stats.c ot_sync.c ot_vector.c ot_clean.c ot_udp.c ot_iovec.c ot_fullscrape.c ot_accesslist.c ot_http.c ot_livesync.c
+HEADERS=trackerlogic.h scan_urlencoded_query.h ot_mutex.h ot_stats.h ot_vector.h ot_clean.h ot_udp.h ot_iovec.h ot_fullscrape.h ot_accesslist.h ot_http.h ot_livesync.h ot_rijndael.h
+SOURCES=opentracker.c trackerlogic.c scan_urlencoded_query.c ot_mutex.c ot_stats.c ot_vector.c ot_clean.c ot_udp.c ot_iovec.c ot_fullscrape.c ot_accesslist.c ot_http.c ot_livesync.c ot_rijndael.c
+SOURCES_proxy=proxy.c ot_vector.c ot_mutex.c
 
 OBJECTS = $(SOURCES:%.c=%.o)
 OBJECTS_debug = $(SOURCES:%.c=%.debug.o)
+OBJECTS_proxy = $(SOURCES_proxy:%.c=%.o)
+OBJECTS_proxy_debug = $(SOURCES_proxy:%.c=%.debug.o)
 
 .SUFFIXES: .debug.o .o .c
 
@@ -57,9 +85,13 @@ CFLAGS_debug = $(CFLAGS) $(OPTS_debug) $(FEATURES)
 
 $(BINARY): $(OBJECTS) $(HEADERS)
 	$(CC) -o $@ $(OBJECTS) $(LDFLAGS)
-	strip $@
+	$(STRIP) $@
 $(BINARY).debug: $(OBJECTS_debug) $(HEADERS)
 	$(CC) -o $@ $(OBJECTS_debug) $(LDFLAGS)
+proxy: $(OBJECTS_proxy) $(HEADERS)
+	$(CC) -o $@ $(OBJECTS_proxy) $(CFLAGS_production) $(LDFLAGS)
+proxy.debug: $(OBJECTS_proxy_debug) $(HEADERS)
+	$(CC) -o $@ $(OBJECTS_proxy_debug) $(LDFLAGS)
 
 .c.debug.o : $(HEADERS)
 	$(CC) -c -o $@ $(CFLAGS_debug) $(<:.debug.o=.c)
@@ -71,4 +103,4 @@ clean:
 	rm -rf opentracker opentracker.debug *.o *~
 
 install:
-	install -m 755 opentracker $(BINDIR)
+	install -m 755 opentracker $(DESTDIR)$(BINDIR)

Makefile.arc4random

@@ -0,0 +1,3 @@
+FEATURES+=-DWANT_ARC4RANDOM
+LDFLAGS+=-lbsd
+

Makefile.gzip

@@ -0,0 +1,4 @@
+FEATURES+=-DWANT_COMPRESSION_GZIP
+#FEATURES+=-DWANT_COMPRESSION_GZIP_ALWAYS
+
+LDFLAGS+=-lz

Makefile.zstd

@@ -0,0 +1,3 @@
+FEATURES+=-DWANT_COMPRESSION_ZSTD
+#FEATURES+=-DWANT_COMPRESSION_ZSTD_ALWAYS
+LDFLAGS+=-lzstd

README

@@ -27,4 +27,4 @@ sysctl kern.maxfiles=10240
 
 License information:
 
-Although the libowfat library is under GPL, Felix von Leitner aggreed that the compiled binary may be distributed under the same beer ware license as the source code for opentracker. However, we like to hear from happy customers.
+Although the libowfat library is under GPL, Felix von Leitner agreed that the compiled binary may be distributed under the same beer ware license as the source code for opentracker. However, we like to hear from happy customers.

README_v6

@@ -1,8 +1 @@
-Q: Why is there no v6-support in opentracker?
-
-A: Although I tried very hard, implementing v6 right now would be a terrible waste of bandwidth, there is no compact format for v6 addresses, so instead of
-answering "d5:peers6:AAAAPPe" I'd have to send "d5:peersld2:ip39:AAAA:AAAA:AAAA:AAAA:AAAA:AAAA:AAAA:AAAA4:port2:PPPPeee" for a single peer. Even if there was a
-compact mode, v6 addresses still would eat up thrice the memory, v4 addresses take. This, however, wouldn't be a show stopper.
-
-Other problems concern efficient peer selection for obviously v6-capable peers and how to select peers for non-v6 clients. v6 addresses eat up more memory on the
-host, too ;)
+IPv6 is implemented in opentracker now. You can chose whether your tracker runs in v4 or v6 mode in Makefile. YMMV.

man1/opentracker.1

@@ -0,0 +1,130 @@
+.Dd 15/4/2024
+.Dt opentracker 1
+.Os Unix
+.Sh opentracker
+.Nm opentracker
+.Nd a free and open bittorrent tracker
+.Sh SYNOPSIS
+.Nm
+.Op Fl f Ar config
+.Op Fl i Ar ip-select
+.Op Fl p Ar port-bind-tcp
+.Op Fl P Ar port-bind-udp
+.Op Fl A Ar blessed-ip
+.Op Fl r Ar redirect-url
+.Op Fl d Ar chdir
+.Op Fl u Ar user
+.Op Fl w| Fl b accesslist
+.Sh DESCRIPTION
+.Nm
+is a bittorrent tracker that implements announce and scrape actions over the
+UDP and the plain http protocol, aiming for minimal resource usage.
+.Pp
+When invoked with parameters, it binds to TCP and UDP port 6969 on all
+interfaces. The recommended way to configure opentracker is by providing a
+config file using the
+.Op Fl f Ar config
+option. See
+.Xr opentracker.conf 4
+for details.
+.Pp
+.Sh OPTIONS
+The following options are available:
+.Bl -tag -width -indent=8
+.It Fl f Ar config
+Parse a config file with a list of options. Consecutive command options
+will override options from the config file. See
+.Xr opentracker.conf 4
+for details.
+.It Fl i Ar ip-select
+Select an ip address that will be used with the next
+.Op Fl p
+or
+.Op Fl P
+command to actually bind to this address. Setting this option without any bind
+options in the config file or
+.Op Fl p
+or
+.Op Fl P
+commands will limit opentracker to only bind to this address.
+.It Fl p Ar port-bind-tcp
+Bind to the TCP port on the last preceding ip address set with the
+.Op Fl i ip-select
+option or to all available addresses if none has been set. Can be given multiple
+times.
+.It Fl P Ar port-bind-udp
+Bind to the UDP port on the last preceding ip address set with the
+.Op Fl i ip-select
+option or to all available addresses if none has been set. Can be given multiple
+times.
+.It Fl A Ar blessed-ip
+Set an ip address in IPv4 or IPv6 or a net in CIDR notation to bless the network
+for access to restricted resources.
+.It Fl r Ar redirect-url
+Set the URL that
+.Nm
+will redirect users to when the / address is requested via HTTP.
+.It Fl d Ar chdir
+Sets the directory
+.Nm
+will
+.Xr chroot 2
+to if ran as root or
+.Xr chdir 2
+to if ran as unprivileged user. Note that any accesslist files need to be
+relative to and within that directory.
+.It Fl u Ar user
+User to run
+.Nm
+under after all operations that need privileges have finished.
+.It Fl w Ar accesslist | Fl b Ar accesslist
+If
+.Nm
+has been compiled with the
+.B WANT_ACCESSLIST_BLACK
+or
+.Br WANT_ACCESSLIST_WHITE
+options, this option sets the location of the accesslist.
+.El
+.Sh EXAMPLES
+Start
+.Nm
+bound on UDP and TCP ports 6969 on IPv6 localhost.
+.Dl # ./opentracker -i ::1 -p 6969 -P 6969
+.Pp
+Start
+.Nm
+bound on UDP port 6868 and TCP port 6868 on IPv4 localhost and allow
+privileged access from the network 192.168/16 while redirecting
+HTTP clients accessing the root directory, which is not covered by the
+bittorrent tracker protocol, to https://my-trackersite.com/.
+.Dl # ./opentracker -i 192.168.0.4 -p 6868 -P 6969 -A 192.168/16 -r https://my-trackersite.com/
+The announce URLs are http://192.168.0.4:6868/announce and
+udp://192.168.0.4:6868/announce respectively.
+.Sh FILES
+.Bl -tag -width indent
+.It Pa opentracker.conf
+The
+.Nm
+config file.
+.El
+.Sh SEE ALSO
+.Xr opentracker.conf 4
+.Pp
+opentracker documentation
+.Lk https://erdgeist.org/arts/software/opentracker
+.Pp
+Bittorrent tracker protocol
+.Lk http://www.bittorrent.org/beps/bep_0015.html
+.Sh AUTHOR
+.An Dirk Engling
+.Aq Mt erdgeist@erdgeist.org .
+.Sh LICENSE
+This software is released under the Beerware License:
+.Pp
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software
+and associated documentation files (the "Software"), to deal in the Software with the following
+terms and conditions:
+.Pp
+If you meet the author(s) someday, and you think this software is worth it, you can buy them
+a beer in return.

man4/opentracker.conf.4

@@ -0,0 +1,86 @@
+.Dd 2024-04-18
+.Dt opentracker.conf 5
+.Os Unix
+.Sh NAME
+.Nm opentracker.conf
+.Nd configuration file for opentracker
+.Sh SYNOPSIS
+.Nm
+.Sh DESCRIPTION
+The
+.Nm
+configuration file specifies various options for configuring the behavior of the opentracker program.
+.Pp
+Lines starting with '#' are comments and are ignored. Options are specified as 'keyword value' pairs.
+.Pp
+The following options are available:
+.Pp
+.Bl -tag -width ".It access.proxy" -compact
+.It listen.tcp_udp Ar address
+Specifies an address opentracker will listen on for both TCP and UDP connections. If none are specified, opentracker listens on 0.0.0.0:6969 by default. Can be added more than once.
+.Pp
+.It listen.tcp Ar address
+Specifies the address opentracker will listen on for TCP connections. Can be added more than once.
+.Pp
+.It listen.udp Ar address
+Specifies the address opentracker will listen on for UDP connections. Can be added more than once.
+.Pp
+.It listen.udp.workers Ar threads
+Specifies how many threads will be spawned to handle UDP connections. Defaults to 4.
+.Pp
+.It access.whitelist Ar path/to/whitelist
+Specifies the path to the whitelist file containing all torrent hashes that opentracker will serve. Use this option if opentracker runs in a non-open mode.
+.Pp
+.It access.blacklist Ar path/to/blacklist
+Specifies the path to the blacklist file containing all torrent hashes that opentracker will not serve. Use this option if opentracker was compiled to allow blacklisting.
+.Pp
+.It access.fifo_add Ar path/to/adder.fifo
+Specifies the path to the FIFO (named pipe) used for dynamic changesets to accesslists. Info hashes written to this FIFO will be added to the main accesslist file.
+.Pp
+.It access.fifo_delete Ar path/to/deleter.fifo
+Specifies the path to the FIFO (named pipe) used for dynamic changesets to accesslists. Info hashes written to this FIFO will be removed from the main accesslist file.
+.Pp
+.It access.stats Ar ip_address_or_network
+Specifies the IP address or network in CIDR notation allowed to fetch stats from opentracker.
+.Pp
+.It access.stats_path Ar path
+Specifies the path to the stats location. You can configure opentracker to appear anywhere on your tracker. Defaults to /stats.
+.Pp
+.It access.proxy Ar ip_address_or_network
+Specifies the IP address or network of the reverse proxies. Opentracker will take the X-Forwarded-For address instead of the source IP address. Can be added more than once.
+.Pp
+.It livesync.cluster.listen Ar ip_address:port
+Specifies the IP address and port opentracker will listen on for incoming live sync packets to keep a cluster of opentrackers synchronized.
+.Pp
+.It livesync.cluster.node_ip Ar ip_address
+Specifies one trusted IP address for sync between trackers running in a cluster. Can be added more than once.
+.Pp
+.It batchsync.cluster.admin_ip Ar ip_address
+Specifies the admin IP address for old-style (HTTP-based) asynchronous tracker syncing.
+.Pp
+.It tracker.rootdir Ar path
+Specifies the directory opentracker will chroot/chdir to. All black/white list files must be located in this directory.
+.Pp
+.It tracker.user Ar username
+Specifies the user opentracker will setuid to after binding to potentially privileged ports.
+.Pp
+.It tracker.redirect_url Ar URL
+Specifies the URL opentracker will redirect to in response to a "GET / HTTP" request.
+.El
+.Sh EXAMPLES
+To specify the address opentracker will listen on for both TCP and UDP connections:
+.Dl listen.tcp_udp 0.0.0.0:6969
+.Pp
+To specify the address opentracker will listen on for TCP connections:
+.Dl listen.tcp 0.0.0.0
+.Pp
+To specify the address opentracker will listen on for UDP connections:
+.Dl listen.udp 0.0.0.0:6969
+.Pp
+.Sh SEE ALSO
+.Xr opentracker 1
+.Pp
+.Sh AUTHOR
+.An Dirk Engling
+.Aq Mt erdgeist@erdgeist.org
+.Pp

opentracker.conf.sample

@@ -2,11 +2,20 @@
 #
 
 # I)   Address opentracker will listen on, using both, tcp AND udp family
-#      (note, that port 6969 is implicite if ommitted).
+#      (note, that port 6969 is implicit if omitted).
 #
 #      If no listen option is given (here or on the command line), opentracker
 #      listens on 0.0.0.0:6969 tcp and udp.
 #
+#      The next variable determines if udp sockets are handled in the event
+#      loop (set it to 0, the default) or are handled in blocking reads in
+#      dedicated worker threads. You have to set this value before the
+#      listen.tcp_udp or listen.udp statements before it takes effect, but you
+#      can re-set it for each listen statement. Normally you should keep it at
+#      the top of the config file.
+#
+# listen.udp.workers 4
+#
 # listen.tcp_udp 0.0.0.0
 # listen.tcp_udp 192.168.0.1:80
 # listen.tcp_udp 10.0.0.5:6969
@@ -35,11 +44,64 @@
 #      listing, so choose one of those options at compile time. File format
 #      is straight forward: "<hex info hash>\n<hex info hash>\n..."
 #
+# IIa) You can enable dynamic changesets to accesslists by enabling
+#      WANT_DYNAMIC_ACCESSLIST.
+#
+#      The suggested way to work with dynamic changeset lists is to keep a
+#      main accesslist file that is loaded when opentracker (re)starts and
+#      reloaded infrequently (hourly or daily).
+#
+#      All changes to the accesslist (e.g. from a web frontend) should be
+#      both appended to or removed from that file and sent to opentracker. By
+#      keeping dynamic changeset lists, you can avoid reloading huge
+#      accesslists whenever just a single entry is added or removed.
+#
+#      Any info_hash (format see above) written to the fifo_add file will be
+#      kept on a dynamic add-changeset, removed from the dynamic
+#      delete-changeset and treated as if it was in the main accesslist file.
+#      The semantic of the respective dynamic changeset depends on whether
+#      WANT_ACCESSLIST_WHITE or WANT_ACCESSLIST_BLACK is enabled.
+#
+# access.fifo_add /var/run/opentracker/adder.fifo
+#
+#      Any info_hash (format see above) written to the fifo_delete file will
+#      be kept on a dynamic delete-changeset, removed from the dynamic
+#      add-changeset and treated as if it was not in the main accesslist
+#      file.
+#
+# access.fifo_delete /var/run/opentracker/deleter.fifo
+#
+#      If you reload the accesslist by sending SIGHUP to the tracker process,
+#      the dynamic lists are flushed, as opentracker assumes thoses lists are
+#      merged into the main accesslist.
+#
+#      NOTE: While you can have multiple writers sending lines to the fifos,
+#      any writes larger than PIPE_BUF (see your limits.h, minimally 512
+#      bytes but usually 4096) may be interleaved with data sent by other
+#      writers. This can lead to unparsable lines of info_hashes.
+#
+# IIb)
 #      If you do not want to grant anyone access to your stats, enable the
 #      WANT_RESTRICT_STATS option in Makefile and bless the ip addresses
-#      allowed to fetch stats here.
+#      or network allowed to fetch stats here.
 #
 # access.stats 192.168.0.23
+# access.stats 10.1.1.23
+#
+#      There is another way of hiding your stats. You can obfuscate the path
+#      to them. Normally it is located at /stats but you can configure it to
+#      appear anywhere on your tracker.
+#
+# access.stats_path stats
+#
+# II
+#      If opentracker lives behind one or multiple reverse proxies,
+#      every http connection appears to come from these proxies. In order to
+#      take the X-Forwarded-For address instead, compile opentracker with the
+#      WANT_IP_FROM_PROXY option and set your proxy addresses or networkss here.
+#
+# access.proxy 10.0.1.23
+# access.proxy 192.0.0.0/8
 #
 
 # III) Live sync uses udp multicast packets to keep a cluster of opentrackers
@@ -74,11 +136,17 @@
 # batchsync.cluster.admin_ip 10.1.1.1
 #
 
-# V)   Control directory where opentracker will chdir to. So all black/white
-#      list files may be put in that directory (shell option -d).
+# V)   Control privilege drop behaviour.
+#      Put in the directory opentracker will chroot/chdir to. All black/white
+#      list files must be put in that directory (shell option -d).
+#
 #
 # tracker.rootdir /usr/local/etc/opentracker
 #
+#      Tell opentracker which user to setuid to.
+#
+# tracker.user    nobody
+#
 
 # VI)  opentracker can be told to answer to a "GET / HTTP"-request with a
 #      redirect to another location (shell option -r).

opentracker.xcodeproj/project.pbxproj

@@ -14,7 +14,6 @@
 		654A808A0CD832FD009035DE /* scan_urlencoded_query.c in Sources */ = {isa = PBXBuildFile; fileRef = 654A80850CD832FC009035DE /* scan_urlencoded_query.c */; };
 		654A808B0CD832FD009035DE /* trackerlogic.c in Sources */ = {isa = PBXBuildFile; fileRef = 654A80870CD832FC009035DE /* trackerlogic.c */; };
 		65542D8B0CE078E800469330 /* ot_vector.c in Sources */ = {isa = PBXBuildFile; fileRef = 65542D8A0CE078E800469330 /* ot_vector.c */; };
-		65542D8E0CE07BA900469330 /* ot_sync.c in Sources */ = {isa = PBXBuildFile; fileRef = 65542D8D0CE07BA900469330 /* ot_sync.c */; };
 		65542D930CE07CED00469330 /* ot_mutex.c in Sources */ = {isa = PBXBuildFile; fileRef = 65542D8F0CE07CED00469330 /* ot_mutex.c */; };
 		65542D940CE07CED00469330 /* ot_stats.c in Sources */ = {isa = PBXBuildFile; fileRef = 65542D910CE07CED00469330 /* ot_stats.c */; };
 		65542E750CE08B9100469330 /* ot_clean.c in Sources */ = {isa = PBXBuildFile; fileRef = 65542E740CE08B9100469330 /* ot_clean.c */; };
@@ -40,7 +39,6 @@
 
 /* Begin PBXFileReference section */
 		6520B7520D036AAF00A43B1F /* libowfat.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = libowfat.a; path = ../libowfat/libowfat.a; sourceTree = SOURCE_ROOT; };
-		6520B8110D0E011000A43B1F /* liblibowfat.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = liblibowfat.a; sourceTree = BUILT_PRODUCTS_DIR; };
 		653A320A0CE7F475007F0D03 /* ot_accesslist.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ot_accesslist.h; sourceTree = "<group>"; };
 		653A320B0CE7F475007F0D03 /* ot_accesslist.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = ot_accesslist.c; sourceTree = "<group>"; };
 		653A56AC0CE201FF000CF140 /* opentracker */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = opentracker; sourceTree = BUILT_PRODUCTS_DIR; };
@@ -53,8 +51,6 @@
 		654A80880CD832FC009035DE /* trackerlogic.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = trackerlogic.h; sourceTree = "<group>"; };
 		65542D890CE078E800469330 /* ot_vector.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ot_vector.h; sourceTree = "<group>"; };
 		65542D8A0CE078E800469330 /* ot_vector.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = ot_vector.c; sourceTree = "<group>"; };
-		65542D8C0CE07BA900469330 /* ot_sync.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ot_sync.h; sourceTree = "<group>"; };
-		65542D8D0CE07BA900469330 /* ot_sync.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = ot_sync.c; sourceTree = "<group>"; };
 		65542D8F0CE07CED00469330 /* ot_mutex.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = ot_mutex.c; sourceTree = "<group>"; };
 		65542D900CE07CED00469330 /* ot_mutex.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ot_mutex.h; sourceTree = "<group>"; };
 		65542D910CE07CED00469330 /* ot_stats.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = ot_stats.c; sourceTree = "<group>"; };
@@ -73,13 +69,6 @@
 /* End PBXFileReference section */
 
 /* Begin PBXFrameworksBuildPhase section */
-		654A80A60CD83615009035DE /* Frameworks */ = {
-			isa = PBXFrameworksBuildPhase;
-			buildActionMask = 2147483647;
-			files = (
-			);
-			runOnlyForDeploymentPostprocessing = 0;
-		};
 		8DD76FAD0486AB0100D96B5E /* Frameworks */ = {
 			isa = PBXFrameworksBuildPhase;
 			buildActionMask = 2147483647;
@@ -96,6 +85,7 @@
 			children = (
 				65542D810CE0786F00469330 /* Headers */,
 				08FB7795FE84155DC02AAC07 /* Source */,
+				92762AC9104EDED700FDCB60 /* Libraries */,
 				C6A0FF2B0290797F04C91782 /* Documentation */,
 				653A56AD0CE201FF000CF140 /* Products */,
 			);
@@ -114,7 +104,6 @@
 				653A56B40CE28EC5000CF140 /* ot_iovec.c */,
 				65542D8F0CE07CED00469330 /* ot_mutex.c */,
 				65542D910CE07CED00469330 /* ot_stats.c */,
-				65542D8D0CE07BA900469330 /* ot_sync.c */,
 				65542EE70CE0CA6B00469330 /* ot_udp.c */,
 				65542D8A0CE078E800469330 /* ot_vector.c */,
 				654A80850CD832FC009035DE /* scan_urlencoded_query.c */,
@@ -127,8 +116,6 @@
 			isa = PBXGroup;
 			children = (
 				653A56AC0CE201FF000CF140 /* opentracker */,
-				6520B7520D036AAF00A43B1F /* libowfat.a */,
-				6520B8110D0E011000A43B1F /* liblibowfat.a */,
 			);
 			name = Products;
 			sourceTree = "<group>";
@@ -136,15 +123,14 @@
 		65542D810CE0786F00469330 /* Headers */ = {
 			isa = PBXGroup;
 			children = (
-				65FA33970E7EF09200F7D5A5 /* ot_livesync.h */,
 				653A320A0CE7F475007F0D03 /* ot_accesslist.h */,
 				65542E730CE08B9100469330 /* ot_clean.h */,
 				65542F900CE17CA900469330 /* ot_fullscrape.h */,
 				65B8DF3A0D0310D20017149E /* ot_http.h */,
 				653A56B30CE28EC5000CF140 /* ot_iovec.h */,
+				65FA33970E7EF09200F7D5A5 /* ot_livesync.h */,
 				65542D900CE07CED00469330 /* ot_mutex.h */,
 				65542D920CE07CED00469330 /* ot_stats.h */,
-				65542D8C0CE07BA900469330 /* ot_sync.h */,
 				65542EE60CE0CA6B00469330 /* ot_udp.h */,
 				65542D890CE078E800469330 /* ot_vector.h */,
 				654A80860CD832FC009035DE /* scan_urlencoded_query.h */,
@@ -153,6 +139,14 @@
 			name = Headers;
 			sourceTree = "<group>";
 		};
+		92762AC9104EDED700FDCB60 /* Libraries */ = {
+			isa = PBXGroup;
+			children = (
+				6520B7520D036AAF00A43B1F /* libowfat.a */,
+			);
+			name = Libraries;
+			sourceTree = "<group>";
+		};
 		C6A0FF2B0290797F04C91782 /* Documentation */ = {
 			isa = PBXGroup;
 			children = (
@@ -163,34 +157,7 @@
 		};
 /* End PBXGroup section */
 
-/* Begin PBXHeadersBuildPhase section */
-		654A80A40CD83615009035DE /* Headers */ = {
-			isa = PBXHeadersBuildPhase;
-			buildActionMask = 2147483647;
-			files = (
-			);
-			runOnlyForDeploymentPostprocessing = 0;
-		};
-/* End PBXHeadersBuildPhase section */
-
 /* Begin PBXNativeTarget section */
-		654A80A70CD83615009035DE /* libowfat */ = {
-			isa = PBXNativeTarget;
-			buildConfigurationList = 654A80AB0CD83635009035DE /* Build configuration list for PBXNativeTarget "libowfat" */;
-			buildPhases = (
-				654A80A40CD83615009035DE /* Headers */,
-				654A80A50CD83615009035DE /* Sources */,
-				654A80A60CD83615009035DE /* Frameworks */,
-			);
-			buildRules = (
-			);
-			dependencies = (
-			);
-			name = libowfat;
-			productName = libowfat;
-			productReference = 6520B8110D0E011000A43B1F /* liblibowfat.a */;
-			productType = "com.apple.product-type.library.static";
-		};
 		8DD76FA90486AB0100D96B5E /* opentracker */ = {
 			isa = PBXNativeTarget;
 			buildConfigurationList = 1DEB928508733DD80010E9CD /* Build configuration list for PBXNativeTarget "opentracker" */;
@@ -223,19 +190,11 @@
 			projectRoot = "";
 			targets = (
 				8DD76FA90486AB0100D96B5E /* opentracker */,
-				654A80A70CD83615009035DE /* libowfat */,
 			);
 		};
 /* End PBXProject section */
 
 /* Begin PBXSourcesBuildPhase section */
-		654A80A50CD83615009035DE /* Sources */ = {
-			isa = PBXSourcesBuildPhase;
-			buildActionMask = 2147483647;
-			files = (
-			);
-			runOnlyForDeploymentPostprocessing = 0;
-		};
 		8DD76FAB0486AB0100D96B5E /* Sources */ = {
 			isa = PBXSourcesBuildPhase;
 			buildActionMask = 2147483647;
@@ -244,7 +203,6 @@
 				654A808A0CD832FD009035DE /* scan_urlencoded_query.c in Sources */,
 				654A808B0CD832FD009035DE /* trackerlogic.c in Sources */,
 				65542D8B0CE078E800469330 /* ot_vector.c in Sources */,
-				65542D8E0CE07BA900469330 /* ot_sync.c in Sources */,
 				65542D930CE07CED00469330 /* ot_mutex.c in Sources */,
 				65542D940CE07CED00469330 /* ot_stats.c in Sources */,
 				65542E750CE08B9100469330 /* ot_clean.c in Sources */,
@@ -282,6 +240,7 @@
 			isa = XCBuildConfiguration;
 			buildSettings = {
 				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
+				GCC_ENABLE_FIX_AND_CONTINUE = YES;
 				GCC_MODEL_TUNING = G5;
 				INSTALL_PATH = /usr/local/bin;
 				LIBRARY_SEARCH_PATHS = (
@@ -295,57 +254,41 @@
 		1DEB928A08733DD80010E9CD /* Debug */ = {
 			isa = XCBuildConfiguration;
 			buildSettings = {
+				ARCHS = "$(ONLY_ACTIVE_ARCH_PRE_XCODE_3_1)";
+				GCC_PREPROCESSOR_DEFINITIONS = (
+					WANT_ACCESSLIST_WHITE,
+					WANT_IP_FROM_QUERY_STRING,
+					WANT_FULLSCRAPE,
+				);
+				GCC_VERSION = com.apple.compilers.llvm.clang.1_0;
 				GCC_WARN_ABOUT_RETURN_TYPE = YES;
 				GCC_WARN_UNUSED_VARIABLE = YES;
 				HEADER_SEARCH_PATHS = ../libowfat/;
 				LIBRARY_SEARCH_PATHS = ../libowfat/;
+				ONLY_ACTIVE_ARCH_PRE_XCODE_3_1 = "$(NATIVE_ARCH_ACTUAL)";
 				PREBINDING = NO;
-				SDKROOT = "$(DEVELOPER_SDK_DIR)/MacOSX10.5.sdk";
+				SDKROOT = "$(DEVELOPER_SDK_DIR)/MacOSX10.6.sdk";
 			};
 			name = Debug;
 		};
 		1DEB928B08733DD80010E9CD /* Release */ = {
 			isa = XCBuildConfiguration;
 			buildSettings = {
-				ARCHS = ppc;
+				ARCHS = "$(ONLY_ACTIVE_ARCH_PRE_XCODE_3_1)";
 				DEAD_CODE_STRIPPING = NO;
+				GCC_PREPROCESSOR_DEFINITIONS = (
+					WANT_ACCESSLIST_WHITE,
+					WANT_IP_FROM_QUERY_STRING,
+					WANT_FULLSCRAPE,
+				);
+				GCC_VERSION = com.apple.compilers.llvm.clang.1_0;
 				GCC_WARN_ABOUT_RETURN_TYPE = YES;
 				GCC_WARN_UNUSED_VARIABLE = YES;
 				HEADER_SEARCH_PATHS = ../libowfat/;
 				LIBRARY_SEARCH_PATHS = ../libowfat/;
-				MACH_O_TYPE = mh_execute;
-				OTHER_LDFLAGS = "-lowfat";
+				ONLY_ACTIVE_ARCH_PRE_XCODE_3_1 = "$(NATIVE_ARCH_ACTUAL)";
 				PREBINDING = NO;
-				SDKROOT = "$(DEVELOPER_SDK_DIR)/MacOSX10.5.sdk";
-				ZERO_LINK = NO;
-			};
-			name = Release;
-		};
-		654A80A90CD83617009035DE /* Debug */ = {
-			isa = XCBuildConfiguration;
-			buildSettings = {
-				COPY_PHASE_STRIP = NO;
-				GCC_DYNAMIC_NO_PIC = NO;
-				GCC_ENABLE_FIX_AND_CONTINUE = YES;
-				GCC_MODEL_TUNING = G5;
-				GCC_OPTIMIZATION_LEVEL = 0;
-				INSTALL_PATH = /usr/local/lib;
-				PREBINDING = NO;
-				PRODUCT_NAME = libowfat;
-				ZERO_LINK = YES;
-			};
-			name = Debug;
-		};
-		654A80AA0CD83617009035DE /* Release */ = {
-			isa = XCBuildConfiguration;
-			buildSettings = {
-				COPY_PHASE_STRIP = YES;
-				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
-				GCC_ENABLE_FIX_AND_CONTINUE = NO;
-				GCC_MODEL_TUNING = G5;
-				INSTALL_PATH = /usr/local/lib;
-				PREBINDING = NO;
-				PRODUCT_NAME = libowfat;
+				SDKROOT = "$(DEVELOPER_SDK_DIR)/MacOSX10.6.sdk";
 				ZERO_LINK = NO;
 			};
 			name = Release;
@@ -371,15 +314,6 @@
 			defaultConfigurationIsVisible = 0;
 			defaultConfigurationName = Release;
 		};
-		654A80AB0CD83635009035DE /* Build configuration list for PBXNativeTarget "libowfat" */ = {
-			isa = XCConfigurationList;
-			buildConfigurations = (
-				654A80A90CD83617009035DE /* Debug */,
-				654A80AA0CD83617009035DE /* Release */,
-			);
-			defaultConfigurationIsVisible = 0;
-			defaultConfigurationName = Release;
-		};
 /* End XCConfigurationList section */
 	};
 	rootObject = 08FB7793FE84155DC02AAC07 /* Project object */;

ot_accesslist.c

@@ -4,126 +4,558 @@
    $id$ */
 
 /* System */
+#include <pthread.h>
+#include <signal.h>
+#include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <stdio.h>
-#include <signal.h>
+#include <unistd.h>
+#ifdef WANT_DYNAMIC_ACCESSLIST
+#include <errno.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#endif
 
 /* Libowfat */
 #include "byte.h"
+#include "fmt.h"
+#include "ip6.h"
+#include "mmap.h"
 #include "scan.h"
 
 /* Opentracker */
-#include "trackerlogic.h"
 #include "ot_accesslist.h"
+#include "ot_vector.h"
+#include "trackerlogic.h"
 
 /* GLOBAL VARIABLES */
 #ifdef WANT_ACCESSLIST
 char *g_accesslist_filename = NULL;
-static ot_vector accesslist;
+#ifdef WANT_DYNAMIC_ACCESSLIST
+char *g_accesslist_pipe_add    = NULL;
+char *g_accesslist_pipe_delete = NULL;
+#endif
+static pthread_mutex_t g_accesslist_mutex;
 
-static void accesslist_reset( void ) {
-  free( accesslist.data );
-  byte_zero( &accesslist, sizeof( accesslist ) );
+/* Accesslists are lock free linked lists. We can not make them locking, because every announce
+   would try to acquire the mutex, making it the most contested mutex in the whole of opentracker,
+   basically creating a central performance choke point.
+
+   The idea is that updating the list heads happens under the g_accesslist_mutex guard and is
+   done atomically, while consumers might potentially still hold pointers deeper inside the list.
+
+   Consumers (for now only via accesslist_hashisvalid) will always fetch the list head pointer
+   that is guaranteed to live for at least five minutes. This should be many orders of magnitudes
+   more than how long it will be needed by the bsearch done on the list. */
+struct ot_accesslist;
+typedef struct ot_accesslist ot_accesslist;
+struct ot_accesslist {
+  ot_hash       *list;
+  size_t         size;
+  ot_time        base;
+  ot_accesslist *next;
+};
+static ot_accesslist *_Atomic g_accesslist = NULL;
+#ifdef WANT_DYNAMIC_ACCESSLIST
+static ot_accesslist *_Atomic g_accesslist_add    = NULL;
+static ot_accesslist *_Atomic g_accesslist_delete = NULL;
+#endif
+
+/* Helpers to work on access lists */
+static int vector_compare_hash(const void *hash1, const void *hash2) { return memcmp(hash1, hash2, OT_HASH_COMPARE_SIZE); }
+
+static ot_accesslist *accesslist_free(ot_accesslist *accesslist) {
+  while (accesslist) {
+    ot_accesslist *this_accesslist = accesslist;
+    accesslist = this_accesslist->next;
+    free(this_accesslist->list);
+    free(this_accesslist);
+  }
+  return NULL;
 }
 
-static int accesslist_addentry( ot_hash *infohash ) {
-  int eger;
-  void *insert = vector_find_or_insert( &accesslist, infohash, OT_HASH_COMPARE_SIZE, OT_HASH_COMPARE_SIZE, &eger );
+static ot_accesslist *accesslist_make(ot_accesslist *next, size_t size) {
+  ot_accesslist *accesslist_new = malloc(sizeof(ot_accesslist));
+  if (accesslist_new) {
+    accesslist_new->list = size ? malloc(sizeof(ot_hash) * size) : NULL;
+    accesslist_new->size = size;
+    accesslist_new->base = g_now_minutes;
+    accesslist_new->next = next;
+    if (size && !accesslist_new->list) {
+      free(accesslist_new);
+      accesslist_new = NULL;
+    }
+  }
+  return accesslist_new;
+}
 
-  if( !insert )
-    return -1;
-
-  memmove( insert, infohash, OT_HASH_COMPARE_SIZE );
-
-  return 0;
+/* This must be called with g_accesslist_mutex held.
+   This will never delete head, because that might still be in use. */
+static void accesslist_clean(ot_accesslist *accesslist) {
+  while (accesslist && accesslist->next) {
+    if (accesslist->next->base + 5 < g_now_minutes)
+      accesslist->next = accesslist_free(accesslist->next);
+    accesslist = accesslist->next;
+  }
 }
 
 /* Read initial access list */
-static void accesslist_readfile( int foo ) {
-  FILE *  accesslist_filehandle;
-  ot_hash infohash;
-  char    inbuf[512];
-  foo = foo;
+static void accesslist_readfile(void) {
+  ot_accesslist *accesslist_new;
+  ot_hash       *info_hash;
+  const char    *map, *map_end, *read_offs;
+  size_t         maplen;
 
-  accesslist_filehandle = fopen( g_accesslist_filename, "r" );
-
-  /* Free accesslist vector in trackerlogic.c*/
-  accesslist_reset();
-
-  if( accesslist_filehandle == NULL ) {
-    fprintf( stderr, "Warning: Can't open accesslist file: %s (but will try to create it later, if necessary and possible).", g_accesslist_filename );
+  if ((map = mmap_read(g_accesslist_filename, &maplen)) == NULL) {
+    char *wd = getcwd(NULL, 0);
+    fprintf(stderr, "Warning: Can't open accesslist file: %s (but will try to create it later, if necessary and possible).\nPWD: %s\n", g_accesslist_filename, wd);
+    free(wd);
     return;
   }
 
-  /* We do ignore anything that is not of the form "^[:xdigit:]{40}[^:xdigit:].*" */
-  while( fgets( inbuf, sizeof(inbuf), accesslist_filehandle ) ) {
-    int i;
-    for( i=0; i<20; ++i ) {
-      int eger = 16 * scan_fromhex( inbuf[ 2*i ] ) + scan_fromhex( inbuf[ 1 + 2*i ] );
-      if( eger < 0 )
-        continue;
-      infohash[i] = eger;
-    }
-    if( scan_fromhex( inbuf[ 40 ] ) >= 0 )
-      continue;
-
-    /* Append accesslist to accesslist vector */
-    accesslist_addentry( &infohash );
+  /* You need at least 41 bytes to pass an info_hash, make enough room
+     for the maximum amount of them */
+  accesslist_new = accesslist_make(g_accesslist, maplen / 41);
+  if (!accesslist_new) {
+    fprintf(stderr, "Warning: Not enough memory to allocate %zd bytes for accesslist buffer. May succeed later.\n", (maplen / 41) * 20);
+    mmap_unmap(map, maplen);
+    return;
   }
+  info_hash = accesslist_new->list;
 
-  fclose( accesslist_filehandle );
+  /* No use to scan if there's not enough room for another full info_hash */
+  map_end   = map + maplen - 40;
+  read_offs = map;
+
+  /* We do ignore anything that is not of the form "^[:xdigit:]{40}[^:xdigit:].*" */
+  while (read_offs <= map_end) {
+    int i;
+    for (i = 0; i < (int)sizeof(ot_hash); ++i) {
+      int eger1 = scan_fromhex((unsigned char)read_offs[2 * i]);
+      int eger2 = scan_fromhex((unsigned char)read_offs[1 + 2 * i]);
+      if (eger1 < 0 || eger2 < 0)
+        break;
+      (*info_hash)[i] = (uint8_t)(eger1 * 16 + eger2);
+    }
+
+    if (i == sizeof(ot_hash)) {
+      read_offs += 40;
+
+      /* Append accesslist to accesslist vector */
+      if (read_offs == map_end || scan_fromhex((unsigned char)*read_offs) < 0)
+        ++info_hash;
+    }
+
+    /* Find start of next line */
+    while (read_offs <= map_end && *(read_offs++) != '\n')
+      ;
+  }
+#ifdef _DEBUG
+  fprintf(stderr, "Added %zd info_hashes to accesslist\n", (size_t)(info_hash - accesslist_new->list));
+#endif
+
+  mmap_unmap(map, maplen);
+
+  qsort(accesslist_new->list, info_hash - accesslist_new->list, sizeof(*info_hash), vector_compare_hash);
+  accesslist_new->size = info_hash - accesslist_new->list;
+
+  /* Now exchange the accesslist vector in the least race condition prone way */
+  pthread_mutex_lock(&g_accesslist_mutex);
+  accesslist_new->next = g_accesslist;
+  g_accesslist         = accesslist_new; /* Only now set a new list */
+
+#ifdef WANT_DYNAMIC_ACCESSLIST
+  /* If we have dynamic accesslists, reloading a new one will always void the add/delete lists.
+     Insert empty ones at the list head */
+  if (g_accesslist_add && (accesslist_new = accesslist_make(g_accesslist_add, 0)) != NULL)
+    g_accesslist_add = accesslist_new;
+  if (g_accesslist_delete && (accesslist_new = accesslist_make(g_accesslist_delete, 0)) != NULL)
+    g_accesslist_delete = accesslist_new;
+#endif
+
+  accesslist_clean(g_accesslist);
+
+  pthread_mutex_unlock(&g_accesslist_mutex);
 }
 
-int accesslist_hashisvalid( ot_hash *hash ) {
-  int exactmatch;
-  binary_search( hash, accesslist.data, accesslist.size, OT_HASH_COMPARE_SIZE, OT_HASH_COMPARE_SIZE, &exactmatch );
+int accesslist_hashisvalid(ot_hash hash) {
+  /* Get working copy of current access list */
+  ot_accesslist *accesslist = g_accesslist;
+#ifdef WANT_DYNAMIC_ACCESSLIST
+  ot_accesslist *accesslist_add, *accesslist_delete;
+#endif
+  void *exactmatch = NULL;
+
+  if (accesslist)
+    exactmatch = bsearch(hash, accesslist->list, accesslist->size, OT_HASH_COMPARE_SIZE, vector_compare_hash);
+
+#ifdef WANT_DYNAMIC_ACCESSLIST
+  /* If we had no match on the main list, scan the list of dynamically added hashes */
+  accesslist_add = g_accesslist_add;
+  if ((exactmatch == NULL) && accesslist_add)
+    exactmatch = bsearch(hash, accesslist_add->list, accesslist_add->size, OT_HASH_COMPARE_SIZE, vector_compare_hash);
+
+  /* If we found a matching hash on the main list, scan the list of dynamically deleted hashes */
+  accesslist_delete = g_accesslist_delete;
+  if ((exactmatch != NULL) && accesslist_delete && bsearch(hash, accesslist_add->list, accesslist_add->size, OT_HASH_COMPARE_SIZE, vector_compare_hash))
+    exactmatch = NULL;
+#endif
 
 #ifdef WANT_ACCESSLIST_BLACK
-  exactmatch = !exactmatch;
+  return exactmatch == NULL;
+#else
+  return exactmatch != NULL;
 #endif
-
-  return exactmatch;
 }
 
-void accesslist_init( ) {
-  byte_zero( &accesslist, sizeof( accesslist ) );
+static void *accesslist_worker(void *args) {
+  int      sig;
+  sigset_t signal_mask;
 
-  /* Passing "0" since read_blacklist_file also is SIGHUP handler */
-  if( g_accesslist_filename ) {
-    accesslist_readfile( 0 );
-    signal( SIGHUP, accesslist_readfile );
+  sigemptyset(&signal_mask);
+  sigaddset(&signal_mask, SIGHUP);
+
+  (void)args;
+
+  while (1) {
+    if (!g_opentracker_running)
+      return NULL;
+
+    /* Initial attempt to read accesslist */
+    accesslist_readfile();
+
+    /* Wait for signals */
+    while (sigwait(&signal_mask, &sig) != 0 && sig != SIGHUP)
+      ;
   }
+  return NULL;
+}
+
+#ifdef WANT_DYNAMIC_ACCESSLIST
+static pthread_t thread_adder_id, thread_deleter_id;
+static void     *accesslist_adddel_worker(char *fifoname, ot_accesslist *_Atomic *adding_to, ot_accesslist *_Atomic *removing_from) {
+  struct stat st;
+
+  if (!stat(fifoname, &st)) {
+    if (!S_ISFIFO(st.st_mode)) {
+      fprintf(stderr, "Error when starting dynamic accesslists: Found Non-FIFO file at %s.\nPlease remove it and restart opentracker.\n", fifoname);
+      return NULL;
+    }
+  } else {
+    int error = mkfifo(fifoname, 0755);
+    if (error && error != EEXIST) {
+      fprintf(stderr, "Error when starting dynamic accesslists: Couldn't create FIFO at %s, error: %s\n", fifoname, strerror(errno));
+      return NULL;
+    }
+  }
+
+  while (g_opentracker_running) {
+    FILE   *fifo    = fopen(fifoname, "r");
+    char   *line    = NULL;
+    size_t  linecap = 0;
+    ssize_t linelen;
+
+    if (!fifo) {
+      fprintf(stderr, "Error when reading dynamic accesslists: Couldn't open FIFO at %s, error: %s\n", fifoname, strerror(errno));
+      return NULL;
+    }
+
+    while ((linelen = getline(&line, &linecap, fifo)) > 0) {
+      ot_hash info_hash;
+      int     i;
+
+      printf("Got line %*s", (int)linelen, line);
+      /* We do ignore anything that is not of the form "^[:xdigit:]{40}[^:xdigit:].*"
+        If there's not enough characters for an info_hash in the line, skip it. */
+      if (linelen < 41)
+        continue;
+
+      for (i = 0; i < (int)sizeof(ot_hash); ++i) {
+        int eger1 = scan_fromhex((unsigned char)line[2 * i]);
+        int eger2 = scan_fromhex((unsigned char)line[1 + 2 * i]);
+        if (eger1 < 0 || eger2 < 0)
+          break;
+        ((uint8_t *)info_hash)[i] = (uint8_t)(eger1 * 16 + eger2);
+      }
+      printf("parsed info_hash %20s\n", info_hash);
+      if (i != sizeof(ot_hash))
+        continue;
+
+      /* From now on we modify g_accesslist_add and g_accesslist_delete, so prevent the
+         other worker threads from doing the same */
+      pthread_mutex_lock(&g_accesslist_mutex);
+
+      /* If the info hash is in the removing_from list, create a new head without that entry */
+      if (*removing_from && (*removing_from)->list) {
+        ot_hash *exactmatch = bsearch(info_hash, (*removing_from)->list, (*removing_from)->size, OT_HASH_COMPARE_SIZE, vector_compare_hash);
+        if (exactmatch) {
+          ptrdiff_t      off            = exactmatch - (*removing_from)->list;
+          ot_accesslist *accesslist_new = accesslist_make(*removing_from, (*removing_from)->size - 1);
+          if (accesslist_new) {
+            memcpy(accesslist_new->list, (*removing_from)->list, sizeof(ot_hash) * off);
+            memcpy(accesslist_new->list + off, (*removing_from)->list + off + 1, (*removing_from)->size - off - 1);
+            *removing_from = accesslist_new;
+          }
+        }
+      }
+
+      /* Simple case: there's no adding_to list yet, create one with one member */
+      if (!*adding_to) {
+        ot_accesslist *accesslist_new = accesslist_make(NULL, 1);
+        if (accesslist_new) {
+          memcpy(accesslist_new->list, info_hash, sizeof(ot_hash));
+          *adding_to = accesslist_new;
+        }
+      } else {
+        int exactmatch   = 0;
+        ot_hash *insert_point = binary_search(info_hash, (*adding_to)->list, (*adding_to)->size, OT_HASH_COMPARE_SIZE, sizeof(ot_hash), &exactmatch);
+
+        /* Only if the info hash is not in the adding_to list, create a new head with that entry */
+        if (!exactmatch) {
+          ot_accesslist *accesslist_new = accesslist_make(*adding_to, (*adding_to)->size + 1);
+          ptrdiff_t      off            = insert_point - (*adding_to)->list;
+          if (accesslist_new) {
+            memcpy(accesslist_new->list, (*adding_to)->list, sizeof(ot_hash) * off);
+            memcpy(accesslist_new->list + off, info_hash, sizeof(info_hash));
+            memcpy(accesslist_new->list + off + 1, (*adding_to)->list + off, (*adding_to)->size - off);
+            *adding_to = accesslist_new;
+          }
+        }
+      }
+
+      pthread_mutex_unlock(&g_accesslist_mutex);
+    }
+
+    fclose(fifo);
+  }
+  return NULL;
+}
+
+static void *accesslist_adder_worker(void *args) {
+  (void)args;
+  return accesslist_adddel_worker(g_accesslist_pipe_add, &g_accesslist_add, &g_accesslist_delete);
+}
+static void *accesslist_deleter_worker(void *args) {
+  (void)args;
+  return accesslist_adddel_worker(g_accesslist_pipe_delete, &g_accesslist_delete, &g_accesslist_add);
 }
 #endif
 
-static uint32_t       g_adminip_addresses[OT_ADMINIP_MAX];
-static ot_permissions g_adminip_permissions[OT_ADMINIP_MAX];
-static unsigned int   g_adminip_count = 0;
-
-int accesslist_blessip( char *ip, ot_permissions permissions ) {
-  if( g_adminip_count >= OT_ADMINIP_MAX )
-    return -1;
-  memmove( g_adminip_addresses + g_adminip_count, ip, 4 );
-  g_adminip_permissions[ g_adminip_count++ ] = permissions;
-#ifdef _DEBUG
-  uint8_t *_ip = (uint8_t*)ip;
-  fprintf( stderr, "Blessing ip address %d.%d.%d.%d with:", _ip[0], _ip[1], _ip[2], _ip[3]);
-  if( permissions & OT_PERMISSION_MAY_STAT       ) fputs( " may_fetch_stats", stderr );
-  if( permissions & OT_PERMISSION_MAY_SYNC       ) fputs( " may_sync_batch", stderr );
-  if( permissions & OT_PERMISSION_MAY_LIVESYNC   ) fputs( " may_sync_live", stderr );
-  if( permissions & OT_PERMISSION_MAY_FULLSCRAPE ) fputs( " may_fetch_fullscrapes", stderr );
-  if( !permissions ) fputs(" nothing.\n", stderr); else fputs(".\n", stderr );
+static pthread_t thread_id;
+void accesslist_init() {
+  pthread_mutex_init(&g_accesslist_mutex, NULL);
+  pthread_create(&thread_id, NULL, accesslist_worker, NULL);
+#ifdef WANT_DYNAMIC_ACCESSLIST
+  if (g_accesslist_pipe_add)
+    pthread_create(&thread_adder_id, NULL, accesslist_adder_worker, NULL);
+  if (g_accesslist_pipe_delete)
+    pthread_create(&thread_deleter_id, NULL, accesslist_deleter_worker, NULL);
 #endif
+}
+
+void accesslist_deinit(void) {
+  /* Wake up sleeping worker */
+  pthread_kill(thread_id, SIGHUP);
+
+  pthread_mutex_lock(&g_accesslist_mutex);
+
+  g_accesslist = accesslist_free(g_accesslist);
+
+#ifdef WANT_DYNAMIC_ACCESSLIST
+  g_accesslist_add    = accesslist_free(g_accesslist_add);
+  g_accesslist_delete = accesslist_free(g_accesslist_delete);
+#endif
+
+  pthread_mutex_unlock(&g_accesslist_mutex);
+  pthread_cancel(thread_id);
+  pthread_mutex_destroy(&g_accesslist_mutex);
+}
+
+void accesslist_cleanup(void) {
+  pthread_mutex_lock(&g_accesslist_mutex);
+
+  accesslist_clean(g_accesslist);
+#if WANT_DYNAMIC_ACCESSLIST
+  accesslist_clean(g_accesslist_add);
+  accesslist_clean(g_accesslist_delete);
+#endif
+
+  pthread_mutex_unlock(&g_accesslist_mutex);
+}
+#endif
+
+int address_in_net(const ot_ip6 address, const ot_net *net) {
+  int bits = net->bits, checkbits = (0x7f00 >> (bits & 7));
+  int result = memcmp(address, &net->address, bits >> 3);
+  if (!result && (bits & 7))
+    result = (checkbits & address[bits >> 3]) - (checkbits & net->address[bits >> 3]);
+  return result == 0;
+}
+
+void *set_value_for_net(const ot_net *net, ot_vector *vector, const void *value, const size_t member_size) {
+  size_t i;
+  int    exactmatch;
+
+  /* Caller must have a concept of ot_net in it's member */
+  if (member_size < sizeof(ot_net))
+    return 0;
+
+  /* Check each net in vector for overlap */
+  uint8_t *member = ((uint8_t *)vector->data);
+  for (i = 0; i < vector->size; ++i) {
+    if (address_in_net(*(ot_ip6 *)member, net) || address_in_net(net->address, (ot_net *)member))
+      return 0;
+    member += member_size;
+  }
+
+  member = vector_find_or_insert(vector, (void *)net, member_size, sizeof(ot_net), &exactmatch);
+  if (member) {
+    memcpy(member, net, sizeof(ot_net));
+    memcpy(member + sizeof(ot_net), value, member_size - sizeof(ot_net));
+  }
+
+  return member;
+}
+
+/* Takes a vector filled with { ot_net net, uint8_t[x] value };
+   Returns value associated with the net, or NULL if not found */
+void *get_value_for_net(const ot_ip6 address, const ot_vector *vector, const size_t member_size) {
+  int     exactmatch;
+  /* This binary search will return a pointer to the first non-containing network... */
+  ot_net *net = binary_search(address, vector->data, vector->size, member_size, sizeof(ot_ip6), &exactmatch);
+  if (!net)
+    return NULL;
+  /* ... so we'll need to move back one step unless we've exactly hit the first address in network */
+  if (!exactmatch && ((void *)net > vector->data))
+    --net;
+  if (!address_in_net(address, net))
+    return NULL;
+  return (void *)net;
+}
+
+#ifdef WANT_FULLLOG_NETWORKS
+static ot_vector       g_lognets_list;
+ot_log                *g_logchain_first, *g_logchain_last;
+static pthread_mutex_t g_lognets_list_mutex = PTHREAD_MUTEX_INITIALIZER;
+
+void loglist_add_network(const ot_net *net) {
+  pthread_mutex_lock(&g_lognets_list_mutex);
+  set_value_for_net(net, &g_lognets_list, NULL, sizeof(ot_net));
+  pthread_mutex_unlock(&g_lognets_list_mutex);
+}
+
+void loglist_reset() {
+  pthread_mutex_lock(&g_lognets_list_mutex);
+  free(g_lognets_list.data);
+  g_lognets_list.data = 0;
+  g_lognets_list.size = g_lognets_list.space = 0;
+  pthread_mutex_unlock(&g_lognets_list_mutex);
+}
+
+int loglist_check_address(const ot_ip6 address) {
+  int result;
+  pthread_mutex_lock(&g_lognets_list_mutex);
+  result = (NULL != get_value_for_net(address, &g_lognets_list, sizeof(ot_net)));
+  pthread_mutex_unlock(&g_lognets_list_mutex);
+  return result;
+}
+#endif
+
+#ifdef WANT_IP_FROM_PROXY
+typedef struct {
+  ot_net   *proxy;
+  ot_vector networks;
+} ot_proxymap;
+
+static ot_vector       g_proxies_list;
+static pthread_mutex_t g_proxies_list_mutex = PTHREAD_MUTEX_INITIALIZER;
+
+int proxylist_add_network(const ot_net *proxy, const ot_net *net) {
+  ot_proxymap *map;
+  int exactmatch, result = 1;
+  pthread_mutex_lock(&g_proxies_list_mutex);
+
+  /* If we have a direct hit, use and extend the vector there */
+  map = binary_search(proxy, g_proxies_list.data, g_proxies_list.size, sizeof(ot_proxymap), sizeof(ot_net), &exactmatch);
+
+  if (!map || !exactmatch) {
+    /* else see, if we've got overlapping networks
+       and get a new empty vector if not */
+    ot_vector empty;
+    memset(&empty, 0, sizeof(ot_vector));
+    map = set_value_for_net(proxy, &g_proxies_list, &empty, sizeof(ot_proxymap));
+  }
+
+  if (map && set_value_for_net(net, &map->networks, NULL, sizeof(ot_net)))
+    result = 1;
+
+  pthread_mutex_unlock(&g_proxies_list_mutex);
+  return result;
+}
+
+int proxylist_check_proxy(const ot_ip6 proxy, const ot_ip6 address) {
+  int result = 0;
+  ot_proxymap *map;
+
+  pthread_mutex_lock(&g_proxies_list_mutex);
+
+  if ((map = get_value_for_net(proxy, &g_proxies_list, sizeof(ot_proxymap))))
+    if (!address || get_value_for_net(address, &map->networks, sizeof(ot_net)))
+      result = 1;
+
+  pthread_mutex_unlock(&g_proxies_list_mutex);
+  return result;
+}
+
+#endif
+
+static ot_net         g_admin_nets[OT_ADMINIP_MAX];
+static ot_permissions g_admin_nets_permissions[OT_ADMINIP_MAX];
+static unsigned int   g_admin_nets_count = 0;
+
+int accesslist_bless_net(ot_net *net, ot_permissions permissions) {
+  if (g_admin_nets_count >= OT_ADMINIP_MAX)
+    return -1;
+
+  memcpy(g_admin_nets + g_admin_nets_count, net, sizeof(ot_net));
+  g_admin_nets_permissions[g_admin_nets_count++] = permissions;
+
+#ifdef _DEBUG
+  {
+    char _debug[512];
+    int  off  = snprintf(_debug, sizeof(_debug), "Blessing ip net ");
+    off      += fmt_ip6c(_debug + off, net->address);
+    if (net->bits < 128) {
+      _debug[off++] = '/';
+      if (ip6_isv4mapped(net->address))
+        off += fmt_long(_debug + off, net->bits - 96);
+      else
+        off += fmt_long(_debug + off, net->bits);
+    }
+
+    if (permissions & OT_PERMISSION_MAY_STAT)
+      off += snprintf(_debug + off, 512 - off, " may_fetch_stats");
+    if (permissions & OT_PERMISSION_MAY_LIVESYNC)
+      off += snprintf(_debug + off, 512 - off, " may_sync_live");
+    if (permissions & OT_PERMISSION_MAY_FULLSCRAPE)
+      off += snprintf(_debug + off, 512 - off, " may_fetch_fullscrapes");
+    if (permissions & OT_PERMISSION_MAY_PROXY)
+      off += snprintf(_debug + off, 512 - off, " may_proxy");
+    if (!permissions)
+      off += snprintf(_debug + off, sizeof(_debug) - off, " nothing");
+    _debug[off++] = '.';
+    _debug[off++] = '\n';
+    (void)write(2, _debug, off);
+  }
+#endif
+
   return 0;
 }
 
-int accesslist_isblessed( char *ip, ot_permissions permissions ) {
+int accesslist_is_blessed(ot_ip6 ip, ot_permissions permissions) {
   unsigned int i;
-  for( i=0; i<g_adminip_count; ++i )
-    if( !memcmp( g_adminip_addresses + i, ip, 4) && ( g_adminip_permissions[ i ] & permissions ) )
+  for (i = 0; i < g_admin_nets_count; ++i)
+    if (address_in_net(ip, g_admin_nets + i) && (g_admin_nets_permissions[i] & permissions))
       return 1;
   return 0;
 }
-
-const char *g_version_accesslist_c = "$Source$: $Revision$\n";

ot_accesslist.h

@@ -3,33 +3,87 @@
 
    $id$ */
 
-#ifndef __OT_ACCESSLIST_H__
-#define __OT_ACCESSLIST_H__
+#ifndef OT_ACCESSLIST_H__
+#define OT_ACCESSLIST_H__
 
-#if defined ( WANT_ACCESSLIST_BLACK ) && defined (WANT_ACCESSLIST_WHITE )
-  #error WANT_ACCESSLIST_BLACK and WANT_ACCESSLIST_WHITE are exclusive.
+#include "trackerlogic.h"
+
+#if defined(WANT_ACCESSLIST_BLACK) && defined(WANT_ACCESSLIST_WHITE)
+#error WANT_ACCESSLIST_BLACK and WANT_ACCESSLIST_WHITE are exclusive.
 #endif
 
-#if defined ( WANT_ACCESSLIST_BLACK ) || defined (WANT_ACCESSLIST_WHITE )
+#if defined(WANT_ACCESSLIST_BLACK) || defined(WANT_ACCESSLIST_WHITE)
 #define WANT_ACCESSLIST
-void accesslist_init( );
-int  accesslist_hashisvalid( ot_hash *hash );
+void         accesslist_init(void);
+void         accesslist_deinit(void);
+int          accesslist_hashisvalid(ot_hash hash);
+void         accesslist_cleanup(void);
 
 extern char *g_accesslist_filename;
+#ifdef WANT_DYNAMIC_ACCESSLIST
+extern char *g_accesslist_pipe_add;
+extern char *g_accesslist_pipe_delete;
+#endif
 
 #else
-#define accesslist_init( accesslist_filename )
-#define accesslist_hashisvalid( hash ) 1
+#ifdef WANT_DYNAMIC_ACCESSLIST
+#error WANT_DYNAMIC_ACCESSLIST needs either WANT_ACCESSLIST_BLACK or WANT_ACCESSLIST_WHITE
+#endif
+
+#define accesslist_init(accesslist_filename)
+#define accesslist_deinit()
+#define accesslist_hashisvalid(hash) 1
+#endif
+
+/* Test if an address is subset of an ot_net, return value is considered a bool */
+int   address_in_net(const ot_ip6 address, const ot_net *net);
+
+/* Store a value into a vector of struct { ot_net net, uint8_t[x] value } member;
+   returns NULL
+     if member_size is too small, or
+     if one of the nets inside the vector are a subnet of _net_, or
+     if _net_ is a subnet of one of the nets inside the vector, or
+     if the vector could not be resized
+   returns pointer to new member in vector for success
+   member_size can be sizeof(ot_net) to reduce the lookup to a boolean mapping
+*/
+void *set_value_for_net(const ot_net *net, ot_vector *vector, const void *value, const size_t member_size);
+
+/* Takes a vector filled with struct { ot_net net, uint8_t[x] value } member;
+   Returns pointer to _member_ associated with the net, or NULL if not found
+   member_size can be sizeof(ot_net) to reduce the lookup to a boolean mapping
+*/
+void *get_value_for_net(const ot_ip6 address, const ot_vector *vector, const size_t member_size);
+
+#ifdef WANT_IP_FROM_PROXY
+int proxylist_add_network(const ot_net *proxy, const ot_net *net);
+int proxylist_check_network(const ot_ip6 *proxy, const ot_ip6 address /* can be NULL to only check proxy */);
+#endif
+
+#ifdef WANT_FULLLOG_NETWORKS
+typedef struct ot_log ot_log;
+struct ot_log {
+  ot_ip6   ip;
+  uint8_t *data;
+  size_t   size;
+  ot_time  time;
+  ot_log  *next;
+};
+extern ot_log *g_logchain_first, *g_logchain_last;
+
+void           loglist_add_network(const ot_net *net);
+void           loglist_reset();
+int            loglist_check_address(const ot_ip6 address);
 #endif
 
 typedef enum {
   OT_PERMISSION_MAY_FULLSCRAPE = 0x1,
-  OT_PERMISSION_MAY_SYNC       = 0x2,
-  OT_PERMISSION_MAY_STAT       = 0x4,
-  OT_PERMISSION_MAY_LIVESYNC   = 0x8
+  OT_PERMISSION_MAY_STAT       = 0x2,
+  OT_PERMISSION_MAY_LIVESYNC   = 0x4,
+  OT_PERMISSION_MAY_PROXY      = 0x8
 } ot_permissions;
 
-int  accesslist_blessip( char * ip, ot_permissions permissions );
-int  accesslist_isblessed( char * ip, ot_permissions permissions );
+int accesslist_bless_net(ot_net *net, ot_permissions permissions);
+int accesslist_is_blessed(ot_ip6 ip, ot_permissions permissions);
 
 #endif

ot_clean.c

@@ -4,138 +4,141 @@
    $id$ */
 
 /* System */
-#include <stdlib.h>
-#include <string.h>
 #include <pthread.h>
-#include <sys/uio.h>
+#include <string.h>
 #include <unistd.h>
 
 /* Libowfat */
-#include "byte.h"
 #include "io.h"
 
 /* Opentracker */
-#include "trackerlogic.h"
+#include "ot_accesslist.h"
+#include "ot_clean.h"
 #include "ot_mutex.h"
+#include "ot_stats.h"
+#include "ot_vector.h"
+#include "trackerlogic.h"
 
-/* Clean a single torrent
-   return 1 if torrent timed out
-*/
-int clean_single_torrent( ot_torrent *torrent ) {
-  ot_peerlist *peer_list = torrent->peer_list;
-  size_t peers_count = 0, seeds_count;
-  time_t timedout = (int)( NOW - peer_list->base );
-  int i;
-#ifdef WANT_SYNC_BATCH
-  char *new_peers;
-#endif
+/* Returns amount of removed peers */
+static ssize_t clean_single_bucket(ot_peer *peers, size_t peer_count, size_t peer_size, time_t timedout, int *removed_seeders) {
+  ot_peer *last_peer = peers + peer_count * peer_size, *insert_point;
 
-  if( !timedout )
+  /* Two scan modes: unless there is one peer removed, just increase ot_peertime */
+  while (peers < last_peer) {
+    time_t timediff = timedout + OT_PEERTIME(peers, peer_size);
+    if (timediff >= OT_PEER_TIMEOUT)
+      break;
+    OT_PEERTIME(peers, peer_size)  = timediff;
+    peers += peer_size;
+  }
+
+  /* If we at least remove one peer, we have to copy */
+  for (insert_point = peers; peers < last_peer; peers += peer_size) {
+    time_t timediff = timedout + OT_PEERTIME(peers, peer_size);
+
+    if (timediff < OT_PEER_TIMEOUT) {
+      OT_PEERTIME(peers, peer_size) = timediff;
+      memcpy(insert_point, peers, peer_size);
+      insert_point += peer_size;
+    } else if (OT_PEERFLAG_D(peers, peer_size) & PEER_FLAG_SEEDING)
+      (*removed_seeders)++;
+  }
+
+  return (peers - insert_point) / peer_size;
+}
+
+int clean_single_peer_list(ot_peerlist *peer_list, size_t peer_size) {
+  ot_vector *peer_vector = &peer_list->peers;
+  time_t     timedout    = (time_t)(g_now_minutes - peer_list->base);
+  int        num_buckets = 1, removed_seeders = 0;
+
+  /* No need to clean empty torrent */
+  if (!timedout)
     return 0;
 
   /* Torrent has idled out */
-  if( timedout > OT_TORRENT_TIMEOUT )
+  if (timedout > OT_TORRENT_TIMEOUT)
     return 1;
 
   /* Nothing to be cleaned here? Test if torrent is worth keeping */
-  if( timedout > OT_POOLS_COUNT ) {
-    if( !peer_list->peer_count )
+  if (timedout > OT_PEER_TIMEOUT) {
+    if (!peer_list->peer_count)
       return peer_list->down_count ? 0 : 1;
-    timedout = OT_POOLS_COUNT;
+    timedout = OT_PEER_TIMEOUT;
   }
 
-  /* Release vectors that have timed out */
-  for( i = OT_POOLS_COUNT - timedout; i < OT_POOLS_COUNT; ++i )
-    free( peer_list->peers[i].data);
-
-  /* Shift vectors back by the amount of pools that were shifted out */
-  memmove( peer_list->peers + timedout, peer_list->peers, sizeof( ot_vector ) * ( OT_POOLS_COUNT - timedout ) );
-  byte_zero( peer_list->peers, sizeof( ot_vector ) * timedout );
-
-  /* Shift back seed counts as well */
-  memmove( peer_list->seed_counts + timedout, peer_list->seed_counts, sizeof( size_t ) * ( OT_POOLS_COUNT - timedout ) );
-  byte_zero( peer_list->seed_counts, sizeof( size_t ) * timedout );
-
-#ifdef WANT_SYNC_BATCH
-  /* Save the block modified within last OT_POOLS_TIMEOUT */
-  if( peer_list->peers[1].size &&
-    ( new_peers = realloc( peer_list->changeset.data, sizeof( ot_peer ) * peer_list->peers[1].size ) ) )
-  {
-    memmove( new_peers, peer_list->peers[1].data, peer_list->peers[1].size );
-    peer_list->changeset.data = new_peers;
-    peer_list->changeset.size = sizeof( ot_peer ) * peer_list->peers[1].size;
-  } else {
-    free( peer_list->changeset.data );
-
-    memset( &peer_list->changeset, 0, sizeof( ot_vector ) );
+  if (OT_PEERLIST_HASBUCKETS(peer_list)) {
+    num_buckets = peer_vector->size;
+    peer_vector = (ot_vector *)peer_vector->data;
   }
-#endif
 
-  peers_count = seeds_count = 0;
-  for( i = 0; i < OT_POOLS_COUNT; ++i ) {
-    peers_count += peer_list->peers[i].size;
-    seeds_count += peer_list->seed_counts[i];
+  while (num_buckets--) {
+    size_t removed_peers   = clean_single_bucket(peer_vector->data, peer_vector->size, peer_size, timedout, &removed_seeders);
+    peer_list->peer_count -= removed_peers;
+    peer_vector->size     -= removed_peers;
+    if (removed_peers)
+      vector_fixup_peers(peer_vector, peer_size);
+
+    /* Skip to next bucket, a vector containing peers */
+    ++peer_vector;
   }
-  peer_list->seed_count = seeds_count;
-  peer_list->peer_count = peers_count;
 
-  if( peers_count )
-    peer_list->base = NOW;
+  peer_list->seed_count -= removed_seeders;
+
+  /* See if we need to convert a torrent from simple vector to bucket list */
+  if ((peer_list->peer_count > OT_PEER_BUCKET_MINCOUNT) || OT_PEERLIST_HASBUCKETS(peer_list))
+    vector_redistribute_buckets(peer_list, peer_size);
+
+  if (peer_list->peer_count)
+    peer_list->base = g_now_minutes;
   else {
     /* When we got here, the last time that torrent
-       has been touched is OT_POOLS_COUNT units before */
-    peer_list->base = NOW - OT_POOLS_COUNT;
+     has been touched is OT_PEER_TIMEOUT Minutes before */
+    peer_list->base = g_now_minutes - OT_PEER_TIMEOUT;
   }
   return 0;
 }
 
-static void clean_make() {
-  int bucket;
-
-  for( bucket = OT_BUCKET_COUNT - 1; bucket >= 0; --bucket ) {
-    ot_vector *torrents_list = mutex_bucket_lock( bucket );
-    size_t     toffs;
-
-    for( toffs=0; toffs<torrents_list->size; ++toffs ) {
-      ot_torrent *torrent = ((ot_torrent*)(torrents_list->data)) + toffs;
-      if( clean_single_torrent( torrent ) ) {
-        vector_remove_torrent( torrents_list, torrent );
-        --toffs; continue;
-      }
-    }
-    mutex_bucket_unlock( bucket );
-    
-    /* We want the cleanup to be spread about 2 Minutes to reduce load spikes
-       during cleanup. Sleeping around two minutes was chosen to allow enough
-       time for the actual work and fluctuations in timer. */
-    usleep( ( 2 * 60 * 1000000 ) / OT_BUCKET_COUNT );
-  }
+/* Clean a single torrent
+   return 1 if torrent timed out
+*/
+int clean_single_torrent(ot_torrent *torrent) {
+  return clean_single_peer_list(torrent->peer_list6, OT_PEER_SIZE6) * clean_single_peer_list(torrent->peer_list4, OT_PEER_SIZE4);
 }
 
 /* Clean up all peers in current bucket, remove timedout pools and
-   torrents */
-static void * clean_worker( void * args ) {
-  args = args;
-  while( 1 ) {
-    ot_tasktype tasktype = TASK_CLEAN;
-    ot_taskid   taskid   = mutex_workqueue_poptask( &tasktype );
-    clean_make(  );
-    mutex_workqueue_pushsuccess( taskid );
+ torrents */
+static void *clean_worker(void *args) {
+  (void)args;
+  while (1) {
+    int bucket = OT_BUCKET_COUNT;
+    while (bucket--) {
+      ot_vector *torrents_list = mutex_bucket_lock(bucket);
+      size_t     toffs;
+      int        delta_torrentcount = 0;
+
+      for (toffs = 0; toffs < torrents_list->size; ++toffs) {
+        ot_torrent *torrent = ((ot_torrent *)(torrents_list->data)) + toffs;
+        if (clean_single_torrent(torrent)) {
+          vector_remove_torrent(torrents_list, torrent);
+          --delta_torrentcount;
+          --toffs;
+        }
+      }
+      mutex_bucket_unlock(bucket, delta_torrentcount);
+      if (!g_opentracker_running)
+        return NULL;
+      usleep(OT_CLEAN_SLEEP);
+    }
+    stats_cleanup();
+#ifdef WANT_ACCESSLIST
+    accesslist_cleanup();
+#endif
   }
   return NULL;
 }
 
-void clean_all_torrents( ) {
-  mutex_workqueue_pushtask( 0, TASK_CLEAN );
-}
-
 static pthread_t thread_id;
-void clean_init( void ) {
-  pthread_create( &thread_id, NULL, clean_worker, NULL );
-}
+void clean_init(void) { pthread_create(&thread_id, NULL, clean_worker, NULL); }
 
-void clean_deinit( void ) {
-  pthread_cancel( thread_id );
-}
-
-const char *g_version_clean_c = "$Source$: $Revision$\n";
+void clean_deinit(void) { pthread_cancel(thread_id); }

ot_clean.h

@@ -3,13 +3,17 @@
 
    $id$ */
 
-#ifndef __OT_CLEAN_H__
-#define __OT_CLEAN_H__
+#ifndef OT_CLEAN_H__
+#define OT_CLEAN_H__
 
-void clean_init( void );
-void clean_deinit( void );
+/* The amount of time a clean cycle should take */
+#define OT_CLEAN_INTERVAL_MINUTES 2
 
-void clean_all_torrents( void );
-int  clean_single_torrent( ot_torrent *torrent );
+/* So after each bucket wait 1 / OT_BUCKET_COUNT intervals */
+#define OT_CLEAN_SLEEP            (((OT_CLEAN_INTERVAL_MINUTES) * 60 * 1000000) / (OT_BUCKET_COUNT))
+
+void clean_init(void);
+void clean_deinit(void);
+int  clean_single_torrent(ot_torrent *torrent);
 
 #endif

ot_fullscrape.c

@@ -6,15 +6,18 @@
 #ifdef WANT_FULLSCRAPE
 
 /* System */
-#include <sys/param.h>
-#include <sys/uio.h>
+#include <arpa/inet.h>
+#include <pthread.h>
 #include <stdio.h>
 #include <string.h>
-#include <pthread.h>
-#include <arpa/inet.h>
+#include <sys/param.h>
 #ifdef WANT_COMPRESSION_GZIP
 #include <zlib.h>
 #endif
+#ifdef WANT_COMPRESSION_ZSTD
+#include <zstd.h>
+#endif
+
 
 /* Libowfat */
 #include "byte.h"
@@ -22,50 +25,64 @@
 #include "textcode.h"
 
 /* Opentracker */
-#include "trackerlogic.h"
-#include "ot_mutex.h"
-#include "ot_iovec.h"
 #include "ot_fullscrape.h"
+#include "ot_iovec.h"
+#include "ot_mutex.h"
+#include "trackerlogic.h"
 
 /* Fetch full scrape info for all torrents
    Full scrapes usually are huge and one does not want to
    allocate more memory. So lets get them in 512k units
 */
-#define OT_SCRAPE_CHUNK_SIZE (512*1024)
+#define OT_SCRAPE_CHUNK_SIZE  (1024 * 1024)
 
 /* "d8:completei%zde10:downloadedi%zde10:incompletei%zdee" */
 #define OT_SCRAPE_MAXENTRYLEN 256
 
-#ifdef WANT_COMPRESSION_GZIP
-#define IF_COMPRESSION( TASK ) if( mode & TASK_FLAG_GZIP ) TASK
-#define WANT_COMPRESSION_GZIP_PARAM( param1, param2, param3 ) , param1, param2, param3
-#else
-#define IF_COMPRESSION( TASK )
-#define WANT_COMPRESSION_GZIP_PARAM( param1, param2, param3 )
-#endif
-
 /* Forward declaration */
-static void fullscrape_make( int *iovec_entries, struct iovec **iovector, ot_tasktype mode );
+static void fullscrape_make(int taskid, ot_tasktype mode);
+#ifdef WANT_COMPRESSION_GZIP
+static void fullscrape_make_gzip(int taskid, ot_tasktype mode);
+#endif
+#ifdef WANT_COMPRESSION_ZSTD
+static void fullscrape_make_zstd(int taskid, ot_tasktype mode);
+#endif
 
 /* Converter function from memory to human readable hex strings
    XXX - Duplicated from ot_stats. Needs fix. */
-static char*to_hex(char*d,uint8_t*s){char*m="0123456789ABCDEF";char *t=d;char*e=d+40;while(d<e){*d++=m[*s>>4];*d++=m[*s++&15];}*d=0;return t;}
+static char *to_hex(char *d, uint8_t *s) {
+  char *m = "0123456789ABCDEF";
+  char *t = d;
+  char *e = d + 40;
+  while (d < e) {
+    *d++ = m[*s >> 4];
+    *d++ = m[*s++ & 15];
+  }
+  *d = 0;
+  return t;
+}
 
 /* This is the entry point into this worker thread
    It grabs tasks from mutex_tasklist and delivers results back
 */
-static void * fullscrape_worker( void * args ) {
-  int iovec_entries;
-  struct iovec *iovector;
+static void *fullscrape_worker(void *args) {
+  (void)args;
 
-  args = args;
-
-  while( 1 ) {
+  while (g_opentracker_running) {
     ot_tasktype tasktype = TASK_FULLSCRAPE;
-    ot_taskid   taskid   = mutex_workqueue_poptask( &tasktype );
-    fullscrape_make( &iovec_entries, &iovector, tasktype );
-    if( mutex_workqueue_pushresult( taskid, iovec_entries, iovector ) )
-      iovec_free( &iovec_entries, &iovector );
+    ot_taskid   taskid   = mutex_workqueue_poptask(&tasktype);
+#ifdef WANT_COMPRESSION_ZSTD
+    if (tasktype & TASK_FLAG_ZSTD)
+      fullscrape_make_zstd(taskid, tasktype);
+    else
+#endif
+#ifdef WANT_COMPRESSION_GZIP
+    if (tasktype & TASK_FLAG_GZIP)
+      fullscrape_make_gzip(taskid, tasktype);
+    else
+#endif
+      fullscrape_make(taskid, tasktype);
+    mutex_workqueue_pushchunked(taskid, NULL);
   }
   return NULL;
 }
@@ -79,155 +96,362 @@ void fullscrape_deinit( ) {
   pthread_cancel( thread_id );
 }
 
-void fullscrape_deliver( int64 socket, ot_tasktype tasktype ) {
-  mutex_workqueue_pushtask( socket, tasktype );
+void fullscrape_deliver( int64 sock, ot_tasktype tasktype ) {
+  mutex_workqueue_pushtask( sock, tasktype );
 }
 
-static int fullscrape_increase( int *iovec_entries, struct iovec **iovector,
-                         char **r, char **re  WANT_COMPRESSION_GZIP_PARAM( z_stream *strm, ot_tasktype mode, int zaction ) ) {
-  /* Allocate a fresh output buffer at the end of our buffers list */
-  if( !( *r = iovec_fix_increase_or_free( iovec_entries, iovector, *r, OT_SCRAPE_CHUNK_SIZE ) ) ) {
+static char * fullscrape_write_one( ot_tasktype mode, char *r, ot_torrent *torrent, ot_hash *hash ) {
+  size_t seed_count = torrent->peer_list6->seed_count + torrent->peer_list4->seed_count;
+  size_t peer_count = torrent->peer_list6->peer_count + torrent->peer_list4->peer_count;
+  size_t down_count = torrent->peer_list6->down_count + torrent->peer_list4->down_count;
 
-    /* Deallocate gzip buffers */
-    IF_COMPRESSION( deflateEnd(strm); )
+  switch (mode & TASK_TASK_MASK) {
+  case TASK_FULLSCRAPE:
+  default:
+    /* push hash as bencoded string */
+    *r++ = '2';
+    *r++ = '0';
+    *r++ = ':';
+    memcpy(r, hash, sizeof(ot_hash));
+    r += sizeof(ot_hash);
+    /* push rest of the scrape string */
+    r += sprintf(r, "d8:completei%zde10:downloadedi%zde10:incompletei%zdee", seed_count, down_count, peer_count - seed_count);
 
-    /* Release lock on current bucket and return */
-    return -1;
+    break;
+  case TASK_FULLSCRAPE_TPB_ASCII:
+    to_hex(r, *hash);
+    r += 2 * sizeof(ot_hash);
+    r += sprintf(r, ":%zd:%zd\n", seed_count, peer_count - seed_count);
+    break;
+  case TASK_FULLSCRAPE_TPB_ASCII_PLUS:
+    to_hex(r, *hash);
+    r += 2 * sizeof(ot_hash);
+    r += sprintf(r, ":%zd:%zd:%zd\n", seed_count, peer_count - seed_count, down_count);
+    break;
+  case TASK_FULLSCRAPE_TPB_BINARY:
+    memcpy(r, *hash, sizeof(ot_hash));
+    r                    += sizeof(ot_hash);
+    *(uint32_t *)(r + 0)  = htonl((uint32_t)seed_count);
+    *(uint32_t *)(r + 4)  = htonl((uint32_t)(peer_count - seed_count));
+    r                    += 8;
+    break;
+  case TASK_FULLSCRAPE_TPB_URLENCODED:
+    r += fmt_urlencoded(r, (char *)*hash, 20);
+    r += sprintf(r, ":%zd:%zd\n", seed_count, peer_count - seed_count);
+    break;
+  case TASK_FULLSCRAPE_TRACKERSTATE:
+    to_hex(r, *hash);
+    r += 2 * sizeof(ot_hash);
+    r += sprintf(r, ":%zd:%zd\n", torrent->peer_list6->base, down_count);
+    break;
   }
-
-  /* Adjust new end of output buffer */
-  *re = *r + OT_SCRAPE_CHUNK_SIZE - OT_SCRAPE_MAXENTRYLEN;
-
-  /* When compressing, we have all the bytes in output buffer */
-#ifdef WANT_COMPRESSION_GZIP
-  if( mode & TASK_FLAG_GZIP ) {
-    *re -= OT_SCRAPE_MAXENTRYLEN;
-    strm->next_out  = (uint8_t*)*r;
-    strm->avail_out = OT_SCRAPE_CHUNK_SIZE;
-    if( deflate( strm, zaction ) < Z_OK )
-      fprintf( stderr, "deflate() failed while in fullscrape_increase(%d).\n", zaction );
-    *r = (char*)strm->next_out;
-  }
-#endif
-
-  return 0;
+  return r;
 }
 
-static void fullscrape_make( int *iovec_entries, struct iovec **iovector, ot_tasktype mode ) {
-  int      bucket;
-  char    *r, *re;
-#ifdef WANT_COMPRESSION_GZIP
-  char     compress_buffer[OT_SCRAPE_MAXENTRYLEN];
-  z_stream strm;
-#endif
+static void fullscrape_make(int taskid, ot_tasktype mode) {
+  int          bucket;
+  char        *r, *re;
+  struct iovec iovector = {NULL, 0};
 
   /* Setup return vector... */
-  *iovec_entries = 0;
-  *iovector = NULL;
-  if( !( r = iovec_increase( iovec_entries, iovector, OT_SCRAPE_CHUNK_SIZE ) ) )
+  r = iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE);
+  if (!r)
     return;
 
   /* re points to low watermark */
   re = r + OT_SCRAPE_CHUNK_SIZE - OT_SCRAPE_MAXENTRYLEN;
 
-#ifdef WANT_COMPRESSION_GZIP
-  if( mode & TASK_FLAG_GZIP ) {
-    re += OT_SCRAPE_MAXENTRYLEN;
-    byte_zero( &strm, sizeof(strm) );
-    strm.next_in   = (uint8_t*)compress_buffer;
-    strm.next_out  = (uint8_t*)r;
-    strm.avail_out = OT_SCRAPE_CHUNK_SIZE;
-    if( deflateInit2(&strm,7,Z_DEFLATED,31,8,Z_DEFAULT_STRATEGY) != Z_OK )
-      fprintf( stderr, "not ok.\n" );
-    r = compress_buffer;
-  }
-#endif
-
-  if( ( mode & TASK_TASK_MASK ) == TASK_FULLSCRAPE )
-    r += sprintf( r, "d5:filesd" );
+  if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE)
+    r += sprintf(r, "d5:filesd");
 
   /* For each bucket... */
-  for( bucket=0; bucket<OT_BUCKET_COUNT; ++bucket ) {
+  for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) {
     /* Get exclusive access to that bucket */
-    ot_vector *torrents_list = mutex_bucket_lock( bucket );
-    size_t tor_offset;
+    ot_vector  *torrents_list = mutex_bucket_lock(bucket);
+    ot_torrent *torrents      = (ot_torrent *)(torrents_list->data);
+    size_t      i;
 
     /* For each torrent in this bucket.. */
-    for( tor_offset=0; tor_offset<torrents_list->size; ++tor_offset ) {
-      /* Address torrents members */
-      ot_peerlist *peer_list = ( ((ot_torrent*)(torrents_list->data))[tor_offset] ).peer_list;
-      ot_hash     *hash      =&( ((ot_torrent*)(torrents_list->data))[tor_offset] ).hash;
+    for (i = 0; i < torrents_list->size; ++i) {
+      r = fullscrape_write_one(mode, r, torrents + i, &torrents[i].hash);
 
-      switch( mode & TASK_TASK_MASK ) {
-      case TASK_FULLSCRAPE:
-      default:
+      if (r > re) {
+        iovector.iov_len = r - (char *)iovector.iov_base;
 
-        /* push hash as bencoded string */
-        *r++='2'; *r++='0'; *r++=':';
-        memmove( r, hash, 20 ); r+=20;
+        if (mutex_workqueue_pushchunked(taskid, &iovector)) {
+          free(iovector.iov_base);
+          return mutex_bucket_unlock(bucket, 0);
+        }
+        /* Allocate a fresh output buffer */
+        r = iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE);
+        if (!r)
+          return mutex_bucket_unlock(bucket, 0);
 
-        /* push rest of the scrape string */
-        r += sprintf( r, "d8:completei%zde10:downloadedi%zde10:incompletei%zdee", peer_list->seed_count, peer_list->down_count, peer_list->peer_count-peer_list->seed_count );
-
-         break;
-      case TASK_FULLSCRAPE_TPB_ASCII:
-        to_hex( r, *hash ); r+=40;
-        r += sprintf( r, ":%zd:%zd\n", peer_list->seed_count, peer_list->peer_count-peer_list->seed_count );
-        break;
-      case TASK_FULLSCRAPE_TPB_BINARY:
-        memmove( r, hash, 20 ); r+=20;
-        *(uint32_t*)r++ = htonl( (uint32_t)peer_list->seed_count );
-        *(uint32_t*)r++ = htonl( (uint32_t)( peer_list->peer_count-peer_list->seed_count) );
-        break;
-      case TASK_FULLSCRAPE_TPB_URLENCODED:
-        r += fmt_urlencoded( r, (char *)*hash, 20 );
-        r += sprintf( r, ":%zd:%zd\n", peer_list->seed_count, peer_list->peer_count-peer_list->seed_count );
-        break;
+        /* re points to low watermark */
+        re = r + OT_SCRAPE_CHUNK_SIZE - OT_SCRAPE_MAXENTRYLEN;
       }
-
-#ifdef WANT_COMPRESSION_GZIP
-     if( mode & TASK_FLAG_GZIP ) {
-        strm.next_in  = (uint8_t*)compress_buffer;
-        strm.avail_in = r - compress_buffer;
-        if( deflate( &strm, Z_NO_FLUSH ) < Z_OK )
-          fprintf( stderr, "deflate() failed while in fullscrape_make().\n" );
-        r = (char*)strm.next_out;
-      }
-#endif
-
-      /* Check if there still is enough buffer left */
-      while( r >= re )
-       if( fullscrape_increase( iovec_entries, iovector, &r, &re WANT_COMPRESSION_GZIP_PARAM( &strm, mode, Z_NO_FLUSH ) ) )
-         return mutex_bucket_unlock( bucket );
-
-      IF_COMPRESSION( r = compress_buffer; )
     }
 
-    /* All torrents done: release lock on currenct bucket */
-    mutex_bucket_unlock( bucket );
+    /* All torrents done: release lock on current bucket */
+    mutex_bucket_unlock(bucket, 0);
+
+    /* Parent thread died? */
+    if (!g_opentracker_running)
+      return;
   }
 
-  if( ( mode & TASK_TASK_MASK ) == TASK_FULLSCRAPE )
-    r += sprintf( r, "ee" );
+  if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE)
+    r += sprintf(r, "ee");
+
+  /* Send rest of data */
+  iovector.iov_len = r - (char *)iovector.iov_base;
+  if (mutex_workqueue_pushchunked(taskid, &iovector))
+    free(iovector.iov_base);
+}
 
 #ifdef WANT_COMPRESSION_GZIP
-  if( mode & TASK_FLAG_GZIP ) {
-    strm.next_in  = (uint8_t*)compress_buffer;
-    strm.avail_in = r - compress_buffer;
-    if( deflate( &strm, Z_FINISH ) < Z_OK )
-      fprintf( stderr, "deflate() failed while in fullscrape_make()'s endgame.\n" );
-    r = (char*)strm.next_out;
 
-    while( r >= re )
-      if( fullscrape_increase( iovec_entries, iovector, &r, &re WANT_COMPRESSION_GZIP_PARAM( &strm, mode, Z_FINISH ) ) )
-        return mutex_bucket_unlock( bucket );
-    deflateEnd(&strm);
+static void fullscrape_make_gzip(int taskid, ot_tasktype mode) {
+  int          bucket;
+  char        *r;
+  struct iovec iovector = {NULL, 0};
+  int          zres;
+  z_stream     strm;
+  /* Setup return vector... */
+  iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE);
+  if (!iovector.iov_base)
+    return;
+
+  byte_zero(&strm, sizeof(strm));
+  strm.next_out  = (uint8_t *)iovector.iov_base;
+  strm.avail_out = OT_SCRAPE_CHUNK_SIZE;
+  if (deflateInit2(&strm, 7, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY) != Z_OK)
+    fprintf(stderr, "not ok.\n");
+
+  if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) {
+    strm.next_in  = (uint8_t *)"d5:filesd";
+    strm.avail_in = strlen("d5:filesd");
+    zres          = deflate(&strm, Z_NO_FLUSH);
   }
-#endif
 
-  /* Release unused memory in current output buffer */
-  iovec_fixlast( iovec_entries, iovector, r );
+  /* For each bucket... */
+  for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) {
+    /* Get exclusive access to that bucket */
+    ot_vector  *torrents_list = mutex_bucket_lock(bucket);
+    ot_torrent *torrents      = (ot_torrent *)(torrents_list->data);
+    size_t      i;
+
+    /* For each torrent in this bucket.. */
+    for (i = 0; i < torrents_list->size; ++i) {
+      char compress_buffer[OT_SCRAPE_MAXENTRYLEN];
+      r             = fullscrape_write_one(mode, compress_buffer, torrents + i, &torrents[i].hash);
+      strm.next_in  = (uint8_t *)compress_buffer;
+      strm.avail_in = r - compress_buffer;
+      zres          = deflate(&strm, Z_NO_FLUSH);
+      if ((zres < Z_OK) && (zres != Z_BUF_ERROR))
+        fprintf(stderr, "deflate() failed while in fullscrape_make().\n");
+
+      /* Check if there still is enough buffer left */
+      while (!strm.avail_out) {
+        iovector.iov_len = (char *)strm.next_out - (char *)iovector.iov_base;
+
+        if (mutex_workqueue_pushchunked(taskid, &iovector)) {
+          free(iovector.iov_base);
+          return mutex_bucket_unlock(bucket, 0);
+        }
+        /* Allocate a fresh output buffer */
+        iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE);
+        if (!iovector.iov_base) {
+          fprintf(stderr, "Out of memory trying to claim ouput buffer\n");
+          deflateEnd(&strm);
+          return mutex_bucket_unlock(bucket, 0);
+        }
+        strm.next_out  = (uint8_t *)iovector.iov_base;
+        strm.avail_out = OT_SCRAPE_CHUNK_SIZE;
+        zres           = deflate(&strm, Z_NO_FLUSH);
+        if ((zres < Z_OK) && (zres != Z_BUF_ERROR))
+          fprintf(stderr, "deflate() failed while in fullscrape_make().\n");
+      }
+    }
+
+    /* All torrents done: release lock on current bucket */
+    mutex_bucket_unlock(bucket, 0);
+
+    /* Parent thread died? */
+    if (!g_opentracker_running) {
+      deflateEnd(&strm);
+      return;
+    }
+  }
+
+  if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) {
+    strm.next_in  = (uint8_t *)"ee";
+    strm.avail_in = strlen("ee");
+  }
+
+  if (deflate(&strm, Z_FINISH) < Z_OK)
+    fprintf(stderr, "deflate() failed while in fullscrape_make()'s endgame.\n");
+
+  iovector.iov_len = (char *)strm.next_out - (char *)iovector.iov_base;
+  if (mutex_workqueue_pushchunked(taskid, &iovector)) {
+    free(iovector.iov_base);
+    deflateEnd(&strm);
+    return;
+  }
+
+  /* Check if there's a last batch of data in the zlib buffer */
+  if (!strm.avail_out) {
+      /* Allocate a fresh output buffer */
+      iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE);
+
+      if (!iovector.iov_base) {
+        fprintf(stderr, "Problem with iovec_fix_increase_or_free\n");
+        deflateEnd(&strm);
+        return;
+      }
+      strm.next_out  = iovector.iov_base;
+      strm.avail_out = OT_SCRAPE_CHUNK_SIZE;
+      if (deflate(&strm, Z_FINISH) < Z_OK)
+        fprintf(stderr, "deflate() failed while in fullscrape_make()'s endgame.\n");
+
+      /* Only pass the new buffer if there actually was some data left in the buffer */
+      iovector.iov_len = (char *)strm.next_out - (char *)iovector.iov_base;
+      if (!iovector.iov_len || mutex_workqueue_pushchunked(taskid, &iovector))
+        free(iovector.iov_base);
+  }
+
+  deflateEnd(&strm);
 }
+/* WANT_COMPRESSION_GZIP */
 #endif
 
-const char *g_version_fullscrape_c = "$Source$: $Revision$\n";
+#ifdef WANT_COMPRESSION_ZSTD
+
+static void fullscrape_make_zstd(int taskid, ot_tasktype mode) {
+  int            bucket;
+  char          *r;
+  struct iovec   iovector = {NULL, 0};
+  ZSTD_CCtx     *zstream = ZSTD_createCCtx();
+  ZSTD_inBuffer  inbuf;
+  ZSTD_outBuffer outbuf;
+  size_t         more_bytes;
+
+  if (!zstream)
+    return;
+
+  /* Setup return vector... */
+  iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE);
+  if (!iovector.iov_base) {
+    ZSTD_freeCCtx(zstream);
+    return;
+  }
+
+  /* Working with a compression level 6 is half as fast as level 3, but
+     seems to be the last reasonable bump that's worth extra cpu */
+  ZSTD_CCtx_setParameter(zstream, ZSTD_c_compressionLevel, 6);
+
+  outbuf.dst  = iovector.iov_base;
+  outbuf.size = OT_SCRAPE_CHUNK_SIZE;
+  outbuf.pos  = 0;
+
+  if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) {
+    inbuf.src  = (const void *)"d5:filesd";
+    inbuf.size = strlen("d5:filesd");
+    inbuf.pos  = 0;
+    ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_continue);
+  }
+
+  /* For each bucket... */
+  for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) {
+    /* Get exclusive access to that bucket */
+    ot_vector  *torrents_list = mutex_bucket_lock(bucket);
+    ot_torrent *torrents      = (ot_torrent *)(torrents_list->data);
+    size_t      i;
+
+    /* For each torrent in this bucket.. */
+    for (i = 0; i < torrents_list->size; ++i) {
+      char compress_buffer[OT_SCRAPE_MAXENTRYLEN];
+      r             = fullscrape_write_one(mode, compress_buffer, torrents + i, &torrents[i].hash);
+      inbuf.src     = compress_buffer;
+      inbuf.size    = r - compress_buffer;
+      inbuf.pos     = 0;
+      ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_continue);
+
+      /* Check if there still is enough buffer left */
+      while (outbuf.pos + OT_SCRAPE_MAXENTRYLEN > outbuf.size) {
+        iovector.iov_len = outbuf.size;
+
+        if (mutex_workqueue_pushchunked(taskid, &iovector)) {
+          free(iovector.iov_base);
+          ZSTD_freeCCtx(zstream);
+          return mutex_bucket_unlock(bucket, 0);
+        }
+        /* Allocate a fresh output buffer */
+        iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE);
+        if (!iovector.iov_base) {
+          fprintf(stderr, "Out of memory trying to claim ouput buffer\n");
+          ZSTD_freeCCtx(zstream);
+          return mutex_bucket_unlock(bucket, 0);
+        }
+
+        outbuf.dst  = iovector.iov_base;
+        outbuf.size = OT_SCRAPE_CHUNK_SIZE;
+        outbuf.pos  = 0;
+
+        ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_continue);
+      }
+    }
+
+    /* All torrents done: release lock on current bucket */
+    mutex_bucket_unlock(bucket, 0);
+
+    /* Parent thread died? */
+    if (!g_opentracker_running)
+      return;
+  }
+
+  if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) {
+    inbuf.src  = (const void *)"ee";
+    inbuf.size = strlen("ee");
+    inbuf.pos  = 0;
+  }
+
+  more_bytes = ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_end);
+
+  iovector.iov_len = outbuf.pos;
+  if (mutex_workqueue_pushchunked(taskid, &iovector)) {
+    free(iovector.iov_base);
+    ZSTD_freeCCtx(zstream);
+    return;
+  }
+
+  /* Check if there's a last batch of data in the zlib buffer */
+  if (more_bytes) {
+      /* Allocate a fresh output buffer */
+      iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE);
+
+      if (!iovector.iov_base) {
+        fprintf(stderr, "Problem with iovec_fix_increase_or_free\n");
+        ZSTD_freeCCtx(zstream);
+        return;
+      }
+
+      outbuf.dst  = iovector.iov_base;
+      outbuf.size = OT_SCRAPE_CHUNK_SIZE;
+      outbuf.pos  = 0;
+
+      ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_end);
+
+      /* Only pass the new buffer if there actually was some data left in the buffer */
+      iovector.iov_len = outbuf.pos;
+      if (!iovector.iov_len || mutex_workqueue_pushchunked(taskid, &iovector))
+        free(iovector.iov_base);
+  }
+
+  ZSTD_freeCCtx(zstream);
+}
+/* WANT_COMPRESSION_ZSTD */
+#endif
+
+/* WANT_FULLSCRAPE */
+#endif

ot_fullscrape.h

@@ -3,14 +3,16 @@
 
    $id$ */
 
-#ifndef __OT_FULLSCRAPE_H__
-#define __OT_FULLSCRAPE_H__
+#ifndef OT_FULLSCRAPE_H__
+#define OT_FULLSCRAPE_H__
 
 #ifdef WANT_FULLSCRAPE
 
-void fullscrape_init( );
-void fullscrape_deinit( );
-void fullscrape_deliver( int64 socket, ot_tasktype tasktype );
+#include "ot_mutex.h"
+
+void fullscrape_init();
+void fullscrape_deinit();
+void fullscrape_deliver(int64 sock, ot_tasktype tasktype);
 
 #else
 

ot_http.h

@@ -3,28 +3,31 @@
 
    $id$ */
 
-#ifndef __OT_HTTP_H__
-#define __OT_HTTP_H__
+#ifndef OT_HTTP_H__
+#define OT_HTTP_H__
 
 typedef enum {
-  STRUCT_HTTP_FLAG_ARRAY_USED     = 1,
-  STRUCT_HTTP_FLAG_IOB_USED       = 2,
-  STRUCT_HTTP_FLAG_WAITINGFORTASK = 4,
-  STRUCT_HTTP_FLAG_GZIP           = 8,
-  STRUCT_HTTP_FLAG_BZIP2          = 16
+  STRUCT_HTTP_FLAG_WAITINGFORTASK      = 1,
+  STRUCT_HTTP_FLAG_GZIP                = 2,
+  STRUCT_HTTP_FLAG_BZIP2               = 4,
+  STRUCT_HTTP_FLAG_ZSTD                = 8,
+  STRUCT_HTTP_FLAG_CHUNKED             = 16,
+  STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER = 32
 } STRUCT_HTTP_FLAG;
 
 struct http_data {
-  union {
-    array          request;
-    io_batch       batch;
-  };
-  char             ip[4];
+  array            request;
+  io_batch        *batch;
+  size_t           batches;
+  ot_ip6           ip;
   STRUCT_HTTP_FLAG flag;
 };
 
-ssize_t http_handle_request( const int64 s, char *data, size_t l );
-ssize_t http_sendiovecdata( const int64 s, int iovec_entries, struct iovec *iovector );
-ssize_t http_issue_error( const int64 s, int code );
+ssize_t        http_handle_request(const int64 s, struct ot_workstruct *ws);
+ssize_t        http_sendiovecdata(const int64 s, struct ot_workstruct *ws, int iovec_entries, struct iovec *iovector, int is_partial);
+ssize_t        http_issue_error(const int64 s, struct ot_workstruct *ws, int code);
+
+extern char   *g_stats_path;
+extern ssize_t g_stats_path_len;
 
 #endif

ot_iovec.c

@@ -4,73 +4,89 @@
    $id$ */
 
 /* System */
-#include <sys/types.h>
-#include <sys/mman.h>
 #include <stdlib.h>
-#include <unistd.h>
+#include <sys/types.h>
 #include <sys/uio.h>
+#include <unistd.h>
 
 /* Libowfat */
 
 /* Opentracker */
 #include "ot_iovec.h"
 
-void *iovec_increase( int *iovec_entries, struct iovec **iovector, size_t new_alloc ) {
-  void *new_ptr = realloc( *iovector, (1 + *iovec_entries ) * sizeof( struct iovec ) );
-  if( !new_ptr )
+void *iovec_increase(int *iovec_entries, struct iovec **iovector, size_t new_alloc) {
+  void         *new_data;
+  int           new_entries = 1 + *iovec_entries;
+  struct iovec *new_vec     = realloc(*iovector, new_entries * sizeof(struct iovec));
+
+  if (!new_vec)
     return NULL;
-  *iovector = new_ptr;
-  new_ptr = mmap( NULL, new_alloc, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0 );
-  if( !new_ptr )
+
+  /* Only allocate after we have a place to store the pointer */
+  new_data = malloc(new_alloc);
+  if (!new_data)
     return NULL;
-  ((*iovector)[*iovec_entries]).iov_base = new_ptr;
-  ((*iovector)[*iovec_entries]).iov_len  = new_alloc;
+
+  new_vec[new_entries - 1].iov_base = new_data;
+  new_vec[new_entries - 1].iov_len  = new_alloc;
+
+  *iovector                         = new_vec;
   ++*iovec_entries;
-  return new_ptr;
+  return new_data;
 }
 
-void iovec_free( int *iovec_entries, struct iovec **iovector ) {
+void *iovec_append(int *iovec_entries, struct iovec **iovector, struct iovec *append_iovector) {
+  int           new_entries = *iovec_entries + 1;
+  struct iovec *new_vec     = realloc(*iovector, new_entries * sizeof(struct iovec));
+  if (!new_vec)
+    return NULL;
+
+  /* Take over data from appended iovec */
+  new_vec[*iovec_entries].iov_base = append_iovector->iov_base;
+  new_vec[*iovec_entries].iov_len  = append_iovector->iov_len;
+
+  append_iovector->iov_base        = NULL;
+  append_iovector->iov_len         = 0;
+
+  *iovector                        = new_vec;
+  *iovec_entries                   = new_entries;
+
+  return new_vec;
+}
+
+void iovec_free(int *iovec_entries, struct iovec **iovector) {
   int i;
-  for( i=0; i<*iovec_entries; ++i )
-    munmap( ((*iovector)[i]).iov_base, ((*iovector)[i]).iov_len );
+  for (i = 0; i < *iovec_entries; ++i)
+    free(((*iovector)[i]).iov_base);
+  *iovector      = NULL;
   *iovec_entries = 0;
 }
 
-void  iovec_fixlast( int *iovec_entries, struct iovec **iovector, void *last_ptr ) {
-  int page_size = getpagesize();
-  size_t old_alloc, new_alloc, old_pages, new_pages;
-  char * base = (char*)((*iovector)[ *iovec_entries - 1 ]).iov_base;
+void iovec_fixlast(int *iovec_entries, struct iovec **iovector, void *last_ptr) {
+  if (*iovec_entries) {
+    char  *base      = (char *)((*iovector)[*iovec_entries - 1]).iov_base;
+    size_t new_alloc = ((char *)last_ptr) - base;
 
-  if( !*iovec_entries ) return;
-
-  old_alloc = ((*iovector)[ *iovec_entries - 1 ]).iov_len;
-  new_alloc = ((char*)last_ptr) - base;
-  old_pages = 1 + old_alloc / page_size;
-  new_pages = 1 + new_alloc / page_size;
-
-  if( old_pages != new_pages )
-    munmap( base + new_pages * page_size, old_alloc - new_pages * page_size );
-  ((*iovector)[*iovec_entries - 1 ]).iov_len = new_alloc;
+    ((*iovector)[*iovec_entries - 1]).iov_base = realloc(base, new_alloc);
+    ((*iovector)[*iovec_entries - 1]).iov_len  = new_alloc;
+  }
 }
 
-void  *iovec_fix_increase_or_free( int *iovec_entries, struct iovec **iovector, void *last_ptr, size_t new_alloc ) {
-  void *new_ptr;
+void *iovec_fix_increase_or_free(int *iovec_entries, struct iovec **iovector, void *last_ptr, size_t new_alloc) {
+  void *new_data;
 
-  iovec_fixlast( iovec_entries, iovector, last_ptr );
+  iovec_fixlast(iovec_entries, iovector, last_ptr);
 
-  if( !( new_ptr = iovec_increase( iovec_entries, iovector, new_alloc ) ) )
-    iovec_free( iovec_entries, iovector );
+  if (!(new_data = iovec_increase(iovec_entries, iovector, new_alloc)))
+    iovec_free(iovec_entries, iovector);
 
-  return new_ptr;
+  return new_data;
 }
 
-
-size_t iovec_length( int *iovec_entries, struct iovec **iovector ) {
+size_t iovec_length(const int *iovec_entries, const struct iovec **iovector) {
   size_t length = 0;
-  int i;
-  for( i=0; i<*iovec_entries; ++i )
+  int    i;
+  for (i = 0; i < *iovec_entries; ++i)
     length += ((*iovector)[i]).iov_len;
   return length;
 }
-
-const char *g_version_iovec_c = "$Source$: $Revision$\n";

ot_iovec.h

@@ -3,15 +3,18 @@
 
    $id$ */
 
-#ifndef __OT_IOVEC_H__
-#define __OT_IOVEC_H__
+#ifndef OT_IOVEC_H__
+#define OT_IOVEC_H__
 
-void  *iovec_increase( int *iovec_entries, struct iovec **iovector, size_t new_alloc );
-void   iovec_fixlast( int *iovec_entries, struct iovec **iovector, void *last_ptr );
-void   iovec_free( int *iovec_entries, struct iovec **iovector );
+#include <sys/uio.h>
 
-size_t iovec_length( int *iovec_entries, struct iovec **iovector );
+void  *iovec_increase(int *iovec_entries, struct iovec **iovector, size_t new_alloc);
+void  *iovec_append(int *iovec_entries, struct iovec **iovector, struct iovec *append_iovector);
+void   iovec_fixlast(int *iovec_entries, struct iovec **iovector, void *last_ptr);
+void   iovec_free(int *iovec_entries, struct iovec **iovector);
 
-void  *iovec_fix_increase_or_free( int *iovec_entries, struct iovec **iovector, void *last_ptr, size_t new_alloc );
+size_t iovec_length(const int *iovec_entries, const struct iovec **iovector);
+
+void  *iovec_fix_increase_or_free(int *iovec_entries, struct iovec **iovector, void *last_ptr, size_t new_alloc);
 
 #endif

ot_livesync.c

@@ -4,159 +4,236 @@
  $id$ */
 
 /* System */
+#include <pthread.h>
+#include <stdlib.h>
+#include <string.h>
 #include <sys/types.h>
 #include <sys/uio.h>
-#include <string.h>
-#include <pthread.h>
+#include <unistd.h>
 
 /* Libowfat */
-#include "socket.h"
+#include "byte.h"
+#include "ip6.h"
 #include "ndelay.h"
+#include "socket.h"
 
 /* Opentracker */
-#include "trackerlogic.h"
-#include "ot_livesync.h"
 #include "ot_accesslist.h"
+#include "ot_livesync.h"
+#include "ot_mutex.h"
+#include "ot_stats.h"
+#include "trackerlogic.h"
 
 #ifdef WANT_SYNC_LIVE
 
-char groupip_1[4] = { 224,0,23,42 };
+char groupip_1[4] = {224, 0, 23, 5};
 
-#define LIVESYNC_BUFFINSIZE (256*256)
-#define LIVESYNC_BUFFSIZE  1504
-#define LIVESYNC_BUFFWATER (sizeof(ot_peer)+sizeof(ot_hash))
+#define LIVESYNC_INCOMING_BUFFSIZE        (256 * 256)
 
-#define LIVESYNC_MAXDELAY  15
+#define LIVESYNC_OUTGOING_BUFFSIZE_PEERS  1480
+#define LIVESYNC_OUTGOING_WATERMARK_PEERS (sizeof(ot_peer) + sizeof(ot_hash))
+
+#define LIVESYNC_MAXDELAY                 15 /* seconds */
+
+enum { OT_SYNC_PEER4, OT_SYNC_PEER6 };
 
 /* Forward declaration */
-static void * livesync_worker( void * args );
+static void           *livesync_worker(void *args);
 
 /* For outgoing packets */
-static int64 g_livesync_socket_in = -1;
+static int64           g_socket_in    = -1;
 
 /* For incoming packets */
-static int64 g_livesync_socket_out = -1;
+static int64           g_socket_out   = -1;
 
-static uint8_t  livesync_inbuffer[LIVESYNC_BUFFINSIZE];
-static uint8_t  livesync_outbuffer_start[ LIVESYNC_BUFFSIZE ];
-static uint8_t *livesync_outbuffer_pos;
-static uint8_t *livesync_outbuffer_highwater = livesync_outbuffer_start + LIVESYNC_BUFFSIZE - LIVESYNC_BUFFWATER;
-static ot_time  livesync_lastpacket_time;
+static pthread_mutex_t g_outbuf_mutex = PTHREAD_MUTEX_INITIALIZER;
+typedef struct {
+  uint8_t data[LIVESYNC_OUTGOING_BUFFSIZE_PEERS];
+  size_t  fill;
+  ot_time next_packet_time;
+} sync_buffer;
 
-static pthread_t thread_id;
-void livesync_init( ) {
-  if( g_livesync_socket_in == -1 )
-    exerr( "No socket address for live sync specified." );
-  livesync_outbuffer_pos = livesync_outbuffer_start;
-  memmove( livesync_outbuffer_pos, &g_tracker_id, sizeof( g_tracker_id ) );
-  livesync_outbuffer_pos += sizeof( g_tracker_id );
-  livesync_lastpacket_time = g_now;
+static sync_buffer g_v6_buf;
+static sync_buffer g_v4_buf;
 
-  pthread_create( &thread_id, NULL, livesync_worker, NULL );
+static pthread_t   thread_id;
+void livesync_init() {
+
+  if (g_socket_in == -1)
+    exerr("No socket address for live sync specified.");
+
+  /* Prepare outgoing peers buffer */
+  memcpy(g_v6_buf.data, &g_tracker_id, sizeof(g_tracker_id));
+  memcpy(g_v4_buf.data, &g_tracker_id, sizeof(g_tracker_id));
+
+  uint32_pack_big((char *)g_v6_buf.data + sizeof(g_tracker_id), OT_SYNC_PEER6);
+  uint32_pack_big((char *)g_v4_buf.data + sizeof(g_tracker_id), OT_SYNC_PEER4);
+
+  g_v6_buf.fill             = sizeof(g_tracker_id) + sizeof(uint32_t);
+  g_v4_buf.fill             = sizeof(g_tracker_id) + sizeof(uint32_t);
+
+  g_v6_buf.next_packet_time = g_now_seconds + LIVESYNC_MAXDELAY;
+  g_v4_buf.next_packet_time = g_now_seconds + LIVESYNC_MAXDELAY;
+
+  pthread_create(&thread_id, NULL, livesync_worker, NULL);
 }
 
 void livesync_deinit() {
-  pthread_cancel( thread_id );
+  if (g_socket_in != -1)
+    close(g_socket_in);
+  if (g_socket_out != -1)
+    close(g_socket_out);
+
+  pthread_cancel(thread_id);
 }
 
-void livesync_bind_mcast( char *ip, uint16_t port) {
-  char tmpip[4] = {0,0,0,0};
+void livesync_bind_mcast(ot_ip6 ip, uint16_t port) {
+  char  tmpip[4] = {0, 0, 0, 0};
+  char *v4ip;
 
-  if( g_livesync_socket_in != -1 )
+  if (!ip6_isv4mapped(ip))
+    exerr("v6 mcast support not yet available.");
+  v4ip = ip + 12;
+
+  if (g_socket_in != -1)
     exerr("Error: Livesync listen ip specified twice.");
 
-  if( ( g_livesync_socket_in = socket_udp4( )) < 0)
-    exerr("Error: Cant create live sync incoming socket." );
-  ndelay_off(g_livesync_socket_in);
+  if ((g_socket_in = socket_udp4()) < 0)
+    exerr("Error: Cant create live sync incoming socket.");
+  ndelay_off(g_socket_in);
 
-  if( socket_bind4_reuse( g_livesync_socket_in, tmpip, port ) == -1 )
-    exerr("Error: Cant bind live sync incoming socket." );
+  if (socket_bind4_reuse(g_socket_in, tmpip, port) == -1)
+    exerr("Error: Cant bind live sync incoming socket.");
 
-  if( socket_mcjoin4( g_livesync_socket_in, groupip_1, ip ) )
+  if (socket_mcjoin4(g_socket_in, groupip_1, v4ip))
     exerr("Error: Cant make live sync incoming socket join mcast group.");
 
-  if( ( g_livesync_socket_out = socket_udp4()) < 0)
-    exerr("Error: Cant create live sync outgoing socket." );
-  if( socket_bind4_reuse( g_livesync_socket_out, ip, port ) == -1 )
-    exerr("Error: Cant bind live sync outgoing socket." );
+  if ((g_socket_out = socket_udp4()) < 0)
+    exerr("Error: Cant create live sync outgoing socket.");
+  if (socket_bind4_reuse(g_socket_out, v4ip, port) == -1)
+    exerr("Error: Cant bind live sync outgoing socket.");
 
-  socket_mcttl4(g_livesync_socket_out, 1);
-  socket_mcloop4(g_livesync_socket_out, 0);
+  socket_mcttl4(g_socket_out, 1);
+  socket_mcloop4(g_socket_out, 0);
 }
 
-static void livesync_issuepacket( ) {
-  socket_send4(g_livesync_socket_out, (char*)livesync_outbuffer_start, livesync_outbuffer_pos - livesync_outbuffer_start,
-               groupip_1, LIVESYNC_PORT);
-  livesync_outbuffer_pos = livesync_outbuffer_start + sizeof( g_tracker_id );
-  livesync_lastpacket_time = g_now;
+/* Caller MUST hold g_outbuf_mutex. Returns with g_outbuf_mutex unlocked */
+static void livesync_issue_peersync(sync_buffer *buf) {
+  char   mycopy[LIVESYNC_OUTGOING_BUFFSIZE_PEERS];
+  size_t fill = buf->fill;
+
+  memcpy(mycopy, buf->data, fill);
+  buf->fill             = sizeof(g_tracker_id) + sizeof(uint32_t);
+  buf->next_packet_time = g_now_seconds + LIVESYNC_MAXDELAY;
+
+  /* From now this thread has a local copy of the buffer and
+     has modified the protected element */
+  pthread_mutex_unlock(&g_outbuf_mutex);
+
+  socket_send4(g_socket_out, mycopy, fill, groupip_1, LIVESYNC_PORT);
 }
 
-/* Inform live sync about whats going on. */
-void livesync_tell( ot_hash * const info_hash, const ot_peer * const peer, const uint8_t peerflag ) {
-  memmove( livesync_outbuffer_pos                  , info_hash, sizeof(ot_hash));
-  memmove( livesync_outbuffer_pos + sizeof(ot_hash), peer,      sizeof(ot_peer));
-  OT_FLAG( livesync_outbuffer_pos + sizeof(ot_hash) ) |= peerflag;
+static void livesync_handle_peersync(struct ot_workstruct *ws, size_t peer_size) {
+  size_t off = sizeof(g_tracker_id) + sizeof(uint32_t);
 
-  livesync_outbuffer_pos += sizeof(ot_hash) + sizeof(ot_peer);
-  if( livesync_outbuffer_pos >= livesync_outbuffer_highwater )
-    livesync_issuepacket();
+  /* Now basic sanity checks have been done on the live sync packet
+     We might add more testing and logging. */
+  while ((ssize_t)(off + sizeof(ot_hash) + peer_size) <= ws->request_size) {
+    memcpy(&ws->peer, ws->request + off + sizeof(ot_hash), peer_size);
+    ws->hash = (ot_hash *)(ws->request + off);
+
+    if (!g_opentracker_running)
+      return;
+
+    if (OT_PEERFLAG(ws->peer) & PEER_FLAG_STOPPED)
+      remove_peer_from_torrent(FLAG_MCA, ws);
+    else
+      add_peer_to_torrent_and_return_peers(FLAG_MCA, ws, /* amount = */ 0);
+
+    off += sizeof(ot_hash) + peer_size;
+  }
+
+  stats_issue_event(EVENT_SYNC, 0, (ws->request_size - sizeof(g_tracker_id) - sizeof(uint32_t)) / ((ssize_t)sizeof(ot_hash) + peer_size));
 }
 
 /* Tickle the live sync module from time to time, so no events get
- stuck when there's not enough traffic to fill udp packets fast
- enough */
-void livesync_ticker( ) {
-  if( ( g_now - livesync_lastpacket_time > LIVESYNC_MAXDELAY) &&
-      ( livesync_outbuffer_pos > livesync_outbuffer_start + sizeof( g_tracker_id ) ) )
-    livesync_issuepacket();
+   stuck when there's not enough traffic to fill udp packets fast
+   enough */
+void livesync_ticker() {
+  /* livesync_issue_peersync sets g_next_packet_time */
+  pthread_mutex_lock(&g_outbuf_mutex);
+  if (g_now_seconds > g_v6_buf.next_packet_time && g_v6_buf.fill > sizeof(g_tracker_id) + sizeof(uint32_t))
+    livesync_issue_peersync(&g_v6_buf);
+  else
+    pthread_mutex_unlock(&g_outbuf_mutex);
+
+  pthread_mutex_lock(&g_outbuf_mutex);
+  if (g_now_seconds > g_v4_buf.next_packet_time && g_v4_buf.fill > sizeof(g_tracker_id) + sizeof(uint32_t))
+    livesync_issue_peersync(&g_v4_buf);
+  else
+    pthread_mutex_unlock(&g_outbuf_mutex);
 }
 
-static void * livesync_worker( void * args ) {
-  uint8_t in_ip[4]; uint16_t in_port;
-  ssize_t datalen;
-  int off;
+/* Inform live sync about whats going on. */
+void livesync_tell(struct ot_workstruct *ws) {
+  size_t       peer_size; /* initialized in next line */
+  ot_peer     *peer_src = peer_from_peer6(&ws->peer, &peer_size);
+  sync_buffer *dest_buf = peer_size == OT_PEER_SIZE6 ? &g_v6_buf : &g_v4_buf;
 
-  args = args;
+  pthread_mutex_lock(&g_outbuf_mutex);
 
-  while( 1 ) {
-    datalen = socket_recv4(g_livesync_socket_in, (char*)livesync_inbuffer, LIVESYNC_BUFFINSIZE, (char*)in_ip, &in_port);
-    off = 4;
+  memcpy(dest_buf->data + dest_buf->fill, ws->hash, sizeof(ot_hash));
+  dest_buf->fill += sizeof(ot_hash);
 
-    if( datalen <= 0 )
+  memcpy(dest_buf->data + dest_buf->fill, peer_src, peer_size);
+  dest_buf->fill += peer_size;
+
+  if (dest_buf->fill >= LIVESYNC_OUTGOING_BUFFSIZE_PEERS - LIVESYNC_OUTGOING_WATERMARK_PEERS)
+    livesync_issue_peersync(dest_buf);
+  else
+    pthread_mutex_unlock(&g_outbuf_mutex);
+}
+
+static void *livesync_worker(void *args) {
+  struct ot_workstruct ws;
+  ot_ip6               in_ip;
+  uint16_t             in_port;
+
+  (void)args;
+
+  /* Initialize our "thread local storage" */
+  ws.inbuf = ws.request = malloc(LIVESYNC_INCOMING_BUFFSIZE);
+  ws.outbuf = ws.reply = 0;
+
+  memcpy(in_ip, V4mappedprefix, sizeof(V4mappedprefix));
+
+  while (1) {
+    ws.request_size = socket_recv4(g_socket_in, (char *)ws.inbuf, LIVESYNC_INCOMING_BUFFSIZE, 12 + (char *)in_ip, &in_port);
+
+    /* Expect at least tracker id and packet type */
+    if (ws.request_size <= (ssize_t)(sizeof(g_tracker_id) + sizeof(uint32_t)))
       continue;
-
-    if( datalen < (ssize_t)(sizeof( g_tracker_id ) + sizeof( ot_hash ) + sizeof( ot_peer ) ) ) {
-      // TODO: log invalid sync packet
+    if (!accesslist_is_blessed(in_ip, OT_PERMISSION_MAY_LIVESYNC))
+      continue;
+    if (!memcmp(ws.inbuf, &g_tracker_id, sizeof(g_tracker_id))) {
+      /* TODO: log packet coming from ourselves */
       continue;
     }
 
-    if( !accesslist_isblessed((char*)in_ip, OT_PERMISSION_MAY_LIVESYNC)) {
-      // TODO: log invalid sync packet
-      continue;
-    }
-
-    if( !memcmp( livesync_inbuffer, &g_tracker_id, sizeof( g_tracker_id ) ) ) {
-      // TODO: log packet coming from ourselves
-      continue;
-    }
-
-    // Now basic sanity checks have been done on the live sync packet
-    // We might add more testing and logging.
-    while( off + (ssize_t)sizeof( ot_hash ) + (ssize_t)sizeof( ot_peer ) <= datalen ) {
-      ot_peer *peer = (ot_peer*)(livesync_inbuffer + off + sizeof(ot_hash));
-      ot_hash *hash = (ot_hash*)(livesync_inbuffer + off);
-
-      if( OT_FLAG(peer) & PEER_FLAG_STOPPED )
-        remove_peer_from_torrent(hash, peer, NULL, FLAG_MCA);
-      else
-        add_peer_to_torrent( hash, peer  WANT_SYNC_PARAM(1));
-
-      off += sizeof( ot_hash ) + sizeof( ot_peer );
+    switch (uint32_read_big(sizeof(g_tracker_id) + (char *)ws.inbuf)) {
+    case OT_SYNC_PEER6:
+      livesync_handle_peersync(&ws, OT_PEER_SIZE6);
+      break;
+    case OT_SYNC_PEER4:
+      livesync_handle_peersync(&ws, OT_PEER_SIZE4);
+      break;
+    default:
+      break;
     }
   }
+
   /* Never returns. */
   return NULL;
 }
 
 #endif
-const char *g_version_livesync_c = "$Source$: $Revision$\n";

ot_livesync.h

@@ -3,40 +3,45 @@
 
  $id$ */
 
-#ifndef __OT_LIVESYNC_H__
-#define __OT_LIVESYNC_H__
+#ifndef OT_LIVESYNC_H__
+#define OT_LIVESYNC_H__
 
 #include "io.h"
 #include "trackerlogic.h"
 
 /*
-  Syncing is done as udp packets in the multicast domain 224.0.42.N port 9696
+  Syncing is done as udp packets in the multicast domain 224.0.42.5 port 9696
 
   Each tracker should join the multicast group and send its live sync packets
   to that group, using a ttl of 1
 
-  Format of a live sync packet is straight forward and depends on N:
+  Format of all sync packets is straight forward, packet type determines
+  which kind of packet this is:
 
-  For N == 23: (simple tracker2tracker sync)
     0x0000 0x04 id of tracker instance
-  [ 0x0004 0x14 info_hash
-    0x0018 0x04 peer's ipv4 address
-    0x001c 0x02 peer's port
-    0x0020 0x02 peer flags v1 ( SEEDING = 0x80, COMPLETE = 0x40, STOPPED = 0x20 )
+    0x0004 0x04 packet type
+
+                             ########
+ ######## PEER SYNC PROTOCOL ########
+ ########
+
+  Each tracker instance accumulates announce requests until its buffer is
+  full or a timeout is reached. Then it broadcasts its live sync packer:
+
+  packet type SYNC_LIVE4
+  [ 0x0008 0x14 info_hash
+    0x001c 0x04 peer's ipv4 address
+    0x0020 0x02 peer's port
+    0x0024 0x02 peer flags v1 ( SEEDING = 0x80, COMPLETE = 0x40, STOPPED = 0x20 )
   ]*
 
-  For N == 24: (aggregator syncs)
-    0x0000 0x04 id of tracker instance
-  [ 0x0004 0x14 info_hash
-    0x0018 0x01 number of peers
-    [ 0x0019 0x04 peer's ipv4 address
-      0x001a 0x02 peer's port
-      0x0021 0x02 peer flags v1 ( SEEDING = 0x80, COMPLETE = 0x40, STOPPED = 0x20 )
-    ]+
+  packet type SYNC_LIVE6
+  [ 0x0008 0x14 info_hash
+    0x001c 0x10 peer's ipv6 address
+    0x002c 0x02 peer's port
+    0x002e 0x02 peer flags v1 ( SEEDING = 0x80, COMPLETE = 0x40, STOPPED = 0x20 )
   ]*
-
-
- */
+*/
 
 #ifdef WANT_SYNC_LIVE
 
@@ -46,24 +51,24 @@ void livesync_init();
 void livesync_deinit();
 
 /* Join multicast group for listening and create sending socket */
-void livesync_bind_mcast( char *ip, uint16_t port );
+void livesync_bind_mcast(char *ip, uint16_t port);
 
 /* Inform live sync about whats going on. */
-void livesync_tell( ot_hash * const info_hash, const ot_peer * const peer, const uint8_t peerflag );
+void livesync_tell(struct ot_workstruct *ws);
 
 /* Tickle the live sync module from time to time, so no events get
    stuck when there's not enough traffic to fill udp packets fast
    enough */
-void livesync_ticker( );
+void livesync_ticker();
 
 /* Handle an incoming live sync packet */
-void handle_livesync( const int64 serversocket );
+void handle_livesync(const int64 sock);
 
 #else
 
 /* If no syncing is required, save calling code from #ifdef
    constructions */
-
+#define livesync_deinit()
 #define livesync_init()
 #define livesync_ticker()
 #define handle_livesync(a)

ot_mutex.c

@@ -13,310 +13,260 @@
 /* Libowfat */
 #include "byte.h"
 #include "io.h"
+#include "uint32.h"
 
 /* Opentracker */
-#include "trackerlogic.h"
+#include "ot_iovec.h"
 #include "ot_mutex.h"
+#include "ot_stats.h"
+#include "trackerlogic.h"
 
-//#define MTX_DBG( STRING ) fprintf( stderr, STRING )
-#define MTX_DBG( STRING )
+/* #define MTX_DBG( STRING ) fprintf( stderr, STRING ) */
+#define MTX_DBG(STRING)
 
 /* Our global all torrents list */
-static ot_vector all_torrents[OT_BUCKET_COUNT];
+static ot_vector       all_torrents[OT_BUCKET_COUNT];
+static pthread_mutex_t bucket_mutex[OT_BUCKET_COUNT];
+static size_t          g_torrent_count;
 
-/* Bucket Magic */
-static int bucket_locklist[ OT_MAX_THREADS ];
-static int bucket_locklist_count = 0;
-static pthread_mutex_t bucket_mutex;
-static pthread_cond_t bucket_being_unlocked;
+/* Self pipe from opentracker.c */
+extern int             g_self_pipe[2];
 
-static int bucket_check( int bucket ) {
-  /* C should come with auto-i ;) */
-  int i;
-
-  /* No more space to acquire lock to bucket -- should not happen */
-  if( bucket_locklist_count == OT_MAX_THREADS ) {
-    fprintf( stderr, "More lock requests than mutexes. Consult source code.\n" );
-    return -1;
-  }
-
-  /* See, if bucket is already locked */
-  for( i=0; i<bucket_locklist_count; ++i )
-    if( bucket_locklist[ i ] == bucket )
-      return -1;
-
-  return 0;
-}
-
-static void bucket_push( int bucket ) {
-  bucket_locklist[ bucket_locklist_count++ ] = bucket;
-}
-
-static void bucket_remove( int bucket ) {
-  int i = 0;
-
-  while( ( i < bucket_locklist_count ) && ( bucket_locklist[ i ] != bucket ) )
-    ++i;
-
-  if( i == bucket_locklist_count ) {
-    fprintf( stderr, "Request to unlock bucket that was never locked. Consult source code.\n" );
-    return;
-  }
-
-  for( ; i < bucket_locklist_count - 1; ++i )
-    bucket_locklist[ i ] = bucket_locklist[ i + 1 ];
-
-  --bucket_locklist_count;
-}
-
-ot_vector *mutex_bucket_lock( int bucket ) {
-  pthread_mutex_lock( &bucket_mutex );
-  while( bucket_check( bucket ) )
-    pthread_cond_wait( &bucket_being_unlocked, &bucket_mutex );
-  bucket_push( bucket );
-  pthread_mutex_unlock( &bucket_mutex );
+ot_vector *mutex_bucket_lock(int bucket) {
+  pthread_mutex_lock(bucket_mutex + bucket);
   return all_torrents + bucket;
 }
 
-ot_vector *mutex_bucket_lock_by_hash( ot_hash *hash ) {
-  unsigned char *local_hash = hash[0];
-  int bucket = ( local_hash[0] << 2 ) | ( local_hash[1] >> 6 );
+ot_vector *mutex_bucket_lock_by_hash(ot_hash const hash) { return mutex_bucket_lock(uint32_read_big((const char *)hash) >> OT_BUCKET_COUNT_SHIFT); }
 
-  /* Can block */
-  mutex_bucket_lock( bucket );
-
-  return all_torrents + bucket;
+void mutex_bucket_unlock(int bucket, int delta_torrentcount) {
+  pthread_mutex_unlock(bucket_mutex + bucket);
+  g_torrent_count += delta_torrentcount;
 }
 
-void mutex_bucket_unlock( int bucket ) {
-  pthread_mutex_lock( &bucket_mutex );
-  bucket_remove( bucket );
-  pthread_cond_broadcast( &bucket_being_unlocked );
-  pthread_mutex_unlock( &bucket_mutex );
+void mutex_bucket_unlock_by_hash(ot_hash const hash, int delta_torrentcount) {
+  mutex_bucket_unlock(uint32_read_big((char *)hash) >> OT_BUCKET_COUNT_SHIFT, delta_torrentcount);
 }
 
-void mutex_bucket_unlock_by_hash( ot_hash *hash ) {
-  unsigned char *local_hash = hash[0];
-  int bucket = ( local_hash[0] << 2 ) | ( local_hash[1] >> 6 );
-  mutex_bucket_unlock( bucket );
-}
+size_t mutex_get_torrent_count() { return g_torrent_count; }
 
 /* TaskQueue Magic */
 
 struct ot_task {
   ot_taskid       taskid;
   ot_tasktype     tasktype;
-  int64           socket;
+  int64           sock;
   int             iovec_entries;
   struct iovec   *iovec;
   struct ot_task *next;
 };
 
-static ot_taskid next_free_taskid = 1;
-static struct ot_task *tasklist = NULL;
+static ot_taskid       next_free_taskid = 1;
+static struct ot_task *tasklist;
 static pthread_mutex_t tasklist_mutex;
-static pthread_cond_t tasklist_being_filled;
+static pthread_cond_t  tasklist_being_filled;
 
-int mutex_workqueue_pushtask( int64 socket, ot_tasktype tasktype ) {
-  struct ot_task ** tmptask, * task;
+int mutex_workqueue_pushtask(int64 sock, ot_tasktype tasktype) {
+  struct ot_task **tmptask, *task;
 
-  /* Want exclusive access to tasklist */
-  MTX_DBG( "pushtask locks.\n" );
-  pthread_mutex_lock( &tasklist_mutex );
-  MTX_DBG( "pushtask locked.\n" );
-
-  task = malloc(sizeof( struct ot_task));
-  if( !task ) {
-    MTX_DBG( "pushtask fail unlocks.\n" );
-    pthread_mutex_unlock( &tasklist_mutex );
-    MTX_DBG( "pushtask fail unlocked.\n" );
+  task = malloc(sizeof(struct ot_task));
+  if (!task)
     return -1;
-  }
-
-  /* Skip to end of list */
-  tmptask = &tasklist;
-  while( *tmptask )
-    tmptask = &(*tmptask)->next;
-  *tmptask = task;
 
   task->taskid        = 0;
   task->tasktype      = tasktype;
-  task->socket        = socket;
+  task->sock          = sock;
   task->iovec_entries = 0;
   task->iovec         = NULL;
   task->next          = 0;
 
+  /* Want exclusive access to tasklist */
+  pthread_mutex_lock(&tasklist_mutex);
+
+  /* Skip to end of list */
+  tmptask = &tasklist;
+  while (*tmptask)
+    tmptask = &(*tmptask)->next;
+  *tmptask = task;
+
   /* Inform waiting workers and release lock */
-  MTX_DBG( "pushtask broadcasts.\n" );
-  pthread_cond_broadcast( &tasklist_being_filled );
-  MTX_DBG( "pushtask broadcasted, mutex unlocks.\n" );
-  pthread_mutex_unlock( &tasklist_mutex );
-  MTX_DBG( "pushtask end mutex unlocked.\n" );
+  pthread_cond_broadcast(&tasklist_being_filled);
+  pthread_mutex_unlock(&tasklist_mutex);
   return 0;
 }
 
-void mutex_workqueue_canceltask( int64 socket ) {
-  struct ot_task ** task;
+void mutex_workqueue_canceltask(int64 sock) {
+  struct ot_task **task;
 
   /* Want exclusive access to tasklist */
-  MTX_DBG( "canceltask locks.\n" );
-  pthread_mutex_lock( &tasklist_mutex );
-  MTX_DBG( "canceltask locked.\n" );
+  pthread_mutex_lock(&tasklist_mutex);
 
-  task = &tasklist;
-  while( *task && ( (*task)->socket != socket ) )
-    *task = (*task)->next;
+  for (task = &tasklist; *task; task = &((*task)->next))
+    if ((*task)->sock == sock) {
+      struct iovec   *iovec = (*task)->iovec;
+      struct ot_task *ptask = *task;
+      int             i;
 
-  if( *task && ( (*task)->socket == socket ) ) {
-    struct iovec *iovec = (*task)->iovec;
-    struct ot_task *ptask = *task;
-    int i;
+      /* Free task's iovec */
+      for (i = 0; i < (*task)->iovec_entries; ++i)
+        free(iovec[i].iov_base);
 
-    /* Free task's iovec */
-    for( i=0; i<(*task)->iovec_entries; ++i )
-      munmap( iovec[i].iov_base , iovec[i].iov_len );
-
-    *task = (*task)->next;
-    free( ptask );
-  }
+      *task = (*task)->next;
+      free(ptask);
+      break;
+    }
 
   /* Release lock */
-  MTX_DBG( "canceltask unlocks.\n" );
-  pthread_mutex_unlock( &tasklist_mutex );
-  MTX_DBG( "canceltask unlocked.\n" );
+  pthread_mutex_unlock(&tasklist_mutex);
 }
 
-ot_taskid mutex_workqueue_poptask( ot_tasktype *tasktype ) {
-  struct ot_task * task;
-  ot_taskid taskid = 0;
+ot_taskid mutex_workqueue_poptask(ot_tasktype *tasktype) {
+  struct ot_task *task;
+  ot_taskid       taskid = 0;
 
   /* Want exclusive access to tasklist */
-  MTX_DBG( "poptask mutex locks.\n" );
-  pthread_mutex_lock( &tasklist_mutex );
-  MTX_DBG( "poptask mutex locked.\n" );
+  pthread_mutex_lock(&tasklist_mutex);
 
-  while( !taskid ) {
+  while (!taskid) {
     /* Skip to the first unassigned task this worker wants to do */
-    task = tasklist;
-    while( task && ( ( ( TASK_CLASS_MASK & task->tasktype ) != *tasktype ) || task->taskid ) )
-      task = task->next;
+    for (task = tasklist; task; task = task->next)
+      if (!task->taskid && (TASK_CLASS_MASK & task->tasktype) == *tasktype) {
+        /* If we found an outstanding task, assign a taskid to it
+           and leave the loop */
+        task->taskid = taskid = ++next_free_taskid;
+        *tasktype             = task->tasktype;
+        break;
+      }
 
-    /* If we found an outstanding task, assign a taskid to it
-       and leave the loop */
-    if( task ) {
-      task->taskid = taskid = ++next_free_taskid;
-      *tasktype = task->tasktype;
-    } else {
-      /* Wait until the next task is being fed */
-      MTX_DBG( "poptask cond waits.\n" );
-      pthread_cond_wait( &tasklist_being_filled, &tasklist_mutex );
-      MTX_DBG( "poptask cond waited.\n" );
-    }
+    /* Wait until the next task is being fed */
+    if (!taskid)
+      pthread_cond_wait(&tasklist_being_filled, &tasklist_mutex);
   }
 
   /* Release lock */
-  MTX_DBG( "poptask end mutex unlocks.\n" );
-  pthread_mutex_unlock( &tasklist_mutex );
-  MTX_DBG( "poptask end mutex unlocked.\n" );
+  pthread_mutex_unlock(&tasklist_mutex);
 
   return taskid;
 }
 
-void mutex_workqueue_pushsuccess( ot_taskid taskid ) {
-  struct ot_task ** task;
+void mutex_workqueue_pushsuccess(ot_taskid taskid) {
+  struct ot_task **task;
 
   /* Want exclusive access to tasklist */
-  MTX_DBG( "pushsuccess locks.\n" );
-  pthread_mutex_lock( &tasklist_mutex );
-  MTX_DBG( "pushsuccess locked.\n" );
+  pthread_mutex_lock(&tasklist_mutex);
 
-  task = &tasklist;
-  while( *task && ( (*task)->taskid != taskid ) )
-    *task = (*task)->next;
-
-  if( *task && ( (*task)->taskid == taskid ) ) {
-    struct ot_task *ptask = *task;
-    *task = (*task)->next;
-    free( ptask );
-  }
+  for (task = &tasklist; *task; task = &((*task)->next))
+    if ((*task)->taskid == taskid) {
+      struct ot_task *ptask = *task;
+      *task = (*task)->next;
+      free(ptask);
+      break;
+    }
 
   /* Release lock */
-  MTX_DBG( "pushsuccess unlocks.\n" );
-  pthread_mutex_unlock( &tasklist_mutex );
-  MTX_DBG( "pushsuccess unlocked.\n" );
+  pthread_mutex_unlock(&tasklist_mutex);
 }
 
-int mutex_workqueue_pushresult( ot_taskid taskid, int iovec_entries, struct iovec *iovec ) {
-  struct ot_task * task;
+int mutex_workqueue_pushresult(ot_taskid taskid, int iovec_entries, struct iovec *iovec) {
+  struct ot_task *task;
+  const char      byte = 'o';
+
   /* Want exclusive access to tasklist */
-  MTX_DBG( "pushresult locks.\n" );
-  pthread_mutex_lock( &tasklist_mutex );
-  MTX_DBG( "pushresult locked.\n" );
+  pthread_mutex_lock(&tasklist_mutex);
 
-  task = tasklist;
-  while( task && ( task->taskid != taskid ) )
-    task = task->next;
-
-  if( task ) {
-    task->iovec_entries = iovec_entries;
-    task->iovec         = iovec;
-    task->tasktype      = TASK_DONE;
-  }
+  for (task = tasklist; task; task = task->next)
+    if (task->taskid == taskid) {
+      task->iovec_entries = iovec_entries;
+      task->iovec         = iovec;
+      task->tasktype      = TASK_DONE;
+      break;
+    }
 
   /* Release lock */
-  MTX_DBG( "pushresult unlocks.\n" );
-  pthread_mutex_unlock( &tasklist_mutex );
-  MTX_DBG( "pushresult unlocked.\n" );
+  pthread_mutex_unlock(&tasklist_mutex);
+
+  io_trywrite(g_self_pipe[1], &byte, 1);
 
   /* Indicate whether the worker has to throw away results */
   return task ? 0 : -1;
 }
 
-int64 mutex_workqueue_popresult( int *iovec_entries, struct iovec ** iovec ) {
-  struct ot_task ** task;
-  int64 socket = -1;
+int mutex_workqueue_pushchunked(ot_taskid taskid, struct iovec *iovec) {
+  struct ot_task *task;
+  const char      byte = 'o';
 
   /* Want exclusive access to tasklist */
-  MTX_DBG( "popresult locks.\n" );
-  pthread_mutex_lock( &tasklist_mutex );
-  MTX_DBG( "popresult locked.\n" );
+  pthread_mutex_lock(&tasklist_mutex);
 
-  task = &tasklist;
-  while( *task && ( (*task)->tasktype != TASK_DONE ) )
-    task = &(*task)->next;
-
-  if( *task && ( (*task)->tasktype == TASK_DONE ) ) {
-    struct ot_task *ptask = *task;
-
-    *iovec_entries = (*task)->iovec_entries;
-    *iovec         = (*task)->iovec;
-    socket         = (*task)->socket;
-
-    *task = (*task)->next;
-    free( ptask );
-  }
+  for (task = tasklist; task; task = task->next)
+    if (task->taskid == taskid) {
+      if (iovec) {
+        if (iovec_append(&task->iovec_entries, &task->iovec, iovec))
+          task->tasktype = TASK_DONE_PARTIAL;
+        else
+          task = NULL;
+      } else
+        task->tasktype = TASK_DONE;
+      break;
+    }
 
   /* Release lock */
-  MTX_DBG( "popresult unlocks.\n" );
-  pthread_mutex_unlock( &tasklist_mutex );
-  MTX_DBG( "popresult unlocked.\n" );
-  return socket;
+  pthread_mutex_unlock(&tasklist_mutex);
+
+  io_trywrite(g_self_pipe[1], &byte, 1);
+
+  /* Indicate whether the worker has to throw away results */
+  return task ? 0 : -1;
 }
 
-void mutex_init( ) {
+int64 mutex_workqueue_popresult(int *iovec_entries, struct iovec **iovec, int *is_partial) {
+  struct ot_task **task;
+  int64            sock = -1;
+
+  *is_partial = 0;
+
+  /* Want exclusive access to tasklist */
+  pthread_mutex_lock(&tasklist_mutex);
+
+  for (task = &tasklist; *task; task = &((*task)->next))
+    if (((*task)->tasktype & TASK_CLASS_MASK) == TASK_DONE) {
+      struct ot_task *ptask = *task;
+      *iovec_entries        = ptask->iovec_entries;
+      *iovec                = ptask->iovec;
+      sock                  = ptask->sock;
+
+      if ((*task)->tasktype == TASK_DONE) {
+        *task = ptask->next;
+        free(ptask);
+      } else {
+        ptask->iovec_entries = 0;
+        ptask->iovec         = NULL;
+        *is_partial          = 1;
+        /* Prevent task from showing up immediately again unless new data was added */
+        (*task)->tasktype    = TASK_FULLSCRAPE;
+      }
+      break;
+    }
+
+  /* Release lock */
+  pthread_mutex_unlock(&tasklist_mutex);
+  return sock;
+}
+
+void mutex_init() {
+  int i;
   pthread_mutex_init(&tasklist_mutex, NULL);
-  pthread_cond_init (&tasklist_being_filled, NULL);
-  pthread_mutex_init(&bucket_mutex, NULL);
-  pthread_cond_init (&bucket_being_unlocked, NULL);
-  byte_zero( all_torrents, sizeof( all_torrents ) );
+  pthread_cond_init(&tasklist_being_filled, NULL);
+  for (i = 0; i < OT_BUCKET_COUNT; ++i)
+    pthread_mutex_init(bucket_mutex + i, NULL);
+  byte_zero(all_torrents, sizeof(all_torrents));
 }
 
-void mutex_deinit( ) {
-  pthread_mutex_destroy(&bucket_mutex);
-  pthread_cond_destroy(&bucket_being_unlocked);
+void mutex_deinit() {
+  int i;
+  for (i = 0; i < OT_BUCKET_COUNT; ++i)
+    pthread_mutex_destroy(bucket_mutex + i);
   pthread_mutex_destroy(&tasklist_mutex);
   pthread_cond_destroy(&tasklist_being_filled);
-  byte_zero( all_torrents, sizeof( all_torrents ) );
+  byte_zero(all_torrents, sizeof(all_torrents));
 }
-
-const char *g_version_mutex_c = "$Source$: $Revision$\n";

ot_mutex.h

@@ -3,69 +3,78 @@
 
    $id$ */
 
-#ifndef __OT_MUTEX_H__
-#define __OT_MUTEX_H__
+#ifndef OT_MUTEX_H__
+#define OT_MUTEX_H__
 
-void mutex_init( );
-void mutex_deinit( );
+#include <sys/uio.h>
+#include "trackerlogic.h"
 
-ot_vector *mutex_bucket_lock( int bucket );
-ot_vector *mutex_bucket_lock_by_hash( ot_hash *hash );
+void       mutex_init(void);
+void       mutex_deinit(void);
 
-void mutex_bucket_unlock( int bucket );
-void mutex_bucket_unlock_by_hash( ot_hash *hash );
+ot_vector *mutex_bucket_lock(int bucket);
+ot_vector *mutex_bucket_lock_by_hash(ot_hash const hash);
+
+void       mutex_bucket_unlock(int bucket, int delta_torrentcount);
+void       mutex_bucket_unlock_by_hash(ot_hash const hash, int delta_torrentcount);
+
+size_t     mutex_get_torrent_count(void);
 
 typedef enum {
-  TASK_STATS_CONNS                 = 0x0001,
-  TASK_STATS_TCP                   = 0x0002,
-  TASK_STATS_UDP                   = 0x0003,
-  TASK_STATS_SCRAPE                = 0x0004,
-  TASK_STATS_FULLSCRAPE            = 0x0005,
-  TASK_STATS_TPB                   = 0x0006,
-  TASK_STATS_HTTPERRORS            = 0x0007,
-  TASK_STATS_STARTSTOP             = 0x0008,
-  TASK_STATS_TORADDREM             = 0x0009,
-  TASK_STATS_VERSION               = 0x000a,
-  TASK_STATS_BUSY_NETWORKS         = 0x000b,
-  TASK_STATS_VECTOR_DEBUG          = 0x000c,
-  TASK_STATS_RENEW                 = 0x000d,
+  TASK_STATS_CONNS               = 0x0001,
+  TASK_STATS_TCP                 = 0x0002,
+  TASK_STATS_UDP                 = 0x0003,
+  TASK_STATS_SCRAPE              = 0x0004,
+  TASK_STATS_FULLSCRAPE          = 0x0005,
+  TASK_STATS_TPB                 = 0x0006,
+  TASK_STATS_HTTPERRORS          = 0x0007,
+  TASK_STATS_VERSION             = 0x0008,
+  TASK_STATS_BUSY_NETWORKS       = 0x0009,
+  TASK_STATS_RENEW               = 0x000a,
+  TASK_STATS_SYNCS               = 0x000b,
+  TASK_STATS_COMPLETED           = 0x000c,
+  TASK_STATS_NUMWANTS            = 0x000d,
 
-  TASK_STATS                       = 0x0100, /* Mask */
-  TASK_STATS_TORRENTS              = 0x0101,
-  TASK_STATS_PEERS                 = 0x0102,
-  TASK_STATS_SLASH24S              = 0x0103,
-  TASK_STATS_TOP10                 = 0x0104,
-  TASK_STATS_MEMORY                = 0x0105,
+  TASK_STATS                     = 0x0100, /* Mask */
+  TASK_STATS_TORRENTS            = 0x0101,
+  TASK_STATS_PEERS               = 0x0102,
+  TASK_STATS_SLASH24S            = 0x0103,
+  TASK_STATS_TOP10               = 0x0104,
+  TASK_STATS_TOP100              = 0x0105,
+  TASK_STATS_EVERYTHING          = 0x0106,
+  TASK_STATS_FULLLOG             = 0x0107,
+  TASK_STATS_WOODPECKERS         = 0x0108,
 
-  TASK_FULLSCRAPE                  = 0x0200, /* Default mode */
-  TASK_FULLSCRAPE_TPB_BINARY       = 0x0201,
-  TASK_FULLSCRAPE_TPB_ASCII        = 0x0202,
-  TASK_FULLSCRAPE_TPB_URLENCODED   = 0x0203,
+  TASK_FULLSCRAPE                = 0x0200, /* Default mode */
+  TASK_FULLSCRAPE_TPB_BINARY     = 0x0201,
+  TASK_FULLSCRAPE_TPB_ASCII      = 0x0202,
+  TASK_FULLSCRAPE_TPB_ASCII_PLUS = 0x0203,
+  TASK_FULLSCRAPE_TPB_URLENCODED = 0x0204,
+  TASK_FULLSCRAPE_TRACKERSTATE   = 0x0205,
 
-  TASK_CLEAN                       = 0x0300,
+  TASK_DMEM                      = 0x0300,
 
-  TASK_SYNC_OUT                    = 0x0400,
-  TASK_SYNC_IN                     = 0x0401,
+  TASK_DONE                      = 0x0f00,
+  TASK_DONE_PARTIAL              = 0x0f01,
 
-  TASK_DMEM                        = 0x0500,
+  TASK_FLAG_GZIP                 = 0x1000,
+  TASK_FLAG_BZIP2                = 0x2000,
+  TASK_FLAG_ZSTD                 = 0x4000,
+  TASK_FLAG_CHUNKED              = 0x8000,
 
-  TASK_DONE                        = 0x0f00,
-
-  TASK_FLAG_GZIP                   = 0x1000,
-  TASK_FLAG_BZIP2                  = 0x2000,
-
-  TASK_TASK_MASK                   = 0x0fff,
-  TASK_CLASS_MASK                  = 0x0f00,
-  TASK_FLAGS_MASK                  = 0xf000
+  TASK_TASK_MASK                 = 0x0fff,
+  TASK_CLASS_MASK                = 0x0f00,
+  TASK_FLAGS_MASK                = 0xf000
 } ot_tasktype;
 
 typedef unsigned long ot_taskid;
 
-int       mutex_workqueue_pushtask( int64 socket, ot_tasktype tasktype );
-void      mutex_workqueue_canceltask( int64 socket );
-void      mutex_workqueue_pushsuccess( ot_taskid taskid );
-ot_taskid mutex_workqueue_poptask( ot_tasktype *tasktype );
-int       mutex_workqueue_pushresult( ot_taskid taskid, int iovec_entries, struct iovec *iovector );
-int64     mutex_workqueue_popresult( int *iovec_entries, struct iovec ** iovector );
+int                   mutex_workqueue_pushtask(int64 sock, ot_tasktype tasktype);
+void                  mutex_workqueue_canceltask(int64 sock);
+void                  mutex_workqueue_pushsuccess(ot_taskid taskid);
+ot_taskid             mutex_workqueue_poptask(ot_tasktype *tasktype);
+int                   mutex_workqueue_pushresult(ot_taskid taskid, int iovec_entries, struct iovec *iovector);
+int                   mutex_workqueue_pushchunked(ot_taskid taskid, struct iovec *iovec);
+int64                 mutex_workqueue_popresult(int *iovec_entries, struct iovec **iovector, int *is_partial);
 
 #endif

ot_rijndael.c

@@ -0,0 +1,488 @@
+/**
+ * rijndael-alg-fst.c
+ *
+ * @version 3.0 (December 2000)
+ *
+ * Optimised ANSI C code for the Rijndael cipher (now AES)
+ *
+ * @author Vincent Rijmen <vincent.rijmen@esat.kuleuven.ac.be>
+ * @author Antoon Bosselaers <antoon.bosselaers@esat.kuleuven.ac.be>
+ * @author Paulo Barreto <paulo.barreto@terra.com.br>
+ *
+ * This code is hereby placed in the public domain.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ''AS IS'' AND ANY EXPRESS
+ * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#include "ot_rijndael.h"
+
+static const uint32_t Te0[256] = {
+    0xc66363a5U, 0xf87c7c84U, 0xee777799U, 0xf67b7b8dU,
+    0xfff2f20dU, 0xd66b6bbdU, 0xde6f6fb1U, 0x91c5c554U,
+    0x60303050U, 0x02010103U, 0xce6767a9U, 0x562b2b7dU,
+    0xe7fefe19U, 0xb5d7d762U, 0x4dababe6U, 0xec76769aU,
+    0x8fcaca45U, 0x1f82829dU, 0x89c9c940U, 0xfa7d7d87U,
+    0xeffafa15U, 0xb25959ebU, 0x8e4747c9U, 0xfbf0f00bU,
+    0x41adadecU, 0xb3d4d467U, 0x5fa2a2fdU, 0x45afafeaU,
+    0x239c9cbfU, 0x53a4a4f7U, 0xe4727296U, 0x9bc0c05bU,
+    0x75b7b7c2U, 0xe1fdfd1cU, 0x3d9393aeU, 0x4c26266aU,
+    0x6c36365aU, 0x7e3f3f41U, 0xf5f7f702U, 0x83cccc4fU,
+    0x6834345cU, 0x51a5a5f4U, 0xd1e5e534U, 0xf9f1f108U,
+    0xe2717193U, 0xabd8d873U, 0x62313153U, 0x2a15153fU,
+    0x0804040cU, 0x95c7c752U, 0x46232365U, 0x9dc3c35eU,
+    0x30181828U, 0x379696a1U, 0x0a05050fU, 0x2f9a9ab5U,
+    0x0e070709U, 0x24121236U, 0x1b80809bU, 0xdfe2e23dU,
+    0xcdebeb26U, 0x4e272769U, 0x7fb2b2cdU, 0xea75759fU,
+    0x1209091bU, 0x1d83839eU, 0x582c2c74U, 0x341a1a2eU,
+    0x361b1b2dU, 0xdc6e6eb2U, 0xb45a5aeeU, 0x5ba0a0fbU,
+    0xa45252f6U, 0x763b3b4dU, 0xb7d6d661U, 0x7db3b3ceU,
+    0x5229297bU, 0xdde3e33eU, 0x5e2f2f71U, 0x13848497U,
+    0xa65353f5U, 0xb9d1d168U, 0x00000000U, 0xc1eded2cU,
+    0x40202060U, 0xe3fcfc1fU, 0x79b1b1c8U, 0xb65b5bedU,
+    0xd46a6abeU, 0x8dcbcb46U, 0x67bebed9U, 0x7239394bU,
+    0x944a4adeU, 0x984c4cd4U, 0xb05858e8U, 0x85cfcf4aU,
+    0xbbd0d06bU, 0xc5efef2aU, 0x4faaaae5U, 0xedfbfb16U,
+    0x864343c5U, 0x9a4d4dd7U, 0x66333355U, 0x11858594U,
+    0x8a4545cfU, 0xe9f9f910U, 0x04020206U, 0xfe7f7f81U,
+    0xa05050f0U, 0x783c3c44U, 0x259f9fbaU, 0x4ba8a8e3U,
+    0xa25151f3U, 0x5da3a3feU, 0x804040c0U, 0x058f8f8aU,
+    0x3f9292adU, 0x219d9dbcU, 0x70383848U, 0xf1f5f504U,
+    0x63bcbcdfU, 0x77b6b6c1U, 0xafdada75U, 0x42212163U,
+    0x20101030U, 0xe5ffff1aU, 0xfdf3f30eU, 0xbfd2d26dU,
+    0x81cdcd4cU, 0x180c0c14U, 0x26131335U, 0xc3ecec2fU,
+    0xbe5f5fe1U, 0x359797a2U, 0x884444ccU, 0x2e171739U,
+    0x93c4c457U, 0x55a7a7f2U, 0xfc7e7e82U, 0x7a3d3d47U,
+    0xc86464acU, 0xba5d5de7U, 0x3219192bU, 0xe6737395U,
+    0xc06060a0U, 0x19818198U, 0x9e4f4fd1U, 0xa3dcdc7fU,
+    0x44222266U, 0x542a2a7eU, 0x3b9090abU, 0x0b888883U,
+    0x8c4646caU, 0xc7eeee29U, 0x6bb8b8d3U, 0x2814143cU,
+    0xa7dede79U, 0xbc5e5ee2U, 0x160b0b1dU, 0xaddbdb76U,
+    0xdbe0e03bU, 0x64323256U, 0x743a3a4eU, 0x140a0a1eU,
+    0x924949dbU, 0x0c06060aU, 0x4824246cU, 0xb85c5ce4U,
+    0x9fc2c25dU, 0xbdd3d36eU, 0x43acacefU, 0xc46262a6U,
+    0x399191a8U, 0x319595a4U, 0xd3e4e437U, 0xf279798bU,
+    0xd5e7e732U, 0x8bc8c843U, 0x6e373759U, 0xda6d6db7U,
+    0x018d8d8cU, 0xb1d5d564U, 0x9c4e4ed2U, 0x49a9a9e0U,
+    0xd86c6cb4U, 0xac5656faU, 0xf3f4f407U, 0xcfeaea25U,
+    0xca6565afU, 0xf47a7a8eU, 0x47aeaee9U, 0x10080818U,
+    0x6fbabad5U, 0xf0787888U, 0x4a25256fU, 0x5c2e2e72U,
+    0x381c1c24U, 0x57a6a6f1U, 0x73b4b4c7U, 0x97c6c651U,
+    0xcbe8e823U, 0xa1dddd7cU, 0xe874749cU, 0x3e1f1f21U,
+    0x964b4bddU, 0x61bdbddcU, 0x0d8b8b86U, 0x0f8a8a85U,
+    0xe0707090U, 0x7c3e3e42U, 0x71b5b5c4U, 0xcc6666aaU,
+    0x904848d8U, 0x06030305U, 0xf7f6f601U, 0x1c0e0e12U,
+    0xc26161a3U, 0x6a35355fU, 0xae5757f9U, 0x69b9b9d0U,
+    0x17868691U, 0x99c1c158U, 0x3a1d1d27U, 0x279e9eb9U,
+    0xd9e1e138U, 0xebf8f813U, 0x2b9898b3U, 0x22111133U,
+    0xd26969bbU, 0xa9d9d970U, 0x078e8e89U, 0x339494a7U,
+    0x2d9b9bb6U, 0x3c1e1e22U, 0x15878792U, 0xc9e9e920U,
+    0x87cece49U, 0xaa5555ffU, 0x50282878U, 0xa5dfdf7aU,
+    0x038c8c8fU, 0x59a1a1f8U, 0x09898980U, 0x1a0d0d17U,
+    0x65bfbfdaU, 0xd7e6e631U, 0x844242c6U, 0xd06868b8U,
+    0x824141c3U, 0x299999b0U, 0x5a2d2d77U, 0x1e0f0f11U,
+    0x7bb0b0cbU, 0xa85454fcU, 0x6dbbbbd6U, 0x2c16163aU,
+};
+static const uint32_t Te1[256] = {
+    0xa5c66363U, 0x84f87c7cU, 0x99ee7777U, 0x8df67b7bU,
+    0x0dfff2f2U, 0xbdd66b6bU, 0xb1de6f6fU, 0x5491c5c5U,
+    0x50603030U, 0x03020101U, 0xa9ce6767U, 0x7d562b2bU,
+    0x19e7fefeU, 0x62b5d7d7U, 0xe64dababU, 0x9aec7676U,
+    0x458fcacaU, 0x9d1f8282U, 0x4089c9c9U, 0x87fa7d7dU,
+    0x15effafaU, 0xebb25959U, 0xc98e4747U, 0x0bfbf0f0U,
+    0xec41adadU, 0x67b3d4d4U, 0xfd5fa2a2U, 0xea45afafU,
+    0xbf239c9cU, 0xf753a4a4U, 0x96e47272U, 0x5b9bc0c0U,
+    0xc275b7b7U, 0x1ce1fdfdU, 0xae3d9393U, 0x6a4c2626U,
+    0x5a6c3636U, 0x417e3f3fU, 0x02f5f7f7U, 0x4f83ccccU,
+    0x5c683434U, 0xf451a5a5U, 0x34d1e5e5U, 0x08f9f1f1U,
+    0x93e27171U, 0x73abd8d8U, 0x53623131U, 0x3f2a1515U,
+    0x0c080404U, 0x5295c7c7U, 0x65462323U, 0x5e9dc3c3U,
+    0x28301818U, 0xa1379696U, 0x0f0a0505U, 0xb52f9a9aU,
+    0x090e0707U, 0x36241212U, 0x9b1b8080U, 0x3ddfe2e2U,
+    0x26cdebebU, 0x694e2727U, 0xcd7fb2b2U, 0x9fea7575U,
+    0x1b120909U, 0x9e1d8383U, 0x74582c2cU, 0x2e341a1aU,
+    0x2d361b1bU, 0xb2dc6e6eU, 0xeeb45a5aU, 0xfb5ba0a0U,
+    0xf6a45252U, 0x4d763b3bU, 0x61b7d6d6U, 0xce7db3b3U,
+    0x7b522929U, 0x3edde3e3U, 0x715e2f2fU, 0x97138484U,
+    0xf5a65353U, 0x68b9d1d1U, 0x00000000U, 0x2cc1ededU,
+    0x60402020U, 0x1fe3fcfcU, 0xc879b1b1U, 0xedb65b5bU,
+    0xbed46a6aU, 0x468dcbcbU, 0xd967bebeU, 0x4b723939U,
+    0xde944a4aU, 0xd4984c4cU, 0xe8b05858U, 0x4a85cfcfU,
+    0x6bbbd0d0U, 0x2ac5efefU, 0xe54faaaaU, 0x16edfbfbU,
+    0xc5864343U, 0xd79a4d4dU, 0x55663333U, 0x94118585U,
+    0xcf8a4545U, 0x10e9f9f9U, 0x06040202U, 0x81fe7f7fU,
+    0xf0a05050U, 0x44783c3cU, 0xba259f9fU, 0xe34ba8a8U,
+    0xf3a25151U, 0xfe5da3a3U, 0xc0804040U, 0x8a058f8fU,
+    0xad3f9292U, 0xbc219d9dU, 0x48703838U, 0x04f1f5f5U,
+    0xdf63bcbcU, 0xc177b6b6U, 0x75afdadaU, 0x63422121U,
+    0x30201010U, 0x1ae5ffffU, 0x0efdf3f3U, 0x6dbfd2d2U,
+    0x4c81cdcdU, 0x14180c0cU, 0x35261313U, 0x2fc3ececU,
+    0xe1be5f5fU, 0xa2359797U, 0xcc884444U, 0x392e1717U,
+    0x5793c4c4U, 0xf255a7a7U, 0x82fc7e7eU, 0x477a3d3dU,
+    0xacc86464U, 0xe7ba5d5dU, 0x2b321919U, 0x95e67373U,
+    0xa0c06060U, 0x98198181U, 0xd19e4f4fU, 0x7fa3dcdcU,
+    0x66442222U, 0x7e542a2aU, 0xab3b9090U, 0x830b8888U,
+    0xca8c4646U, 0x29c7eeeeU, 0xd36bb8b8U, 0x3c281414U,
+    0x79a7dedeU, 0xe2bc5e5eU, 0x1d160b0bU, 0x76addbdbU,
+    0x3bdbe0e0U, 0x56643232U, 0x4e743a3aU, 0x1e140a0aU,
+    0xdb924949U, 0x0a0c0606U, 0x6c482424U, 0xe4b85c5cU,
+    0x5d9fc2c2U, 0x6ebdd3d3U, 0xef43acacU, 0xa6c46262U,
+    0xa8399191U, 0xa4319595U, 0x37d3e4e4U, 0x8bf27979U,
+    0x32d5e7e7U, 0x438bc8c8U, 0x596e3737U, 0xb7da6d6dU,
+    0x8c018d8dU, 0x64b1d5d5U, 0xd29c4e4eU, 0xe049a9a9U,
+    0xb4d86c6cU, 0xfaac5656U, 0x07f3f4f4U, 0x25cfeaeaU,
+    0xafca6565U, 0x8ef47a7aU, 0xe947aeaeU, 0x18100808U,
+    0xd56fbabaU, 0x88f07878U, 0x6f4a2525U, 0x725c2e2eU,
+    0x24381c1cU, 0xf157a6a6U, 0xc773b4b4U, 0x5197c6c6U,
+    0x23cbe8e8U, 0x7ca1ddddU, 0x9ce87474U, 0x213e1f1fU,
+    0xdd964b4bU, 0xdc61bdbdU, 0x860d8b8bU, 0x850f8a8aU,
+    0x90e07070U, 0x427c3e3eU, 0xc471b5b5U, 0xaacc6666U,
+    0xd8904848U, 0x05060303U, 0x01f7f6f6U, 0x121c0e0eU,
+    0xa3c26161U, 0x5f6a3535U, 0xf9ae5757U, 0xd069b9b9U,
+    0x91178686U, 0x5899c1c1U, 0x273a1d1dU, 0xb9279e9eU,
+    0x38d9e1e1U, 0x13ebf8f8U, 0xb32b9898U, 0x33221111U,
+    0xbbd26969U, 0x70a9d9d9U, 0x89078e8eU, 0xa7339494U,
+    0xb62d9b9bU, 0x223c1e1eU, 0x92158787U, 0x20c9e9e9U,
+    0x4987ceceU, 0xffaa5555U, 0x78502828U, 0x7aa5dfdfU,
+    0x8f038c8cU, 0xf859a1a1U, 0x80098989U, 0x171a0d0dU,
+    0xda65bfbfU, 0x31d7e6e6U, 0xc6844242U, 0xb8d06868U,
+    0xc3824141U, 0xb0299999U, 0x775a2d2dU, 0x111e0f0fU,
+    0xcb7bb0b0U, 0xfca85454U, 0xd66dbbbbU, 0x3a2c1616U,
+};
+static const uint32_t Te2[256] = {
+    0x63a5c663U, 0x7c84f87cU, 0x7799ee77U, 0x7b8df67bU,
+    0xf20dfff2U, 0x6bbdd66bU, 0x6fb1de6fU, 0xc55491c5U,
+    0x30506030U, 0x01030201U, 0x67a9ce67U, 0x2b7d562bU,
+    0xfe19e7feU, 0xd762b5d7U, 0xabe64dabU, 0x769aec76U,
+    0xca458fcaU, 0x829d1f82U, 0xc94089c9U, 0x7d87fa7dU,
+    0xfa15effaU, 0x59ebb259U, 0x47c98e47U, 0xf00bfbf0U,
+    0xadec41adU, 0xd467b3d4U, 0xa2fd5fa2U, 0xafea45afU,
+    0x9cbf239cU, 0xa4f753a4U, 0x7296e472U, 0xc05b9bc0U,
+    0xb7c275b7U, 0xfd1ce1fdU, 0x93ae3d93U, 0x266a4c26U,
+    0x365a6c36U, 0x3f417e3fU, 0xf702f5f7U, 0xcc4f83ccU,
+    0x345c6834U, 0xa5f451a5U, 0xe534d1e5U, 0xf108f9f1U,
+    0x7193e271U, 0xd873abd8U, 0x31536231U, 0x153f2a15U,
+    0x040c0804U, 0xc75295c7U, 0x23654623U, 0xc35e9dc3U,
+    0x18283018U, 0x96a13796U, 0x050f0a05U, 0x9ab52f9aU,
+    0x07090e07U, 0x12362412U, 0x809b1b80U, 0xe23ddfe2U,
+    0xeb26cdebU, 0x27694e27U, 0xb2cd7fb2U, 0x759fea75U,
+    0x091b1209U, 0x839e1d83U, 0x2c74582cU, 0x1a2e341aU,
+    0x1b2d361bU, 0x6eb2dc6eU, 0x5aeeb45aU, 0xa0fb5ba0U,
+    0x52f6a452U, 0x3b4d763bU, 0xd661b7d6U, 0xb3ce7db3U,
+    0x297b5229U, 0xe33edde3U, 0x2f715e2fU, 0x84971384U,
+    0x53f5a653U, 0xd168b9d1U, 0x00000000U, 0xed2cc1edU,
+    0x20604020U, 0xfc1fe3fcU, 0xb1c879b1U, 0x5bedb65bU,
+    0x6abed46aU, 0xcb468dcbU, 0xbed967beU, 0x394b7239U,
+    0x4ade944aU, 0x4cd4984cU, 0x58e8b058U, 0xcf4a85cfU,
+    0xd06bbbd0U, 0xef2ac5efU, 0xaae54faaU, 0xfb16edfbU,
+    0x43c58643U, 0x4dd79a4dU, 0x33556633U, 0x85941185U,
+    0x45cf8a45U, 0xf910e9f9U, 0x02060402U, 0x7f81fe7fU,
+    0x50f0a050U, 0x3c44783cU, 0x9fba259fU, 0xa8e34ba8U,
+    0x51f3a251U, 0xa3fe5da3U, 0x40c08040U, 0x8f8a058fU,
+    0x92ad3f92U, 0x9dbc219dU, 0x38487038U, 0xf504f1f5U,
+    0xbcdf63bcU, 0xb6c177b6U, 0xda75afdaU, 0x21634221U,
+    0x10302010U, 0xff1ae5ffU, 0xf30efdf3U, 0xd26dbfd2U,
+    0xcd4c81cdU, 0x0c14180cU, 0x13352613U, 0xec2fc3ecU,
+    0x5fe1be5fU, 0x97a23597U, 0x44cc8844U, 0x17392e17U,
+    0xc45793c4U, 0xa7f255a7U, 0x7e82fc7eU, 0x3d477a3dU,
+    0x64acc864U, 0x5de7ba5dU, 0x192b3219U, 0x7395e673U,
+    0x60a0c060U, 0x81981981U, 0x4fd19e4fU, 0xdc7fa3dcU,
+    0x22664422U, 0x2a7e542aU, 0x90ab3b90U, 0x88830b88U,
+    0x46ca8c46U, 0xee29c7eeU, 0xb8d36bb8U, 0x143c2814U,
+    0xde79a7deU, 0x5ee2bc5eU, 0x0b1d160bU, 0xdb76addbU,
+    0xe03bdbe0U, 0x32566432U, 0x3a4e743aU, 0x0a1e140aU,
+    0x49db9249U, 0x060a0c06U, 0x246c4824U, 0x5ce4b85cU,
+    0xc25d9fc2U, 0xd36ebdd3U, 0xacef43acU, 0x62a6c462U,
+    0x91a83991U, 0x95a43195U, 0xe437d3e4U, 0x798bf279U,
+    0xe732d5e7U, 0xc8438bc8U, 0x37596e37U, 0x6db7da6dU,
+    0x8d8c018dU, 0xd564b1d5U, 0x4ed29c4eU, 0xa9e049a9U,
+    0x6cb4d86cU, 0x56faac56U, 0xf407f3f4U, 0xea25cfeaU,
+    0x65afca65U, 0x7a8ef47aU, 0xaee947aeU, 0x08181008U,
+    0xbad56fbaU, 0x7888f078U, 0x256f4a25U, 0x2e725c2eU,
+    0x1c24381cU, 0xa6f157a6U, 0xb4c773b4U, 0xc65197c6U,
+    0xe823cbe8U, 0xdd7ca1ddU, 0x749ce874U, 0x1f213e1fU,
+    0x4bdd964bU, 0xbddc61bdU, 0x8b860d8bU, 0x8a850f8aU,
+    0x7090e070U, 0x3e427c3eU, 0xb5c471b5U, 0x66aacc66U,
+    0x48d89048U, 0x03050603U, 0xf601f7f6U, 0x0e121c0eU,
+    0x61a3c261U, 0x355f6a35U, 0x57f9ae57U, 0xb9d069b9U,
+    0x86911786U, 0xc15899c1U, 0x1d273a1dU, 0x9eb9279eU,
+    0xe138d9e1U, 0xf813ebf8U, 0x98b32b98U, 0x11332211U,
+    0x69bbd269U, 0xd970a9d9U, 0x8e89078eU, 0x94a73394U,
+    0x9bb62d9bU, 0x1e223c1eU, 0x87921587U, 0xe920c9e9U,
+    0xce4987ceU, 0x55ffaa55U, 0x28785028U, 0xdf7aa5dfU,
+    0x8c8f038cU, 0xa1f859a1U, 0x89800989U, 0x0d171a0dU,
+    0xbfda65bfU, 0xe631d7e6U, 0x42c68442U, 0x68b8d068U,
+    0x41c38241U, 0x99b02999U, 0x2d775a2dU, 0x0f111e0fU,
+    0xb0cb7bb0U, 0x54fca854U, 0xbbd66dbbU, 0x163a2c16U,
+};
+static const uint32_t Te3[256] = {
+
+    0x6363a5c6U, 0x7c7c84f8U, 0x777799eeU, 0x7b7b8df6U,
+    0xf2f20dffU, 0x6b6bbdd6U, 0x6f6fb1deU, 0xc5c55491U,
+    0x30305060U, 0x01010302U, 0x6767a9ceU, 0x2b2b7d56U,
+    0xfefe19e7U, 0xd7d762b5U, 0xababe64dU, 0x76769aecU,
+    0xcaca458fU, 0x82829d1fU, 0xc9c94089U, 0x7d7d87faU,
+    0xfafa15efU, 0x5959ebb2U, 0x4747c98eU, 0xf0f00bfbU,
+    0xadadec41U, 0xd4d467b3U, 0xa2a2fd5fU, 0xafafea45U,
+    0x9c9cbf23U, 0xa4a4f753U, 0x727296e4U, 0xc0c05b9bU,
+    0xb7b7c275U, 0xfdfd1ce1U, 0x9393ae3dU, 0x26266a4cU,
+    0x36365a6cU, 0x3f3f417eU, 0xf7f702f5U, 0xcccc4f83U,
+    0x34345c68U, 0xa5a5f451U, 0xe5e534d1U, 0xf1f108f9U,
+    0x717193e2U, 0xd8d873abU, 0x31315362U, 0x15153f2aU,
+    0x04040c08U, 0xc7c75295U, 0x23236546U, 0xc3c35e9dU,
+    0x18182830U, 0x9696a137U, 0x05050f0aU, 0x9a9ab52fU,
+    0x0707090eU, 0x12123624U, 0x80809b1bU, 0xe2e23ddfU,
+    0xebeb26cdU, 0x2727694eU, 0xb2b2cd7fU, 0x75759feaU,
+    0x09091b12U, 0x83839e1dU, 0x2c2c7458U, 0x1a1a2e34U,
+    0x1b1b2d36U, 0x6e6eb2dcU, 0x5a5aeeb4U, 0xa0a0fb5bU,
+    0x5252f6a4U, 0x3b3b4d76U, 0xd6d661b7U, 0xb3b3ce7dU,
+    0x29297b52U, 0xe3e33eddU, 0x2f2f715eU, 0x84849713U,
+    0x5353f5a6U, 0xd1d168b9U, 0x00000000U, 0xeded2cc1U,
+    0x20206040U, 0xfcfc1fe3U, 0xb1b1c879U, 0x5b5bedb6U,
+    0x6a6abed4U, 0xcbcb468dU, 0xbebed967U, 0x39394b72U,
+    0x4a4ade94U, 0x4c4cd498U, 0x5858e8b0U, 0xcfcf4a85U,
+    0xd0d06bbbU, 0xefef2ac5U, 0xaaaae54fU, 0xfbfb16edU,
+    0x4343c586U, 0x4d4dd79aU, 0x33335566U, 0x85859411U,
+    0x4545cf8aU, 0xf9f910e9U, 0x02020604U, 0x7f7f81feU,
+    0x5050f0a0U, 0x3c3c4478U, 0x9f9fba25U, 0xa8a8e34bU,
+    0x5151f3a2U, 0xa3a3fe5dU, 0x4040c080U, 0x8f8f8a05U,
+    0x9292ad3fU, 0x9d9dbc21U, 0x38384870U, 0xf5f504f1U,
+    0xbcbcdf63U, 0xb6b6c177U, 0xdada75afU, 0x21216342U,
+    0x10103020U, 0xffff1ae5U, 0xf3f30efdU, 0xd2d26dbfU,
+    0xcdcd4c81U, 0x0c0c1418U, 0x13133526U, 0xecec2fc3U,
+    0x5f5fe1beU, 0x9797a235U, 0x4444cc88U, 0x1717392eU,
+    0xc4c45793U, 0xa7a7f255U, 0x7e7e82fcU, 0x3d3d477aU,
+    0x6464acc8U, 0x5d5de7baU, 0x19192b32U, 0x737395e6U,
+    0x6060a0c0U, 0x81819819U, 0x4f4fd19eU, 0xdcdc7fa3U,
+    0x22226644U, 0x2a2a7e54U, 0x9090ab3bU, 0x8888830bU,
+    0x4646ca8cU, 0xeeee29c7U, 0xb8b8d36bU, 0x14143c28U,
+    0xdede79a7U, 0x5e5ee2bcU, 0x0b0b1d16U, 0xdbdb76adU,
+    0xe0e03bdbU, 0x32325664U, 0x3a3a4e74U, 0x0a0a1e14U,
+    0x4949db92U, 0x06060a0cU, 0x24246c48U, 0x5c5ce4b8U,
+    0xc2c25d9fU, 0xd3d36ebdU, 0xacacef43U, 0x6262a6c4U,
+    0x9191a839U, 0x9595a431U, 0xe4e437d3U, 0x79798bf2U,
+    0xe7e732d5U, 0xc8c8438bU, 0x3737596eU, 0x6d6db7daU,
+    0x8d8d8c01U, 0xd5d564b1U, 0x4e4ed29cU, 0xa9a9e049U,
+    0x6c6cb4d8U, 0x5656faacU, 0xf4f407f3U, 0xeaea25cfU,
+    0x6565afcaU, 0x7a7a8ef4U, 0xaeaee947U, 0x08081810U,
+    0xbabad56fU, 0x787888f0U, 0x25256f4aU, 0x2e2e725cU,
+    0x1c1c2438U, 0xa6a6f157U, 0xb4b4c773U, 0xc6c65197U,
+    0xe8e823cbU, 0xdddd7ca1U, 0x74749ce8U, 0x1f1f213eU,
+    0x4b4bdd96U, 0xbdbddc61U, 0x8b8b860dU, 0x8a8a850fU,
+    0x707090e0U, 0x3e3e427cU, 0xb5b5c471U, 0x6666aaccU,
+    0x4848d890U, 0x03030506U, 0xf6f601f7U, 0x0e0e121cU,
+    0x6161a3c2U, 0x35355f6aU, 0x5757f9aeU, 0xb9b9d069U,
+    0x86869117U, 0xc1c15899U, 0x1d1d273aU, 0x9e9eb927U,
+    0xe1e138d9U, 0xf8f813ebU, 0x9898b32bU, 0x11113322U,
+    0x6969bbd2U, 0xd9d970a9U, 0x8e8e8907U, 0x9494a733U,
+    0x9b9bb62dU, 0x1e1e223cU, 0x87879215U, 0xe9e920c9U,
+    0xcece4987U, 0x5555ffaaU, 0x28287850U, 0xdfdf7aa5U,
+    0x8c8c8f03U, 0xa1a1f859U, 0x89898009U, 0x0d0d171aU,
+    0xbfbfda65U, 0xe6e631d7U, 0x4242c684U, 0x6868b8d0U,
+    0x4141c382U, 0x9999b029U, 0x2d2d775aU, 0x0f0f111eU,
+    0xb0b0cb7bU, 0x5454fca8U, 0xbbbbd66dU, 0x16163a2cU,
+};
+static const uint32_t Te4[256] = {
+    0x63636363U, 0x7c7c7c7cU, 0x77777777U, 0x7b7b7b7bU,
+    0xf2f2f2f2U, 0x6b6b6b6bU, 0x6f6f6f6fU, 0xc5c5c5c5U,
+    0x30303030U, 0x01010101U, 0x67676767U, 0x2b2b2b2bU,
+    0xfefefefeU, 0xd7d7d7d7U, 0xababababU, 0x76767676U,
+    0xcacacacaU, 0x82828282U, 0xc9c9c9c9U, 0x7d7d7d7dU,
+    0xfafafafaU, 0x59595959U, 0x47474747U, 0xf0f0f0f0U,
+    0xadadadadU, 0xd4d4d4d4U, 0xa2a2a2a2U, 0xafafafafU,
+    0x9c9c9c9cU, 0xa4a4a4a4U, 0x72727272U, 0xc0c0c0c0U,
+    0xb7b7b7b7U, 0xfdfdfdfdU, 0x93939393U, 0x26262626U,
+    0x36363636U, 0x3f3f3f3fU, 0xf7f7f7f7U, 0xccccccccU,
+    0x34343434U, 0xa5a5a5a5U, 0xe5e5e5e5U, 0xf1f1f1f1U,
+    0x71717171U, 0xd8d8d8d8U, 0x31313131U, 0x15151515U,
+    0x04040404U, 0xc7c7c7c7U, 0x23232323U, 0xc3c3c3c3U,
+    0x18181818U, 0x96969696U, 0x05050505U, 0x9a9a9a9aU,
+    0x07070707U, 0x12121212U, 0x80808080U, 0xe2e2e2e2U,
+    0xebebebebU, 0x27272727U, 0xb2b2b2b2U, 0x75757575U,
+    0x09090909U, 0x83838383U, 0x2c2c2c2cU, 0x1a1a1a1aU,
+    0x1b1b1b1bU, 0x6e6e6e6eU, 0x5a5a5a5aU, 0xa0a0a0a0U,
+    0x52525252U, 0x3b3b3b3bU, 0xd6d6d6d6U, 0xb3b3b3b3U,
+    0x29292929U, 0xe3e3e3e3U, 0x2f2f2f2fU, 0x84848484U,
+    0x53535353U, 0xd1d1d1d1U, 0x00000000U, 0xededededU,
+    0x20202020U, 0xfcfcfcfcU, 0xb1b1b1b1U, 0x5b5b5b5bU,
+    0x6a6a6a6aU, 0xcbcbcbcbU, 0xbebebebeU, 0x39393939U,
+    0x4a4a4a4aU, 0x4c4c4c4cU, 0x58585858U, 0xcfcfcfcfU,
+    0xd0d0d0d0U, 0xefefefefU, 0xaaaaaaaaU, 0xfbfbfbfbU,
+    0x43434343U, 0x4d4d4d4dU, 0x33333333U, 0x85858585U,
+    0x45454545U, 0xf9f9f9f9U, 0x02020202U, 0x7f7f7f7fU,
+    0x50505050U, 0x3c3c3c3cU, 0x9f9f9f9fU, 0xa8a8a8a8U,
+    0x51515151U, 0xa3a3a3a3U, 0x40404040U, 0x8f8f8f8fU,
+    0x92929292U, 0x9d9d9d9dU, 0x38383838U, 0xf5f5f5f5U,
+    0xbcbcbcbcU, 0xb6b6b6b6U, 0xdadadadaU, 0x21212121U,
+    0x10101010U, 0xffffffffU, 0xf3f3f3f3U, 0xd2d2d2d2U,
+    0xcdcdcdcdU, 0x0c0c0c0cU, 0x13131313U, 0xececececU,
+    0x5f5f5f5fU, 0x97979797U, 0x44444444U, 0x17171717U,
+    0xc4c4c4c4U, 0xa7a7a7a7U, 0x7e7e7e7eU, 0x3d3d3d3dU,
+    0x64646464U, 0x5d5d5d5dU, 0x19191919U, 0x73737373U,
+    0x60606060U, 0x81818181U, 0x4f4f4f4fU, 0xdcdcdcdcU,
+    0x22222222U, 0x2a2a2a2aU, 0x90909090U, 0x88888888U,
+    0x46464646U, 0xeeeeeeeeU, 0xb8b8b8b8U, 0x14141414U,
+    0xdedededeU, 0x5e5e5e5eU, 0x0b0b0b0bU, 0xdbdbdbdbU,
+    0xe0e0e0e0U, 0x32323232U, 0x3a3a3a3aU, 0x0a0a0a0aU,
+    0x49494949U, 0x06060606U, 0x24242424U, 0x5c5c5c5cU,
+    0xc2c2c2c2U, 0xd3d3d3d3U, 0xacacacacU, 0x62626262U,
+    0x91919191U, 0x95959595U, 0xe4e4e4e4U, 0x79797979U,
+    0xe7e7e7e7U, 0xc8c8c8c8U, 0x37373737U, 0x6d6d6d6dU,
+    0x8d8d8d8dU, 0xd5d5d5d5U, 0x4e4e4e4eU, 0xa9a9a9a9U,
+    0x6c6c6c6cU, 0x56565656U, 0xf4f4f4f4U, 0xeaeaeaeaU,
+    0x65656565U, 0x7a7a7a7aU, 0xaeaeaeaeU, 0x08080808U,
+    0xbabababaU, 0x78787878U, 0x25252525U, 0x2e2e2e2eU,
+    0x1c1c1c1cU, 0xa6a6a6a6U, 0xb4b4b4b4U, 0xc6c6c6c6U,
+    0xe8e8e8e8U, 0xddddddddU, 0x74747474U, 0x1f1f1f1fU,
+    0x4b4b4b4bU, 0xbdbdbdbdU, 0x8b8b8b8bU, 0x8a8a8a8aU,
+    0x70707070U, 0x3e3e3e3eU, 0xb5b5b5b5U, 0x66666666U,
+    0x48484848U, 0x03030303U, 0xf6f6f6f6U, 0x0e0e0e0eU,
+    0x61616161U, 0x35353535U, 0x57575757U, 0xb9b9b9b9U,
+    0x86868686U, 0xc1c1c1c1U, 0x1d1d1d1dU, 0x9e9e9e9eU,
+    0xe1e1e1e1U, 0xf8f8f8f8U, 0x98989898U, 0x11111111U,
+    0x69696969U, 0xd9d9d9d9U, 0x8e8e8e8eU, 0x94949494U,
+    0x9b9b9b9bU, 0x1e1e1e1eU, 0x87878787U, 0xe9e9e9e9U,
+    0xcecececeU, 0x55555555U, 0x28282828U, 0xdfdfdfdfU,
+    0x8c8c8c8cU, 0xa1a1a1a1U, 0x89898989U, 0x0d0d0d0dU,
+    0xbfbfbfbfU, 0xe6e6e6e6U, 0x42424242U, 0x68686868U,
+    0x41414141U, 0x99999999U, 0x2d2d2d2dU, 0x0f0f0f0fU,
+    0xb0b0b0b0U, 0x54545454U, 0xbbbbbbbbU, 0x16161616U,
+};
+static const uint32_t rcon[] = {
+    0x01000000, 0x02000000, 0x04000000, 0x08000000,
+    0x10000000, 0x20000000, 0x40000000, 0x80000000,
+    0x1B000000, 0x36000000, /* for 128-bit blocks, Rijndael never uses more than 10 rcon values */
+};
+
+#define GETU32(pt) (((uint32_t)(pt)[0] << 24) ^ ((uint32_t)(pt)[1] << 16) ^ ((uint32_t)(pt)[2] <<  8) ^ ((uint32_t)(pt)[3]))
+#define PUTU32(ct, st) { (ct)[0] = (uint8_t)((st) >> 24); (ct)[1] = (uint8_t)((st) >> 16); (ct)[2] = (uint8_t)((st) >>  8); (ct)[3] = (uint8_t)(st); }
+
+/**
+ * Expand the cipher key into the encryption key schedule.
+ *
+ * @return  the number of rounds for the given cipher key size.
+ */
+int rijndaelKeySetupEnc128(uint32_t rk[44], const uint8_t cipherKey[] ) {
+    int i = 0;
+    uint32_t temp;
+
+    rk[0] = GETU32(cipherKey     );
+    rk[1] = GETU32(cipherKey +  4);
+    rk[2] = GETU32(cipherKey +  8);
+    rk[3] = GETU32(cipherKey + 12);
+    for (;;) {
+        temp  = rk[3];
+        rk[4] = rk[0] ^
+            (Te4[(temp >> 16) & 0xff] & 0xff000000) ^
+            (Te4[(temp >>  8) & 0xff] & 0x00ff0000) ^
+            (Te4[(temp      ) & 0xff] & 0x0000ff00) ^
+            (Te4[(temp >> 24)       ] & 0x000000ff) ^
+            rcon[i];
+        rk[5] = rk[1] ^ rk[4];
+        rk[6] = rk[2] ^ rk[5];
+        rk[7] = rk[3] ^ rk[6];
+        if (++i == 10) {
+            return 10;
+        }
+        rk += 4;
+    }
+}
+
+void rijndaelEncrypt128(const uint32_t rk[44], const uint8_t pt[16], uint8_t ct[16]) {
+    uint32_t s0, s1, s2, s3, t0, t1, t2, t3;
+
+    /*
+     * map byte array block to cipher state
+     * and add initial round key:
+     */
+    s0 = GETU32(pt     ) ^ rk[0];
+    s1 = GETU32(pt +  4) ^ rk[1];
+    s2 = GETU32(pt +  8) ^ rk[2];
+    s3 = GETU32(pt + 12) ^ rk[3];
+    /* round 1: */
+    t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >>  8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[ 4];
+    t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >>  8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[ 5];
+    t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >>  8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[ 6];
+    t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >>  8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[ 7];
+    /* round 2: */
+    s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >>  8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[ 8];
+    s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >>  8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[ 9];
+    s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >>  8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[10];
+    s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >>  8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[11];
+    /* round 3: */
+    t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >>  8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[12];
+    t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >>  8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[13];
+    t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >>  8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[14];
+    t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >>  8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[15];
+    /* round 4: */
+    s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >>  8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[16];
+    s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >>  8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[17];
+    s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >>  8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[18];
+    s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >>  8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[19];
+    /* round 5: */
+    t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >>  8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[20];
+    t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >>  8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[21];
+    t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >>  8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[22];
+    t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >>  8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[23];
+    /* round 6: */
+    s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >>  8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[24];
+    s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >>  8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[25];
+    s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >>  8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[26];
+    s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >>  8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[27];
+    /* round 7: */
+    t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >>  8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[28];
+    t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >>  8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[29];
+    t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >>  8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[30];
+    t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >>  8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[31];
+    /* round 8: */
+    s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >>  8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[32];
+    s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >>  8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[33];
+    s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >>  8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[34];
+    s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >>  8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[35];
+    /* round 9: */
+    t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >>  8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[36];
+    t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >>  8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[37];
+    t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >>  8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[38];
+    t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >>  8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[39];
+
+    /*
+     * apply last round and
+     * map cipher state to byte array block:
+     */
+    s0 =
+        (Te4[(t0 >> 24)       ] & 0xff000000) ^
+        (Te4[(t1 >> 16) & 0xff] & 0x00ff0000) ^
+        (Te4[(t2 >>  8) & 0xff] & 0x0000ff00) ^
+        (Te4[(t3      ) & 0xff] & 0x000000ff) ^
+        rk[40];
+    PUTU32(ct     , s0);
+    s1 =
+        (Te4[(t1 >> 24)       ] & 0xff000000) ^
+        (Te4[(t2 >> 16) & 0xff] & 0x00ff0000) ^
+        (Te4[(t3 >>  8) & 0xff] & 0x0000ff00) ^
+        (Te4[(t0      ) & 0xff] & 0x000000ff) ^
+        rk[41];
+    PUTU32(ct +  4, s1);
+    s2 =
+        (Te4[(t2 >> 24)       ] & 0xff000000) ^
+        (Te4[(t3 >> 16) & 0xff] & 0x00ff0000) ^
+        (Te4[(t0 >>  8) & 0xff] & 0x0000ff00) ^
+        (Te4[(t1      ) & 0xff] & 0x000000ff) ^
+        rk[42];
+    PUTU32(ct +  8, s2);
+    s3 =
+        (Te4[(t3 >> 24)       ] & 0xff000000) ^
+        (Te4[(t0 >> 16) & 0xff] & 0x00ff0000) ^
+        (Te4[(t1 >>  8) & 0xff] & 0x0000ff00) ^
+        (Te4[(t2      ) & 0xff] & 0x000000ff) ^
+        rk[43];
+    PUTU32(ct + 12, s3);
+}

ot_rijndael.h

@@ -0,0 +1,22 @@
+/* This software was written by Dirk Engling <erdgeist@erdgeist.org>
+   It is considered beerware. Prost. Skol. Cheers or whatever.
+
+   The rijndael implementation was taken from
+
+     http://www.cs.ucdavis.edu/~rogaway/ocb/ocb-ref/rijndael-alg-fst.c
+
+   and modified to work with 128 bits (this is 10 rounds) only.
+
+   $id$ */
+
+#ifndef OT_RIJNDAEL_H__
+#define OT_RIJNDAEL_H__
+
+#include <stdint.h>
+
+int rijndaelKeySetupEnc128(uint32_t rk[44], const uint8_t cipherKey[] );
+void rijndaelEncrypt128(const uint32_t rk[44], const uint8_t pt[16], uint8_t ct[16]);
+
+extern const char *g_version_rijndael_c;
+
+#endif

ot_stats.h

@@ -3,24 +3,28 @@
 
    $id$ */
 
-#ifndef __OT_STATS_H__
-#define __OT_STATS_H__
+#ifndef OT_STATS_H__
+#define OT_STATS_H__
+
+#include "trackerlogic.h"
 
 typedef enum {
   EVENT_ACCEPT,
   EVENT_READ,
-  EVENT_CONNECT,      /* UDP only */
+  EVENT_CONNECT, /* UDP only */
   EVENT_ANNOUNCE,
+  EVENT_COMPLETED,
   EVENT_RENEW,
+  EVENT_SYNC,
   EVENT_SCRAPE,
   EVENT_FULLSCRAPE_REQUEST,
   EVENT_FULLSCRAPE_REQUEST_GZIP,
-  EVENT_FULLSCRAPE,   /* TCP only */
-  EVENT_SYNC_IN_REQUEST,
-  EVENT_SYNC_IN,
-  EVENT_SYNC_OUT_REQUEST,
-  EVENT_SYNC_OUT,
-  EVENT_FAILED
+  EVENT_FULLSCRAPE_REQUEST_ZSTD,
+  EVENT_FULLSCRAPE, /* TCP only */
+  EVENT_FAILED,
+  EVENT_BUCKET_LOCKED,
+  EVENT_WOODPECKER,
+  EVENT_CONNID_MISSMATCH
 } ot_status_event;
 
 enum {
@@ -28,6 +32,8 @@ enum {
   CODE_HTTPERROR_400,
   CODE_HTTPERROR_400_PARAM,
   CODE_HTTPERROR_400_COMPACT,
+  CODE_HTTPERROR_402_NOTMODEST,
+  CODE_HTTPERROR_402_PAYMENT_REQUIRED,
   CODE_HTTPERROR_403_IP,
   CODE_HTTPERROR_404,
   CODE_HTTPERROR_500,
@@ -35,11 +41,12 @@ enum {
   CODE_HTTPERROR_COUNT
 };
 
-void   stats_issue_event( ot_status_event event, PROTO_FLAG proto, uint32_t event_data );
-void   stats_deliver( int64 socket, int tasktype );
-size_t return_stats_for_tracker( char *reply, int mode, int format );
-size_t stats_return_tracker_version( char *reply );
-void   stats_init( );
-void   stats_deinit( );
+void               stats_issue_event(ot_status_event event, PROTO_FLAG proto, uintptr_t event_data);
+void               stats_deliver(int64 sock, int tasktype);
+void               stats_cleanup(void);
+size_t             return_stats_for_tracker(char *reply, int mode, int format);
+size_t             stats_return_tracker_version(char *reply);
+void               stats_init(void);
+void               stats_deinit(void);
 
 #endif

ot_sync.c

@@ -4,64 +4,66 @@
    $id$ */
 
 /* System */
-#include <sys/types.h>
-#include <sys/mman.h>
-#include <sys/uio.h>
+#include <pthread.h>
 #include <stdio.h>
 #include <string.h>
-#include <pthread.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <sys/uio.h>
 
 /* Libowfat */
-#include "scan.h"
 #include "byte.h"
 #include "io.h"
+#include "scan.h"
 
 /* Opentracker */
-#include "trackerlogic.h"
-#include "ot_mutex.h"
-#include "ot_sync.h"
-#include "ot_stats.h"
 #include "ot_iovec.h"
+#include "ot_mutex.h"
+#include "ot_stats.h"
+#include "ot_sync.h"
+#include "trackerlogic.h"
 
 #ifdef WANT_SYNC_BATCH
 
-#define OT_SYNC_CHUNK_SIZE (512*1024)
+#define OT_SYNC_CHUNK_SIZE (512 * 1024)
 
 /* Import Changeset from an external authority
    format: d4:syncd[..]ee
    [..]:   ( 20:01234567890abcdefghij16:XXXXYYYY )+
 */
-int add_changeset_to_tracker( uint8_t *data, size_t len ) {
-  ot_hash    *hash;
-  uint8_t    *end = data + len;
-  unsigned long      peer_count;
+int add_changeset_to_tracker(uint8_t *data, size_t len) {
+  ot_hash      *hash;
+  uint8_t      *end = data + len;
+  unsigned long peer_count;
 
   /* We do know, that the string is \n terminated, so it cant
      overflow */
-  if( byte_diff( data, 8, "d4:syncd" ) ) return -1;
+  if (byte_diff(data, 8, "d4:syncd"))
+    return -1;
   data += 8;
 
-  while( 1 ) {
-    if( byte_diff( data, 3, "20:" ) ) {
-      if( byte_diff( data, 2, "ee" ) )
+  while (1) {
+    if (byte_diff(data, 3, "20:")) {
+      if (byte_diff(data, 2, "ee"))
         return -1;
       return 0;
     }
     data += 3;
-    hash = (ot_hash*)data;
-    data += sizeof( ot_hash );
+    hash  = (ot_hash *)data;
+    data += sizeof(ot_hash);
 
     /* Scan string length indicator */
-    data += ( len = scan_ulong( (char*)data, &peer_count ) );
+    data += (len = scan_ulong((char *)data, &peer_count));
 
     /* If no long was scanned, it is not divisible by 8, it is not
        followed by a colon or claims to need to much memory, we fail */
-    if( !len || !peer_count || ( peer_count & 7 ) || ( *data++ != ':' ) || ( data + peer_count > end ) )
+    if (!len || !peer_count || (peer_count & 7) || (*data++ != ':') || (data + peer_count > end))
       return -1;
 
-    while( peer_count > 0 ) {
-      add_peer_to_torrent( hash, (ot_peer*)data, 1 );
-      data += 8; peer_count -= 8;
+    while (peer_count > 0) {
+      add_peer_to_torrent(hash, (ot_peer *)data, 1);
+      data       += 8;
+      peer_count -= 8;
     }
   }
   return 0;
@@ -70,80 +72,86 @@ int add_changeset_to_tracker( uint8_t *data, size_t len ) {
 /* Proposed output format
    d4:syncd20:<info_hash>8*N:(xxxxyyyy)*Nee
 */
-static void sync_make( int *iovec_entries, struct iovec **iovector ) {
-  int    bucket;
-  char  *r, *re;
+static void sync_make(int *iovec_entries, struct iovec **iovector) {
+  int   bucket;
+  char *r, *re;
 
   /* Setup return vector... */
   *iovec_entries = 0;
-  *iovector = NULL;
-  if( !( r = iovec_increase( iovec_entries, iovector, OT_SYNC_CHUNK_SIZE ) ) )
+  *iovector      = NULL;
+  if (!(r = iovec_increase(iovec_entries, iovector, OT_SYNC_CHUNK_SIZE)))
     return;
 
   /* ... and pointer to end of current output buffer.
      This works as a low watermark */
   re = r + OT_SYNC_CHUNK_SIZE;
 
-  memmove( r, "d4:syncd", 8 ); r += 8;
+  memmove(r, "d4:syncd", 8);
+  r += 8;
 
   /* For each bucket... */
-  for( bucket=0; bucket<OT_BUCKET_COUNT; ++bucket ) {
+  for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) {
     /* Get exclusive access to that bucket */
-    ot_vector *torrents_list = mutex_bucket_lock( bucket );
-    size_t tor_offset;
+    ot_vector *torrents_list = mutex_bucket_lock(bucket);
+    size_t     tor_offset;
 
     /* For each torrent in this bucket.. */
-    for( tor_offset=0; tor_offset<torrents_list->size; ++tor_offset ) {
+    for (tor_offset = 0; tor_offset < torrents_list->size; ++tor_offset) {
       /* Address torrents members */
-      ot_peerlist *peer_list = ( ((ot_torrent*)(torrents_list->data))[tor_offset] ).peer_list;
-      ot_hash     *hash      =&( ((ot_torrent*)(torrents_list->data))[tor_offset] ).hash;
+      ot_peerlist *peer_list  = (((ot_torrent *)(torrents_list->data))[tor_offset]).peer_list;
+      ot_hash     *hash       = &(((ot_torrent *)(torrents_list->data))[tor_offset]).hash;
       const size_t byte_count = sizeof(ot_peer) * peer_list->changeset.size;
 
       /* If we reached our low watermark in buffer... */
-      if( re - r <= (ssize_t)(/* strlen( "20:" ) == */ 3 + sizeof( ot_hash ) + /* strlen_max( "%zd" ) == */ 12 + byte_count ) ) {
+      if (re - r <= (ssize_t)(/* strlen( "20:" ) == */ 3 + sizeof(ot_hash) + /* strlen_max( "%zd" ) == */ 12 + byte_count)) {
 
         /* Allocate a fresh output buffer at the end of our buffers list
            release bucket and return, if that fails */
-        if( !( r = iovec_fix_increase_or_free( iovec_entries, iovector, r, OT_SYNC_CHUNK_SIZE ) ) )
-          return mutex_bucket_unlock( bucket );
+        if (!(r = iovec_fix_increase_or_free(iovec_entries, iovector, r, OT_SYNC_CHUNK_SIZE)))
+          return mutex_bucket_unlock(bucket);
 
         /* Adjust new end of output buffer */
         re = r + OT_SYNC_CHUNK_SIZE;
       }
 
-      *r++ = '2'; *r++ = '0'; *r++ = ':';
-      memmove( r, hash, sizeof( ot_hash ) ); r += sizeof( ot_hash );
-      r += sprintf( r, "%zd:", byte_count );
-      memmove( r, peer_list->changeset.data, byte_count ); r += byte_count;
+      *r++ = '2';
+      *r++ = '0';
+      *r++ = ':';
+      memmove(r, hash, sizeof(ot_hash));
+      r += sizeof(ot_hash);
+      r += sprintf(r, "%zd:", byte_count);
+      memmove(r, peer_list->changeset.data, byte_count);
+      r += byte_count;
     }
 
     /* All torrents done: release lock on currenct bucket */
-    mutex_bucket_unlock( bucket );
+    mutex_bucket_unlock(bucket);
   }
 
   /* Close bencoded sync dictionary */
-  *r++='e'; *r++='e';
+  *r++ = 'e';
+  *r++ = 'e';
 
   /* Release unused memory in current output buffer */
-  iovec_fixlast( iovec_entries, iovector, r );
+  iovec_fixlast(iovec_entries, iovector, r);
 }
 
 /* This is the entry point into this worker thread
    It grabs tasks from mutex_tasklist and delivers results back
 */
-static void * sync_worker( void * args) {
-  int iovec_entries;
+static void *sync_worker(void *args) {
+  int           iovec_entries;
   struct iovec *iovector;
 
   args = args;
 
-  while( 1 ) {
+  while (1) {
     ot_tasktype tasktype = TASK_SYNC_OUT;
-    ot_taskid   taskid   = mutex_workqueue_poptask( &tasktype );
-    sync_make( &iovec_entries, &iovector );
-    stats_issue_event( EVENT_SYNC_OUT, FLAG_TCP, iovec_length( &iovec_entries, &iovector) );
-    if( mutex_workqueue_pushresult( taskid, iovec_entries, iovector ) )
-      iovec_free( &iovec_entries, &iovector );
+    ot_taskid   taskid   = mutex_workqueue_poptask(&tasktype);
+    sync_make(&iovec_entries, &iovector);
+    stats_issue_event(EVENT_SYNC_OUT, FLAG_TCP, iovec_length(&iovec_entries, &iovector));
+    if (mutex_workqueue_pushresult(taskid, iovec_entries, iovector))
+      iovec_free(&iovec_entries, &iovector);
   }
   return NULL;
 }
@@ -162,5 +170,3 @@ void sync_deliver( int64 socket ) {
 }
 
 #endif
-
-const char *g_version_sync_c = "$Source$: $Revision$\n";

ot_sync.h

@@ -3,17 +3,17 @@
 
    $id$ */
 
-#ifndef __OT_SYNC_H__
-#define __OT_SYNC_H__
+#ifndef OT_SYNC_H__
+#define OT_SYNC_H__
 
 #ifdef WANT_SYNC_BATCH
 enum { SYNC_IN, SYNC_OUT };
 
-void sync_init( );
-void sync_deinit( );
-void sync_deliver( int64 socket );
+void sync_init();
+void sync_deinit();
+void sync_deliver(int64 socket);
 
-int  add_changeset_to_tracker( uint8_t *data, size_t len );
+int  add_changeset_to_tracker(uint8_t *data, size_t len);
 #else
 
 #define sync_init()

ot_udp.c

@@ -4,138 +4,233 @@
    $id$ */
 
 /* System */
-#include <string.h>
 #include <arpa/inet.h>
+#include <pthread.h>
 #include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
 
 /* Libowfat */
-#include "socket.h"
 #include "io.h"
+#include "ip6.h"
+#include "socket.h"
 
 /* Opentracker */
-#include "trackerlogic.h"
-#include "ot_udp.h"
+#include "ot_rijndael.h"
 #include "ot_stats.h"
+#include "ot_udp.h"
+#include "trackerlogic.h"
 
-static char static_inbuf[8192];
-static char static_outbuf[8192];
-
+#if 0
 static const uint8_t g_static_connid[8] = { 0x23, 0x42, 0x05, 0x17, 0xde, 0x41, 0x50, 0xff };
+#endif
+static uint32_t g_rijndael_round_key[44] = {0};
+static uint32_t g_key_of_the_hour[2]     = {0};
+static ot_time  g_hour_of_the_key;
 
-static void udp_make_connectionid( uint32_t * connid, const char * remoteip ) {
-  /* Touch unused variable */
-  (void)remoteip;
+static void     udp_generate_rijndael_round_key() {
+  uint32_t key[16];
+#ifdef WANT_ARC4RANDOM
+  arc4random_buf(&key[0], sizeof(key));
+#else
+  key[0] = random();
+  key[1] = random();
+  key[2] = random();
+  key[3] = random();
+#endif
+  rijndaelKeySetupEnc128(g_rijndael_round_key, (uint8_t *)key);
 
-  /* Use a static secret for now */
-  memcpy( connid, g_static_connid, 8 );
+#ifdef WANT_ARC4RANDOM
+  g_key_of_the_hour[0] = arc4random();
+#else
+  g_key_of_the_hour[0] = random();
+#endif
+  g_hour_of_the_key = g_now_minutes;
 }
 
-static int udp_test_connectionid( const uint32_t * const connid, const char * remoteip ) {
-  /* Touch unused variable */
-  (void)remoteip;
+/* Generate current and previous connection id for ip */
+static void udp_make_connectionid(uint32_t connid[2], const ot_ip6 remoteip, int age) {
+  uint32_t plain[4], crypt[4];
+  int      i;
+  if (g_now_minutes + 60 > g_hour_of_the_key) {
+    g_hour_of_the_key    = g_now_minutes;
+    g_key_of_the_hour[1] = g_key_of_the_hour[0];
+#ifdef WANT_ARC4RANDOM
+    g_key_of_the_hour[0] = arc4random();
+#else
+    g_key_of_the_hour[0] = random();
+#endif
+  }
 
-  /* Test against our static secret */
-  return !memcmp( connid, g_static_connid, 8 );
+  memcpy(plain, remoteip, sizeof(plain));
+  for (i = 0; i < 4; ++i)
+    plain[i] ^= g_key_of_the_hour[age];
+  rijndaelEncrypt128(g_rijndael_round_key, (uint8_t *)remoteip, (uint8_t *)crypt);
+  connid[0] = crypt[0] ^ crypt[1];
+  connid[1] = crypt[2] ^ crypt[3];
 }
 
 /* UDP implementation according to http://xbtt.sourceforge.net/udp_tracker_protocol.html */
-void handle_udp4( int64 serversocket ) {
-  ot_peer     peer;
-  ot_torrent *torrent;
-  ot_hash    *hash = NULL;
-  char        remoteip[4];
-  uint32_t   *inpacket = (uint32_t*)static_inbuf;
-  uint32_t   *outpacket = (uint32_t*)static_outbuf;
-  uint32_t    numwant, left, event;
-  uint16_t    port, remoteport;
-  size_t      r, r_out;
+int handle_udp6(int64 serversocket, struct ot_workstruct *ws) {
+  ot_ip6    remoteip;
+  uint32_t *inpacket  = (uint32_t *)ws->inbuf;
+  uint32_t *outpacket = (uint32_t *)ws->outbuf;
+  uint32_t  left, event, scopeid;
+  uint32_t  connid[2];
+  uint32_t  action;
+  uint16_t  port, remoteport;
+  size_t    byte_count, scrape_count;
 
-  r = socket_recv4( serversocket, static_inbuf, sizeof( static_inbuf ), remoteip, &remoteport);
+  byte_count = socket_recv6(serversocket, ws->inbuf, G_INBUF_SIZE, remoteip, &remoteport, &scopeid);
+  if (!byte_count)
+    return 0;
 
-  stats_issue_event( EVENT_ACCEPT, FLAG_UDP, ntohl(*(uint32_t*)remoteip) );
-  stats_issue_event( EVENT_READ, FLAG_UDP, r );
+  stats_issue_event(EVENT_ACCEPT, FLAG_UDP, (uintptr_t)remoteip);
+  stats_issue_event(EVENT_READ, FLAG_UDP, byte_count);
 
   /* Minimum udp tracker packet size, also catches error */
-  if( r < 16 )
-    return;
+  if (byte_count < 16)
+    return 1;
 
-/*  fprintf( stderr, "UDP Connection id: %16llX\n", *(uint64_t*)inpacket ); */
+  /* Get action to take. Ignore error messages and broken packets */
+  action = ntohl(inpacket[2]);
+  if (action > 2)
+    return 1;
 
-  switch( ntohl( inpacket[2] ) ) {
-    case 0: /* This is a connect action */
-      /* look for udp bittorrent magic id */
-      if( (ntohl(inpacket[0]) != 0x00000417) || (ntohl(inpacket[1]) != 0x27101980) )
-        return;
+  /* Generate the connection id we give out and expect to and from
+     the requesting ip address, this prevents udp spoofing */
+  udp_make_connectionid(connid, remoteip, 0);
 
-      outpacket[0] = 0;
-      outpacket[1] = inpacket[3];
-      udp_make_connectionid( outpacket + 2, remoteip );
+  /* Initialise hash pointer */
+  ws->hash    = NULL;
+  ws->peer_id = NULL;
 
-      socket_send4( serversocket, static_outbuf, 16, remoteip, remoteport );
-      stats_issue_event( EVENT_CONNECT, FLAG_UDP, 16 );
-      break;
-    case 1: /* This is an announce action */
-      /* Minimum udp announce packet size */
-      if( r < 98 )
-        return;
-
-        if( !udp_test_connectionid( inpacket, remoteip ))
-        fprintf( stderr, "UDP connect Connection id missmatch.\n" );
-
-      /* We do only want to know, if it is zero */
-      left  = inpacket[64/4] | inpacket[68/4];
-
-      numwant = ntohl( inpacket[92/4] );
-      if (numwant > 200) numwant = 200;
-
-      event = ntohl( inpacket[80/4] );
-      port  = *(uint16_t*)( static_inbuf + 96 );
-      hash  = (ot_hash*)( static_inbuf + 16 );
-
-      OT_SETIP( &peer, remoteip );
-      OT_SETPORT( &peer, &port );
-      OT_FLAG( &peer ) = 0;
-
-      switch( event ) {
-        case 1: OT_FLAG( &peer ) |= PEER_FLAG_COMPLETED; break;
-        case 3: OT_FLAG( &peer ) |= PEER_FLAG_STOPPED; break;
-        default: break;
-      }
-
-      if( !left )
-        OT_FLAG( &peer )         |= PEER_FLAG_SEEDING;
-
-      outpacket[0] = htonl( 1 );    /* announce action */
-      outpacket[1] = inpacket[12/4];
-
-      if( OT_FLAG( &peer ) & PEER_FLAG_STOPPED ) /* Peer is gone. */
-        r = remove_peer_from_torrent( hash, &peer, static_outbuf, FLAG_UDP );
-      else {
-        torrent = add_peer_to_torrent( hash, &peer  WANT_SYNC_PARAM( 0 ) );
-        if( !torrent )
-          return; /* XXX maybe send error */
-
-        r = 8 + return_peers_for_torrent( hash, numwant, static_outbuf + 8, FLAG_UDP );
-      }
-
-      socket_send4( serversocket, static_outbuf, r, remoteip, remoteport );
-      stats_issue_event( EVENT_ANNOUNCE, FLAG_UDP, r );
-      break;
-
-    case 2: /* This is a scrape action */
-        if( !udp_test_connectionid( inpacket, remoteip ))
-        fprintf( stderr, "UDP scrape Connection id missmatch.\n" );
-
-      outpacket[0] = htonl( 2 );    /* scrape action */
-      outpacket[1] = inpacket[12/4];
-
-      for( r_out = 0; ( r_out * 20 < r - 16) && ( r_out <= 74 ); r_out++ )
-        return_udp_scrape_for_torrent( (ot_hash*)( static_inbuf + 16 + 20 * r_out ), static_outbuf + 8 + 12 * r_out );
-
-      socket_send4( serversocket, static_outbuf, 8 + 12 * r_out, remoteip, remoteport );
-      stats_issue_event( EVENT_SCRAPE, FLAG_UDP, r );
-      break;
+  /* If action is not 0 (connect), then we expect the derived
+     connection id in first 64 bit */
+  if ((action > 0) && (inpacket[0] != connid[0] || inpacket[1] != connid[1])) {
+    /* If connection id does not match, try the one that was
+       valid in the previous hour. Only if this also does not
+       match, return an error packet */
+    udp_make_connectionid(connid, remoteip, 1);
+    if (inpacket[0] != connid[0] || inpacket[1] != connid[1]) {
+      const size_t s = sizeof("Connection ID missmatch.");
+      outpacket[0]   = htonl(3);
+      outpacket[1]   = inpacket[3];
+      memcpy(&outpacket[2], "Connection ID missmatch.", s);
+      socket_send6(serversocket, ws->outbuf, 8 + s, remoteip, remoteport, 0);
+      stats_issue_event(EVENT_CONNID_MISSMATCH, FLAG_UDP, 8 + s);
+      return 1;
+    }
   }
+
+  switch (action) {
+  case 0: /* This is a connect action */
+    /* look for udp bittorrent magic id */
+    if ((ntohl(inpacket[0]) != 0x00000417) || (ntohl(inpacket[1]) != 0x27101980))
+      return 1;
+
+    outpacket[0] = 0;
+    outpacket[1] = inpacket[3];
+    outpacket[2] = connid[0];
+    outpacket[3] = connid[1];
+
+    socket_send6(serversocket, ws->outbuf, 16, remoteip, remoteport, 0);
+    stats_issue_event(EVENT_CONNECT, FLAG_UDP, 16);
+    break;
+  case 1: /* This is an announce action */
+    /* Minimum udp announce packet size */
+    if (byte_count < 98)
+      return 1;
+
+    /* We do only want to know, if it is zero */
+    left     = inpacket[64 / 4] | inpacket[68 / 4];
+
+    event    = ntohl(inpacket[80 / 4]);
+    port     = *(uint16_t *)(((char *)inpacket) + 96);
+    ws->hash = (ot_hash *)(((char *)inpacket) + 16);
+
+    OT_SETIP(ws->peer, remoteip);
+    OT_SETPORT(ws->peer, &port);
+    OT_PEERFLAG(ws->peer) = 0;
+
+    switch (event) {
+    case 1:
+      OT_PEERFLAG(ws->peer) |= PEER_FLAG_COMPLETED;
+      break;
+    case 3:
+      OT_PEERFLAG(ws->peer) |= PEER_FLAG_STOPPED;
+      break;
+    default:
+      break;
+    }
+
+    if (!left)
+      OT_PEERFLAG(ws->peer) |= PEER_FLAG_SEEDING;
+
+    outpacket[0] = htonl(1); /* announce action */
+    outpacket[1] = inpacket[12 / 4];
+
+    if (OT_PEERFLAG(ws->peer) & PEER_FLAG_STOPPED) { /* Peer is gone. */
+      ws->reply      = ws->outbuf;
+      ws->reply_size = remove_peer_from_torrent(FLAG_UDP, ws);
+    } else {
+      /* Limit amount of peers to OT_MAX_PEERS_UDP */
+      uint32_t numwant   = ntohl(inpacket[92 / 4]);
+      size_t   max_peers = ip6_isv4mapped(remoteip) ? OT_MAX_PEERS_UDP4 : OT_MAX_PEERS_UDP6;
+      if (numwant > max_peers)
+        numwant = max_peers;
+
+      ws->reply      = ws->outbuf + 8;
+      ws->reply_size = 8 + add_peer_to_torrent_and_return_peers(FLAG_UDP, ws, numwant);
+    }
+
+    socket_send6(serversocket, ws->outbuf, ws->reply_size, remoteip, remoteport, 0);
+    stats_issue_event(EVENT_ANNOUNCE, FLAG_UDP, ws->reply_size);
+    break;
+
+  case 2:                    /* This is a scrape action */
+    outpacket[0] = htonl(2); /* scrape action */
+    outpacket[1] = inpacket[12 / 4];
+
+    for (scrape_count = 0; (scrape_count * 20 < byte_count - 16) && (scrape_count <= 74); scrape_count++)
+      return_udp_scrape_for_torrent(*(ot_hash *)(((char *)inpacket) + 16 + 20 * scrape_count), ((char *)outpacket) + 8 + 12 * scrape_count);
+
+    socket_send6(serversocket, ws->outbuf, 8 + 12 * scrape_count, remoteip, remoteport, 0);
+    stats_issue_event(EVENT_SCRAPE, FLAG_UDP, scrape_count);
+    break;
+  }
+  return 1;
 }
 
-const char *g_version_udp_c = "$Source$: $Revision$\n";
+static void *udp_worker(void *args) {
+  int64                sock = (int64)args;
+  struct ot_workstruct ws;
+  memset(&ws, 0, sizeof(ws));
+
+  ws.inbuf  = malloc(G_INBUF_SIZE);
+  ws.outbuf = malloc(G_OUTBUF_SIZE);
+#ifdef _DEBUG_HTTPERROR
+  ws.debugbuf = malloc(G_DEBUGBUF_SIZE);
+#endif
+
+  while (g_opentracker_running)
+    handle_udp6(sock, &ws);
+
+  free(ws.inbuf);
+  free(ws.outbuf);
+#ifdef _DEBUG_HTTPERROR
+  free(ws.debugbuf);
+#endif
+  return NULL;
+}
+
+void udp_init(int64 sock, unsigned int worker_count) {
+  pthread_t thread_id;
+  if (!g_rijndael_round_key[0])
+    udp_generate_rijndael_round_key();
+#ifdef _DEBUG
+  fprintf(stderr, " installing %d workers on udp socket %ld\n", worker_count, (unsigned long)sock);
+#endif
+  while (worker_count--)
+    pthread_create(&thread_id, NULL, udp_worker, (void *)sock);
+}

ot_udp.h

@@ -3,9 +3,10 @@
 
    $id$ */
 
-#ifndef __OT_UDP_H__
-#define __OT_UDP_H__
+#ifndef OT_UDP_H__
+#define OT_UDP_H__
 
-void handle_udp4( int64 serversocket );
+void udp_init(int64 sock, unsigned int worker_count);
+int  handle_udp6(int64 serversocket, struct ot_workstruct *ws);
 
 #endif

ot_vector.c

@@ -4,69 +4,53 @@
    $id$ */
 
 /* System */
+#include <stddef.h>
+#include <stdint.h>
 #include <stdlib.h>
 #include <string.h>
+#include <strings.h>
 
 /* Opentracker */
 #include "trackerlogic.h"
-#include "ot_vector.h"
 
-#ifdef _DEBUG_VECTOR
-#include <stdio.h>
+/* Libowfat */
+#include "uint16.h"
+#include "uint32.h"
 
-static uint64_t vector_debug_inc[32];
-static uint64_t vector_debug_noinc[32];
-static uint64_t vector_debug_dec[32];
-static uint64_t vector_debug_nodec[32];
-static void vector_debug( size_t old_size, ssize_t diff_size, size_t old_space, ssize_t diff_space ) {
-  int x = 0;
-  while( old_space ) { old_space>>=1; ++x; }
-  old_size = old_size;
-
-  if( diff_size == -1 )
-    if( diff_space ) vector_debug_dec[x]++; else vector_debug_nodec[x]++;
-  else
-    if( diff_space ) vector_debug_inc[x]++; else vector_debug_noinc[x]++;
-
-}
-
-size_t vector_info( char * reply ) {
-  char * r = reply;
-  int i;
-  for( i=1; i<28; ++i )
-    r += sprintf( r, "  inc % 12d -> % 12d: % 16lld\n", 1<<(i-1), 8<<(i-1), vector_debug_inc[i] );
-  for( i=1; i<28; ++i )
-    r += sprintf( r, "noinc % 12d -> % 12d: % 16lld\n", 1<<(i-1), 1<<(i-1), vector_debug_noinc[i] );
-  for( i=1; i<28; ++i )
-    r += sprintf( r, "  dec % 12d -> % 12d: % 16lld\n", 1<<(i-1), 4<<(i-1), vector_debug_dec[i] );
-  for( i=1; i<28; ++i )
-    r += sprintf( r, "nodec % 12d -> % 12d: % 16lld\n", 1<<(i-1), 1<<(i-1), vector_debug_nodec[i] );
-  return r - reply;
-}
-#endif
+static int vector_compare_peer6(const void *peer1, const void *peer2) { return memcmp(peer1, peer2, OT_PEER_COMPARE_SIZE6); }
+static int vector_compare_peer4(const void *peer1, const void *peer2) { return memcmp(peer1, peer2, OT_PEER_COMPARE_SIZE4); }
 
 /* This function gives us a binary search that returns a pointer, even if
    no exact match is found. In that case it sets exactmatch 0 and gives
    calling functions the chance to insert data
 */
-void *binary_search( const void * const key, const void * base, const size_t member_count, const size_t member_size,
-                     size_t compare_size, int *exactmatch ) {
-  size_t mc = member_count;
-  uint8_t *lookat = ((uint8_t*)base) + member_size * (member_count >> 1);
-  *exactmatch = 1;
+void *binary_search(const void *const key, const void *base, const size_t member_count, const size_t member_size, size_t compare_size, int *exactmatch) {
+  size_t interval = member_count;
 
-  while( mc ) {
-    int cmp = memcmp( lookat, key, compare_size);
-    if (cmp == 0) return (void *)lookat;
-    if (cmp < 0) {
-      base = (void*)(lookat + member_size);
-      --mc;
+  while (interval) {
+    uint8_t *lookat = ((uint8_t *)base) + member_size * (interval / 2);
+    int      cmp    = memcmp(lookat, key, compare_size);
+    if (cmp == 0) {
+      base = lookat;
+      break;
     }
-    mc >>= 1;
-    lookat = ((uint8_t*)base) + member_size * (mc >> 1);
+    if (cmp < 0) {
+      base = lookat + member_size;
+      interval--;
+    }
+    interval /= 2;
   }
-  *exactmatch = 0;
-  return (void*)lookat;
+
+  *exactmatch = interval;
+  return (void *)base;
+}
+
+static uint8_t vector_hash_peer(ot_peer const *peer, size_t compare_size, int bucket_count) {
+  unsigned int hash = 5381;
+  uint8_t     *p    = (uint8_t *)peer;
+  while (compare_size--)
+    hash += (hash << 5) + *(p++);
+  return hash % bucket_count;
 }
 
 /* This is the generic insert operation for our vector type.
@@ -76,92 +60,225 @@ void *binary_search( const void * const key, const void * base, const size_t mem
    if it wasn't found in vector. Caller needs to check the passed "exactmatch" variable to see, whether an insert
    took place. If resizing the vector failed, NULL is returned, else the pointer to the object in vector.
 */
-void *vector_find_or_insert( ot_vector *vector, void *key, size_t member_size, size_t compare_size, int *exactmatch ) {
-  uint8_t *match = binary_search( key, vector->data, vector->size, member_size, compare_size, exactmatch );
-#ifdef _DEBUG_VECTOR
-  size_t old_space = vector->space;
-#endif
+void *vector_find_or_insert(ot_vector *vector, void *key, size_t member_size, size_t compare_size, int *exactmatch) {
+  uint8_t *match = binary_search(key, vector->data, vector->size, member_size, compare_size, exactmatch);
 
-  if( *exactmatch ) return match;
+  if (*exactmatch)
+    return match;
 
-  if( vector->size + 1 >= vector->space ) {
+  if (vector->size + 1 > vector->space) {
     size_t   new_space = vector->space ? OT_VECTOR_GROW_RATIO * vector->space : OT_VECTOR_MIN_MEMBERS;
-    uint8_t *new_data = realloc( vector->data, new_space * member_size );
-    if( !new_data ) return NULL;
-
+    uint8_t *new_data  = realloc(vector->data, new_space * member_size);
+    if (!new_data)
+      return NULL;
     /* Adjust pointer if it moved by realloc */
-    match = new_data + (match - (uint8_t*)vector->data);
+    match         = new_data + (match - (uint8_t *)vector->data);
 
-    vector->data = new_data;
+    vector->data  = new_data;
     vector->space = new_space;
   }
-  memmove( match + member_size, match, ((uint8_t*)vector->data) + member_size * vector->size - match );
+  memmove(match + member_size, match, ((uint8_t *)vector->data) + member_size * vector->size - match);
+
+  vector->size++;
+  return match;
+}
+
+ot_peer *vector_find_or_insert_peer(ot_vector *vector, ot_peer const *peer, size_t peer_size, int *exactmatch) {
+  ot_peer     *match, *end;
+  const size_t compare_size = OT_PEER_COMPARE_SIZE_FROM_PEER_SIZE(peer_size);
+  size_t       match_to_end;
+
+  /* If space is zero but size is set, we're dealing with a list of vector->size buckets */
+  if (vector->space < vector->size)
+    vector = ((ot_vector *)vector->data) + vector_hash_peer(peer, compare_size, vector->size);
+  match = binary_search(peer, vector->data, vector->size, peer_size, compare_size, exactmatch);
+
+  if (*exactmatch)
+    return match;
+
+  /* This is the amount of bytes that needs to be pushed backwards by peer_size bytes to make room for new peer */
+  end          = (ot_peer *)vector->data + vector->size * peer_size;
+  match_to_end = end - match;
+
+  if (vector->size + 1 > vector->space) {
+    ptrdiff_t offset    = match - (ot_peer *)vector->data;
+    size_t    new_space = vector->space ? OT_VECTOR_GROW_RATIO * vector->space : OT_VECTOR_MIN_MEMBERS;
+    ot_peer  *new_data  = realloc(vector->data, new_space * peer_size);
+
+    if (!new_data)
+      return NULL;
+    /* Adjust pointer if it moved by realloc */
+    match         = new_data + offset;
+
+    vector->data  = new_data;
+    vector->space = new_space;
+  }
+
+  /* Here we're guaranteed to have enough space in vector to move the block of peers after insertion point */
+  memmove(match + peer_size, match, match_to_end);
 
-#ifdef _DEBUG_VECTOR
-  vector_debug( vector->size, 1, old_space, vector->space - old_space );
-#endif
   vector->size++;
   return match;
 }
 
 /* This is the non-generic delete from vector-operation specialized for peers in pools.
-   Set hysteresis == 0 if you expect the vector not to ever grow again.
    It returns 0 if no peer was found (and thus not removed)
               1 if a non-seeding peer was removed
               2 if a seeding peer was removed
 */
-int vector_remove_peer( ot_vector *vector, ot_peer *peer, int hysteresis ) {
-  int      exactmatch;
-  size_t   shrink_thresh = hysteresis ? OT_VECTOR_SHRINK_THRESH : OT_VECTOR_SHRINK_RATIO;
-  ot_peer *end = ((ot_peer*)vector->data) + vector->size;
-  ot_peer *match;
-#ifdef _DEBUG_VECTOR
-  size_t   old_space = vector->space;
-#endif
+int vector_remove_peer(ot_vector *vector, ot_peer const *peer, size_t peer_size) {
+  int      exactmatch, was_seeder;
+  ot_peer *match, *end;
+  size_t   compare_size = OT_PEER_COMPARE_SIZE_FROM_PEER_SIZE(peer_size);
 
-  if( !vector->size ) return 0;
-  match = binary_search( peer, vector->data, vector->size, sizeof( ot_peer ), OT_PEER_COMPARE_SIZE, &exactmatch );
+  if (!vector->size)
+    return 0;
 
-  if( !exactmatch ) return 0;
-  exactmatch = ( OT_FLAG( match ) & PEER_FLAG_SEEDING ) ? 2 : 1;
-  memmove( match, match + 1, sizeof(ot_peer) * ( end - match - 1 ) );
-  if( ( --vector->size * shrink_thresh < vector->space ) && ( vector->space >= OT_VECTOR_SHRINK_RATIO * OT_VECTOR_MIN_MEMBERS ) ) {
-    vector->space /= OT_VECTOR_SHRINK_RATIO;
-    vector->data = realloc( vector->data, vector->space * sizeof( ot_peer ) );
-  }
-  if( !vector->size ) {
-    /* for peer pools its safe to let them go,
-       in 999 of 1000 this happens in older pools, that won't ever grow again */
-    free( vector->data );
-    vector->data = NULL;
-    vector->space = 0;
-  }
-#ifdef _DEBUG_VECTOR
-  vector_debug( vector->size+1, -1, old_space, vector->space - old_space );
-#endif
-  return exactmatch;
+  /* If space is zero but size is set, we're dealing with a list of vector->size buckets */
+  if (vector->space < vector->size)
+    vector = ((ot_vector *)vector->data) + vector_hash_peer(peer, compare_size, vector->size);
+
+  end   = ((ot_peer *)vector->data) + peer_size * vector->size;
+  match = (ot_peer *)binary_search(peer, vector->data, vector->size, peer_size, compare_size, &exactmatch);
+  if (!exactmatch)
+    return 0;
+
+  was_seeder = (OT_PEERFLAG_D(match, peer_size) & PEER_FLAG_SEEDING) ? 2 : 1;
+  memmove(match, match + peer_size, end - match - peer_size);
+
+  vector->size--;
+  vector_fixup_peers(vector, peer_size);
+  return was_seeder;
 }
 
-void vector_remove_torrent( ot_vector *vector, ot_torrent *match ) {
-  ot_torrent *end = ((ot_torrent*)vector->data) + vector->size;
-#ifdef _DEBUG_VECTOR
-  size_t      old_space = vector->space;
-#endif
+void vector_remove_torrent(ot_vector *vector, ot_torrent *match) {
+  ot_torrent *end = ((ot_torrent *)vector->data) + vector->size;
 
-  if( !vector->size ) return;
+  if (!vector->size)
+    return;
 
   /* If this is being called after a unsuccessful malloc() for peer_list
      in add_peer_to_torrent, match->peer_list actually might be NULL */
-  if( match->peer_list) free_peerlist( match->peer_list );
+  free_peerlist(match->peer_list6);
+  free_peerlist(match->peer_list4);
 
-  memmove( match, match + 1, sizeof(ot_torrent) * ( end - match - 1 ) );
-  if( ( --vector->size * OT_VECTOR_SHRINK_THRESH < vector->space ) && ( vector->space >= OT_VECTOR_SHRINK_RATIO * OT_VECTOR_MIN_MEMBERS ) ) {
+  memmove(match, match + 1, sizeof(ot_torrent) * (end - match - 1));
+  if ((--vector->size * OT_VECTOR_SHRINK_THRESH < vector->space) && (vector->space >= OT_VECTOR_SHRINK_RATIO * OT_VECTOR_MIN_MEMBERS)) {
     vector->space /= OT_VECTOR_SHRINK_RATIO;
-    vector->data = realloc( vector->data, vector->space * sizeof( ot_torrent ) );
+    vector->data   = realloc(vector->data, vector->space * sizeof(ot_torrent));
   }
-#ifdef _DEBUG_VECTOR
-  vector_debug( vector->size+1, -1, old_space, vector->space - old_space );
-#endif
 }
 
-const char *g_version_vector_c = "$Source$: $Revision$\n";
+void vector_clean_list(ot_vector *vector, int num_buckets) {
+  while (num_buckets--)
+    free(vector[num_buckets].data);
+  free(vector);
+  return;
+}
+
+void vector_redistribute_buckets(ot_peerlist *peer_list, size_t peer_size) {
+  int        tmp, bucket, bucket_size_new, num_buckets_new, num_buckets_old = 1;
+  ot_vector *bucket_list_new, *bucket_list_old = &peer_list->peers;
+  int (*sort_func)(const void *, const void *) = peer_size == OT_PEER_SIZE6 ? &vector_compare_peer6 : &vector_compare_peer4;
+
+  if (OT_PEERLIST_HASBUCKETS(peer_list)) {
+    num_buckets_old = peer_list->peers.size;
+    bucket_list_old = peer_list->peers.data;
+  }
+
+  if (peer_list->peer_count < 255)
+    num_buckets_new = 1;
+  else if (peer_list->peer_count > 8192)
+    num_buckets_new = 64;
+  else if (peer_list->peer_count >= 512 && peer_list->peer_count < 4096)
+    num_buckets_new = 16;
+  else if (peer_list->peer_count < 512 && num_buckets_old <= 16)
+    num_buckets_new = num_buckets_old;
+  else if (peer_list->peer_count < 512)
+    num_buckets_new = 1;
+  else if (peer_list->peer_count < 8192 && num_buckets_old > 1)
+    num_buckets_new = num_buckets_old;
+  else
+    num_buckets_new = 16;
+
+  if (num_buckets_new == num_buckets_old)
+    return;
+
+  /* Assume near perfect distribution */
+  bucket_list_new = malloc(num_buckets_new * sizeof(ot_vector));
+  if (!bucket_list_new)
+    return;
+  bzero(bucket_list_new, num_buckets_new * sizeof(ot_vector));
+
+  tmp             = peer_list->peer_count / num_buckets_new;
+  bucket_size_new = OT_VECTOR_MIN_MEMBERS;
+  while (bucket_size_new < tmp)
+    bucket_size_new *= OT_VECTOR_GROW_RATIO;
+
+  /* preallocate vectors to hold all peers */
+  for (bucket = 0; bucket < num_buckets_new; ++bucket) {
+    bucket_list_new[bucket].space = bucket_size_new;
+    bucket_list_new[bucket].data  = malloc(bucket_size_new * peer_size);
+    if (!bucket_list_new[bucket].data)
+      return vector_clean_list(bucket_list_new, num_buckets_new);
+  }
+
+  /* Now sort them into the correct bucket */
+  for (bucket = 0; bucket < num_buckets_old; ++bucket) {
+    ot_peer *peers_old      = bucket_list_old[bucket].data;
+    int      peer_count_old = bucket_list_old[bucket].size;
+    while (peer_count_old--) {
+      ot_vector *bucket_dest = bucket_list_new;
+      if (num_buckets_new > 1)
+        bucket_dest += vector_hash_peer(peers_old, OT_PEER_COMPARE_SIZE_FROM_PEER_SIZE(peer_size), num_buckets_new);
+      if (bucket_dest->size + 1 > bucket_dest->space) {
+        void *tmp = realloc(bucket_dest->data, peer_size * OT_VECTOR_GROW_RATIO * bucket_dest->space);
+        if (!tmp)
+          return vector_clean_list(bucket_list_new, num_buckets_new);
+        bucket_dest->data   = tmp;
+        bucket_dest->space *= OT_VECTOR_GROW_RATIO;
+      }
+      memcpy((ot_peer *)bucket_dest->data + peer_size * bucket_dest->size++, peers_old, peer_size);
+      peers_old += peer_size;
+    }
+  }
+
+  /* Now sort each bucket to later allow bsearch */
+  for (bucket = 0; bucket < num_buckets_new; ++bucket)
+    qsort(bucket_list_new[bucket].data, bucket_list_new[bucket].size, peer_size, sort_func);
+
+  /* Everything worked fine. Now link new bucket_list to peer_list */
+  if (OT_PEERLIST_HASBUCKETS(peer_list))
+    vector_clean_list((ot_vector *)peer_list->peers.data, peer_list->peers.size);
+  else
+    free(peer_list->peers.data);
+
+  if (num_buckets_new > 1) {
+    peer_list->peers.data  = bucket_list_new;
+    peer_list->peers.size  = num_buckets_new;
+    peer_list->peers.space = 0; /* Magic marker for "is list of buckets" */
+  } else {
+    peer_list->peers.data  = bucket_list_new->data;
+    peer_list->peers.size  = bucket_list_new->size;
+    peer_list->peers.space = bucket_list_new->space;
+    free(bucket_list_new);
+  }
+}
+
+void vector_fixup_peers(ot_vector *vector, size_t peer_size) {
+  int need_fix = 0;
+
+  if (!vector->size) {
+    free(vector->data);
+    vector->data  = NULL;
+    vector->space = 0;
+    return;
+  }
+
+  while ((vector->size * OT_VECTOR_SHRINK_THRESH < vector->space) && (vector->space >= OT_VECTOR_SHRINK_RATIO * OT_VECTOR_MIN_MEMBERS)) {
+    vector->space /= OT_VECTOR_SHRINK_RATIO;
+    need_fix++;
+  }
+  if (need_fix)
+    vector->data = realloc(vector->data, vector->space * peer_size);
+}

ot_vector.h

@@ -3,8 +3,8 @@
 
    $id$ */
 
-#ifndef __OT_VECTOR_H__
-#define __OT_VECTOR_H__
+#ifndef OT_VECTOR_H__
+#define OT_VECTOR_H__
 
 /* These defines control vectors behaviour */
 #define OT_VECTOR_MIN_MEMBERS   2
@@ -12,21 +12,25 @@
 #define OT_VECTOR_SHRINK_THRESH 4
 #define OT_VECTOR_SHRINK_RATIO  2
 
+#define OT_PEER_BUCKET_MINCOUNT 512
+#define OT_PEER_BUCKET_MAXCOUNT 256
+
 typedef struct {
-  void   *data;
-  size_t  size;
-  size_t  space;
+  void  *data;
+  size_t size;
+  size_t space;
 } ot_vector;
 
-void *binary_search( const void * const key, const void * base, const size_t member_count, const size_t member_size,
-                     size_t compare_size, int *exactmatch );
-void *vector_find_or_insert( ot_vector *vector, void *key, size_t member_size, size_t compare_size, int *exactmatch );
+void    *binary_search(const void *const key, const void *base, const size_t member_count, const size_t member_size, size_t compare_size, int *exactmatch);
+void    *vector_find_or_insert(ot_vector *vector, void *key, size_t member_size, size_t compare_size, int *exactmatch);
+ot_peer *vector_find_or_insert_peer(ot_vector *vector, ot_peer const *peer, size_t peer_size, int *exactmatch);
 
-int   vector_remove_peer( ot_vector *vector, ot_peer *peer, int hysteresis );
-void  vector_remove_torrent( ot_vector *vector, ot_torrent *match );
+int      vector_remove_peer(ot_vector *vector, ot_peer const *peer, size_t peer_size);
+void     vector_remove_torrent(ot_vector *vector, ot_torrent *match);
 
-#ifdef _DEBUG_VECTOR
-size_t vector_info( char * reply );
-#endif
+/* For ot_clean.c */
+void     vector_redistribute_buckets(ot_peerlist *peer_list, size_t peer_size);
+void     vector_fixup_peers(ot_vector *vector, size_t peer_size);
+void     vector_clean_list(ot_vector *vector, int num_buckets);
 
 #endif

proxy.c

@@ -0,0 +1,896 @@
+/* This software was written by Dirk Engling <erdgeist@erdgeist.org>
+   It is considered beerware. Prost. Skol. Cheers or whatever.
+
+   $Id$ */
+
+/* System */
+#include <arpa/inet.h>
+#include <ctype.h>
+#include <errno.h>
+#include <pthread.h>
+#include <pwd.h>
+#include <signal.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <unistd.h>
+
+/* Libowfat */
+#include "byte.h"
+#include "io.h"
+#include "iob.h"
+#include "ip6.h"
+#include "ndelay.h"
+#include "scan.h"
+#include "socket.h"
+
+/* Opentracker */
+#include "ot_mutex.h"
+#include "ot_stats.h"
+#include "ot_vector.h"
+#include "trackerlogic.h"
+
+#ifndef WANT_SYNC_LIVE
+#define WANT_SYNC_LIVE
+#endif
+#include "ot_livesync.h"
+
+ot_ip6   g_serverip;
+uint16_t g_serverport = 9009;
+uint32_t g_tracker_id;
+char     groupip_1[4] = {224, 0, 23, 5};
+int      g_self_pipe[2];
+
+/* If you have more than 10 peers, don't use this proxy
+   Use 20 slots for 10 peers to have room for 10 incoming connection slots
+ */
+#define MAX_PEERS                         20
+
+#define LIVESYNC_INCOMING_BUFFSIZE        (256 * 256)
+#define STREAMSYNC_OUTGOING_BUFFSIZE      (256 * 256)
+
+#define LIVESYNC_OUTGOING_BUFFSIZE_PEERS  1480
+#define LIVESYNC_OUTGOING_WATERMARK_PEERS (sizeof(ot_peer) + sizeof(ot_hash))
+#define LIVESYNC_MAXDELAY                 15 /* seconds */
+
+/* The amount of time a complete sync cycle should take */
+#define OT_SYNC_INTERVAL_MINUTES          2
+
+/* So after each bucket wait 1 / OT_BUCKET_COUNT intervals */
+#define OT_SYNC_SLEEP                     (((OT_SYNC_INTERVAL_MINUTES) * 60 * 1000000) / (OT_BUCKET_COUNT))
+
+enum { OT_SYNC_PEER4, OT_SYNC_PEER6 };
+enum { FLAG_SERVERSOCKET = 1 };
+
+/* For incoming packets */
+static int64    g_socket_in = -1;
+static uint8_t  g_inbuffer[LIVESYNC_INCOMING_BUFFSIZE];
+
+/* For outgoing packets */
+static int64    g_socket_out = -1;
+static uint8_t  g_peerbuffer_start[LIVESYNC_OUTGOING_BUFFSIZE_PEERS];
+static uint8_t *g_peerbuffer_pos;
+static uint8_t *g_peerbuffer_highwater = g_peerbuffer_start + LIVESYNC_OUTGOING_BUFFSIZE_PEERS - LIVESYNC_OUTGOING_WATERMARK_PEERS;
+static ot_time  g_next_packet_time;
+
+static void *livesync_worker(void *args);
+static void *streamsync_worker(void *args);
+static void livesync_proxytell(uint8_t prefix, uint8_t *info_hash, uint8_t *peer);
+
+void exerr(char *message) {
+  fprintf(stderr, "%s\n", message);
+  exit(111);
+}
+
+void stats_issue_event(ot_status_event event, PROTO_FLAG proto, uintptr_t event_data) {
+  (void)event;
+  (void)proto;
+  (void)event_data;
+}
+
+void livesync_bind_mcast(ot_ip6 ip, uint16_t port) {
+  char  tmpip[4] = {0, 0, 0, 0};
+  char *v4ip;
+
+  if (!ip6_isv4mapped(ip))
+    exerr("v6 mcast support not yet available.");
+  v4ip = ip + 12;
+
+  if (g_socket_in != -1)
+    exerr("Error: Livesync listen ip specified twice.");
+
+  if ((g_socket_in = socket_udp4()) < 0)
+    exerr("Error: Cant create live sync incoming socket.");
+  ndelay_off(g_socket_in);
+
+  if (socket_bind4_reuse(g_socket_in, tmpip, port) == -1)
+    exerr("Error: Cant bind live sync incoming socket.");
+
+  if (socket_mcjoin4(g_socket_in, groupip_1, v4ip))
+    exerr("Error: Cant make live sync incoming socket join mcast group.");
+
+  if ((g_socket_out = socket_udp4()) < 0)
+    exerr("Error: Cant create live sync outgoing socket.");
+  if (socket_bind4_reuse(g_socket_out, v4ip, port) == -1)
+    exerr("Error: Cant bind live sync outgoing socket.");
+
+  socket_mcttl4(g_socket_out, 1);
+  socket_mcloop4(g_socket_out, 1);
+}
+
+size_t add_peer_to_torrent_proxy(ot_hash hash, ot_peer *peer, size_t peer_size) {
+  int          exactmatch;
+  ot_torrent  *torrent;
+  ot_peerlist *peer_list;
+  ot_peer     *peer_dest;
+  ot_vector   *torrents_list = mutex_bucket_lock_by_hash(hash);
+  size_t       compare_size  = OT_PEER_COMPARE_SIZE_FROM_PEER_SIZE(peer_size);
+
+  torrent                    = vector_find_or_insert(torrents_list, (void *)hash, sizeof(ot_torrent), compare_size, &exactmatch);
+  if (!torrent)
+    return -1;
+
+  if (!exactmatch) {
+    /* Create a new torrent entry, then */
+    memcpy(torrent->hash, hash, sizeof(ot_hash));
+
+    if (!(torrent->peer_list6 = malloc(sizeof(ot_peerlist))) || !(torrent->peer_list4 = malloc(sizeof(ot_peerlist)))) {
+      vector_remove_torrent(torrents_list, torrent);
+      mutex_bucket_unlock_by_hash(hash, 0);
+      return -1;
+    }
+
+    byte_zero(torrent->peer_list6, sizeof(ot_peerlist));
+    byte_zero(torrent->peer_list4, sizeof(ot_peerlist));
+  }
+
+  peer_list = peer_size == OT_PEER_SIZE6 ? torrent->peer_list6 : torrent->peer_list4;
+
+  /* Check for peer in torrent */
+  peer_dest = vector_find_or_insert_peer(&(peer_list->peers), peer, peer_size, &exactmatch);
+  if (!peer_dest) {
+    mutex_bucket_unlock_by_hash(hash, 0);
+    return -1;
+  }
+  /* Tell peer that it's fresh */
+  OT_PEERTIME(peer, peer_size) = 0;
+
+  /* If we hadn't had a match create peer there */
+  if (!exactmatch) {
+    peer_list->peer_count++;
+    if (OT_PEERFLAG_D(peer, peer_size) & PEER_FLAG_SEEDING)
+      peer_list->seed_count++;
+  }
+  memcpy(peer_dest, peer, peer_size);
+  mutex_bucket_unlock_by_hash(hash, 0);
+  return 0;
+}
+
+size_t remove_peer_from_torrent_proxy(ot_hash hash, ot_peer *peer, size_t peer_size) {
+  int         exactmatch;
+  ot_vector  *torrents_list = mutex_bucket_lock_by_hash(hash);
+  ot_torrent *torrent       = binary_search(hash, torrents_list->data, torrents_list->size, sizeof(ot_torrent), OT_HASH_COMPARE_SIZE, &exactmatch);
+
+  if (exactmatch) {
+    ot_peerlist *peer_list = peer_list = peer_size == OT_PEER_SIZE6 ? torrent->peer_list6 : torrent->peer_list4;
+    switch (vector_remove_peer(&peer_list->peers, peer, peer_size)) {
+    case 2:
+      peer_list->seed_count--; /* Intentional fallthrough */
+    case 1:
+      peer_list->peer_count--; /* Intentional fallthrough */
+    default:
+      break;
+    }
+  }
+
+  mutex_bucket_unlock_by_hash(hash, 0);
+  return 0;
+}
+
+void free_peerlist(ot_peerlist *peer_list) {
+  if (peer_list->peers.data) {
+    if (OT_PEERLIST_HASBUCKETS(peer_list)) {
+      ot_vector *bucket_list = (ot_vector *)(peer_list->peers.data);
+
+      while (peer_list->peers.size--)
+        free(bucket_list++->data);
+    }
+    free(peer_list->peers.data);
+  }
+  free(peer_list);
+}
+
+static void livesync_handle_peersync(ssize_t datalen, size_t peer_size) {
+  int off = sizeof(g_tracker_id) + sizeof(uint32_t);
+
+  fprintf(stderr, ".");
+
+  while ((ssize_t)(off + sizeof(ot_hash) + peer_size) <= datalen) {
+    ot_peer *peer = (ot_peer *)(g_inbuffer + off + sizeof(ot_hash));
+    ot_hash *hash = (ot_hash *)(g_inbuffer + off);
+
+    if (OT_PEERFLAG_D(peer, peer_size) & PEER_FLAG_STOPPED)
+      remove_peer_from_torrent_proxy(*hash, peer, peer_size);
+    else
+      add_peer_to_torrent_proxy(*hash, peer, peer_size);
+
+    off += sizeof(ot_hash) + peer_size;
+  }
+}
+
+int usage(char *self) {
+  fprintf(stderr, "Usage: %s -L <livesync_iface_ip> -l <listenip>:<listenport> -c <connectip>:<connectport>\n", self);
+  return 0;
+}
+
+enum {
+  FLAG_OUTGOING      = 0x80,
+
+  FLAG_DISCONNECTED  = 0x00,
+  FLAG_CONNECTING    = 0x01,
+  FLAG_WAITTRACKERID = 0x02,
+  FLAG_CONNECTED     = 0x03,
+
+  FLAG_MASK          = 0x07
+};
+
+#define PROXYPEER_NEEDSCONNECT(flag)     ((flag) == FLAG_OUTGOING)
+#define PROXYPEER_ISCONNECTED(flag)      (((flag) & FLAG_MASK) == FLAG_CONNECTED)
+#define PROXYPEER_SETDISCONNECTED(flag)  (flag) = (((flag) & FLAG_OUTGOING) | FLAG_DISCONNECTED)
+#define PROXYPEER_SETCONNECTING(flag)    (flag) = (((flag) & FLAG_OUTGOING) | FLAG_CONNECTING)
+#define PROXYPEER_SETWAITTRACKERID(flag) (flag) = (((flag) & FLAG_OUTGOING) | FLAG_WAITTRACKERID)
+#define PROXYPEER_SETCONNECTED(flag)     (flag) = (((flag) & FLAG_OUTGOING) | FLAG_CONNECTED)
+
+typedef struct {
+  int      state;             /* Whether we want to connect, how far our handshake is, etc. */
+  ot_ip6   ip;                /* The peer to connect to */
+  uint16_t port;              /* The peers port */
+  uint8_t  indata[8192 * 16]; /* Any data not processed yet */
+  size_t   indata_length;     /* Length of unprocessed data */
+  uint32_t tracker_id;        /* How the other end greeted */
+  int64    fd;                /* A file handle, if connected, <= 0 is disconnected (0 initially, -1 else) */
+  io_batch outdata;           /* The iobatch containing our sync data */
+
+  size_t   packet_tcount;     /* Number of unprocessed torrents in packet we currently receive */
+  uint8_t  packet_tprefix;    /* Prefix byte for all torrents in current packet */
+  uint8_t  packet_type;       /* Type of current packet */
+  uint32_t packet_tid;        /* Tracker id for current packet */
+
+} proxy_peer;
+static void process_indata(proxy_peer *peer);
+
+void reset_info_block(proxy_peer *peer) {
+  peer->indata_length = 0;
+  peer->tracker_id    = 0;
+  peer->fd            = -1;
+  peer->packet_tcount = 0;
+  iob_reset(&peer->outdata);
+  PROXYPEER_SETDISCONNECTED(peer->state);
+}
+
+/* Number of connections to peers
+ * If a peer's IP is set, we try to reconnect, when the connection drops
+ * If we already have a connected tracker_id in our records for an _incoming_ connection, drop it
+ * Multiple connections to/from the same ip are okay, if tracker_id doesn't match
+ * Reconnect attempts occur only twice a minute
+ */
+static int        g_connection_count;
+static ot_time    g_connection_reconn;
+static proxy_peer g_connections[MAX_PEERS];
+
+static void handle_reconnects(void) {
+  int i;
+  for (i = 0; i < g_connection_count; ++i)
+    if (PROXYPEER_NEEDSCONNECT(g_connections[i].state)) {
+      int64 newfd = socket_tcp6();
+      fprintf(stderr, "(Re)connecting to peer...");
+      if (newfd < 0)
+        continue; /* No socket for you */
+      io_fd(newfd);
+      if (socket_bind6_reuse(newfd, g_serverip, g_serverport, 0)) {
+        io_close(newfd);
+        continue;
+      }
+      if (socket_connect6(newfd, g_connections[i].ip, g_connections[i].port, 0) == -1 && errno != EINPROGRESS && errno != EWOULDBLOCK) {
+        close(newfd);
+        continue;
+      }
+      io_wantwrite(newfd); /* So we will be informed when it is connected */
+      io_setcookie(newfd, g_connections + i);
+
+      /* Prepare connection info block */
+      reset_info_block(g_connections + i);
+      g_connections[i].fd = newfd;
+      PROXYPEER_SETCONNECTING(g_connections[i].state);
+    }
+  g_connection_reconn = time(NULL) + 30;
+}
+
+/* Handle incoming connection requests, check against whitelist */
+static void handle_accept(int64 serversocket) {
+  int64  newfd;
+  ot_ip6 ip;
+  uint16 port;
+
+  while ((newfd = socket_accept6(serversocket, ip, &port, NULL)) != -1) {
+
+    /* XXX some access control */
+
+    /* Put fd into a non-blocking mode */
+    io_nonblock(newfd);
+
+    if (!io_fd(newfd))
+      io_close(newfd);
+    else {
+      /* Find a new home for our incoming connection */
+      int i;
+      for (i = 0; i < MAX_PEERS; ++i)
+        if (g_connections[i].state == FLAG_DISCONNECTED)
+          break;
+      if (i == MAX_PEERS) {
+        fprintf(stderr, "No room for incoming connection.");
+        close(newfd);
+        continue;
+      }
+
+      /* Prepare connection info block */
+      reset_info_block(g_connections + i);
+      PROXYPEER_SETCONNECTING(g_connections[i].state);
+      g_connections[i].port = port;
+      g_connections[i].fd   = newfd;
+
+      io_setcookie(newfd, g_connections + i);
+
+      /* We expect the connecting side to begin with its tracker_id */
+      io_wantread(newfd);
+    }
+  }
+
+  return;
+}
+
+/* New sync data on the stream */
+static void handle_read(int64 peersocket) {
+  int         i;
+  int64       datalen;
+  uint32_t    tracker_id;
+  proxy_peer *peer = io_getcookie(peersocket);
+
+  if (!peer) {
+    /* Can't happen ;) */
+    io_close(peersocket);
+    return;
+  }
+  switch (peer->state & FLAG_MASK) {
+  case FLAG_DISCONNECTED:
+    io_close(peersocket);
+    break; /* Shouldnt happen */
+  case FLAG_CONNECTING:
+  case FLAG_WAITTRACKERID:
+    /* We want at least the first four bytes to come at once, to avoid keeping extra states (for now)
+       This also catches 0 bytes reads == EOF and negative values, denoting connection errors */
+    if (io_tryread(peersocket, (void *)&tracker_id, sizeof(tracker_id)) != sizeof(tracker_id))
+      goto close_socket;
+
+    /* See, if we already have a connection to that peer */
+    for (i = 0; i < MAX_PEERS; ++i)
+      if ((g_connections[i].state & FLAG_MASK) == FLAG_CONNECTED && g_connections[i].tracker_id == tracker_id) {
+        fprintf(stderr, "Peer already connected. Closing connection.\n");
+        goto close_socket;
+      }
+
+    /* Also no need for soliloquy */
+    if (tracker_id == g_tracker_id)
+      goto close_socket;
+
+    /* The new connection is good, send our tracker_id on incoming connections */
+    if (peer->state == FLAG_CONNECTING)
+      if (io_trywrite(peersocket, (void *)&g_tracker_id, sizeof(g_tracker_id)) != sizeof(g_tracker_id))
+        goto close_socket;
+
+    peer->tracker_id = tracker_id;
+    PROXYPEER_SETCONNECTED(peer->state);
+
+    if (peer->state & FLAG_OUTGOING)
+      fprintf(stderr, "succeeded.\n");
+    else
+      fprintf(stderr, "Incoming connection successful.\n");
+
+    break;
+  close_socket:
+    fprintf(stderr, "Handshake incomplete, closing socket\n");
+    io_close(peersocket);
+    reset_info_block(peer);
+    break;
+  case FLAG_CONNECTED:
+    /* Here we acutally expect data from peer
+       indata_length should be less than 20+256*7 bytes, for incomplete torrent entries */
+    datalen = io_tryread(peersocket, (void *)(peer->indata + peer->indata_length), sizeof(peer->indata) - peer->indata_length);
+    if (!datalen || datalen < -1) {
+      fprintf(stderr, "Connection closed by remote peer.\n");
+      io_close(peersocket);
+      reset_info_block(peer);
+    } else if (datalen > 0) {
+      peer->indata_length += datalen;
+      process_indata(peer);
+    }
+    break;
+  }
+}
+
+/* Can write new sync data to the stream */
+static void handle_write(int64 peersocket) {
+  proxy_peer *peer = io_getcookie(peersocket);
+
+  if (!peer) {
+    /* Can't happen ;) */
+    io_close(peersocket);
+    return;
+  }
+
+  switch (peer->state & FLAG_MASK) {
+  case FLAG_DISCONNECTED:
+  default: /* Should not happen */
+    io_close(peersocket);
+    break;
+  case FLAG_CONNECTING:
+    /* Ensure that the connection is established and handle connection error */
+    if (peer->state & FLAG_OUTGOING && !socket_connected(peersocket)) {
+      fprintf(stderr, "failed\n");
+      reset_info_block(peer);
+      io_close(peersocket);
+      break;
+    }
+
+    if (io_trywrite(peersocket, (void *)&g_tracker_id, sizeof(g_tracker_id)) == sizeof(g_tracker_id)) {
+      PROXYPEER_SETWAITTRACKERID(peer->state);
+      io_dontwantwrite(peersocket);
+      io_wantread(peersocket);
+    } else {
+      fprintf(stderr, "Handshake incomplete, closing socket\n");
+      io_close(peersocket);
+      reset_info_block(peer);
+    }
+    break;
+  case FLAG_CONNECTED:
+    switch (iob_send(peersocket, &peer->outdata)) {
+    case 0: /* all data sent */
+      io_dontwantwrite(peersocket);
+      break;
+    case -3: /* an error occured */
+      io_close(peersocket);
+      reset_info_block(peer);
+      break;
+    default: /* Normal operation or eagain */
+      break;
+    }
+    break;
+  }
+
+  return;
+}
+
+static void server_mainloop() {
+  int64 sock;
+
+  /* inlined livesync_init() */
+  memset(g_peerbuffer_start, 0, sizeof(g_peerbuffer_start));
+  g_peerbuffer_pos = g_peerbuffer_start;
+  memcpy(g_peerbuffer_pos, &g_tracker_id, sizeof(g_tracker_id));
+  uint32_pack_big((char *)g_peerbuffer_pos + sizeof(g_tracker_id), OT_SYNC_PEER);
+  g_peerbuffer_pos   += sizeof(g_tracker_id) + sizeof(uint32_t);
+  g_next_packet_time  = time(NULL) + LIVESYNC_MAXDELAY;
+
+  while (1) {
+    /* See if we need to connect to anyone */
+    if (time(NULL) > g_connection_reconn)
+      handle_reconnects();
+
+    /* Wait for io events until next approx reconn check time */
+    io_waituntil2(30 * 1000);
+
+    /* Loop over readable sockets */
+    while ((sock = io_canread()) != -1) {
+      const void *cookie = io_getcookie(sock);
+      if ((uintptr_t)cookie == FLAG_SERVERSOCKET)
+        handle_accept(sock);
+      else
+        handle_read(sock);
+    }
+
+    /* Loop over writable sockets */
+    while ((sock = io_canwrite()) != -1)
+      handle_write(sock);
+
+    livesync_ticker();
+  }
+}
+
+static void panic(const char *routine) {
+  fprintf(stderr, "%s: %s\n", routine, strerror(errno));
+  exit(111);
+}
+
+static int64_t ot_try_bind(ot_ip6 ip, uint16_t port) {
+  int64 sock = socket_tcp6();
+
+  if (socket_bind6_reuse(sock, ip, port, 0) == -1)
+    panic("socket_bind6_reuse");
+
+  if (socket_listen(sock, SOMAXCONN) == -1)
+    panic("socket_listen");
+
+  if (!io_fd(sock))
+    panic("io_fd");
+
+  io_setcookie(sock, (void *)FLAG_SERVERSOCKET);
+  io_wantread(sock);
+  return sock;
+}
+
+static int scan_ip6_port(const char *src, ot_ip6 ip, uint16 *port) {
+  const char *s = src;
+  int         off, bracket = 0;
+  while (isspace(*s))
+    ++s;
+  if (*s == '[')
+    ++s, ++bracket; /* for v6 style notation */
+  if (!(off = scan_ip6(s, ip)))
+    return 0;
+  s += off;
+  if (*s == 0 || isspace(*s))
+    return s - src;
+  if (*s == ']' && bracket)
+    ++s;
+  if (!ip6_isv4mapped(ip)) {
+    if ((bracket && *(s) != ':') || (*(s) != '.'))
+      return 0;
+    s++;
+  } else {
+    if (*(s++) != ':')
+      return 0;
+  }
+  if (!(off = scan_ushort(s, port)))
+    return 0;
+  return off + s - src;
+}
+
+int main(int argc, char **argv) {
+  static pthread_t sync_in_thread_id;
+  static pthread_t sync_out_thread_id;
+  ot_ip6           serverip;
+  uint16_t         tmpport;
+  int              scanon = 1, lbound = 0, sbound = 0;
+
+  srandom(time(NULL));
+#ifdef WANT_ARC4RANDOM
+  g_tracker_id = arc4random();
+#else
+  g_tracker_id = random();
+#endif
+
+  while (scanon) {
+    switch (getopt(argc, argv, ":l:c:L:h")) {
+    case -1:
+      scanon = 0;
+      break;
+    case 'l':
+      tmpport = 0;
+      if (!scan_ip6_port(optarg, serverip, &tmpport) || !tmpport) {
+        usage(argv[0]);
+        exit(1);
+      }
+      ot_try_bind(serverip, tmpport);
+      ++sbound;
+      break;
+    case 'c':
+      if (g_connection_count > MAX_PEERS / 2)
+        exerr("Connection limit exceeded.\n");
+      tmpport = 0;
+      if (!scan_ip6_port(optarg, g_connections[g_connection_count].ip, &g_connections[g_connection_count].port) || !g_connections[g_connection_count].port) {
+        usage(argv[0]);
+        exit(1);
+      }
+      g_connections[g_connection_count++].state = FLAG_OUTGOING;
+      break;
+    case 'L':
+      tmpport = 9696;
+      if (!scan_ip6_port(optarg, serverip, &tmpport) || !tmpport) {
+        usage(argv[0]);
+        exit(1);
+      }
+      livesync_bind_mcast(serverip, tmpport);
+      ++lbound;
+      break;
+    default:
+    case '?':
+      usage(argv[0]);
+      exit(1);
+    }
+  }
+
+  if (!lbound)
+    exerr("No livesync port bound.");
+  if (!g_connection_count && !sbound)
+    exerr("No streamsync port bound.");
+  pthread_create(&sync_in_thread_id, NULL, livesync_worker, NULL);
+  pthread_create(&sync_out_thread_id, NULL, streamsync_worker, NULL);
+
+  server_mainloop();
+  return 0;
+}
+
+static void *streamsync_worker(void *args) {
+  (void)args;
+  while (1) {
+    int bucket;
+    /* For each bucket... */
+    for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) {
+      /* Get exclusive access to that bucket */
+      ot_vector *torrents_list = mutex_bucket_lock(bucket);
+      size_t     tor_offset, count_def = 0, count_one = 0, count_two = 0, count_peers = 0;
+      size_t     mem, mem_a = 0, mem_b = 0;
+      uint8_t   *ptr = 0, *ptr_a, *ptr_b, *ptr_c;
+
+      if (!torrents_list->size)
+        goto unlock_continue;
+
+      /* For each torrent in this bucket.. */
+      for (tor_offset = 0; tor_offset < torrents_list->size; ++tor_offset) {
+        /* Address torrents members */
+        ot_peerlist *peer_list = (((ot_torrent *)(torrents_list->data))[tor_offset]).peer_list;
+        switch (peer_list->peer_count) {
+        case 2:
+          count_two++;
+          break;
+        case 1:
+          count_one++;
+          break;
+        case 0:
+          break;
+        default:
+          count_def++;
+          count_peers += peer_list->peer_count;
+        }
+      }
+
+      /* Maximal memory requirement: max 3 blocks, max torrents * 20 + max peers * 7 */
+      mem = 3 * (1 + 1 + 2) + (count_one + count_two) * (19 + 1) + count_def * (19 + 8) + (count_one + 2 * count_two + count_peers) * 7;
+
+      fprintf(stderr, "Mem: %zd\n", mem);
+
+      ptr = ptr_a = ptr_b = ptr_c = malloc(mem);
+      if (!ptr)
+        goto unlock_continue;
+
+      if (count_one > 4 || !count_def) {
+        mem_a     = 1 + 1 + 2 + count_one * (19 + 7);
+        ptr_b    += mem_a;
+        ptr_c    += mem_a;
+        ptr_a[0]  = 1;                                     /* Offset 0: packet type 1 */
+        ptr_a[1]  = (bucket << 8) >> OT_BUCKET_COUNT_BITS; /* Offset 1: the shared prefix */
+        ptr_a[2]  = count_one >> 8;
+        ptr_a[3]  = count_one & 255;
+        ptr_a    += 4;
+      } else
+        count_def += count_one;
+
+      if (count_two > 4 || !count_def) {
+        mem_b     = 1 + 1 + 2 + count_two * (19 + 14);
+        ptr_c    += mem_b;
+        ptr_b[0]  = 2;                                     /* Offset 0: packet type 2 */
+        ptr_b[1]  = (bucket << 8) >> OT_BUCKET_COUNT_BITS; /* Offset 1: the shared prefix */
+        ptr_b[2]  = count_two >> 8;
+        ptr_b[3]  = count_two & 255;
+        ptr_b    += 4;
+      } else
+        count_def += count_two;
+
+      if (count_def) {
+        ptr_c[0]  = 0;                                     /* Offset 0: packet type 0 */
+        ptr_c[1]  = (bucket << 8) >> OT_BUCKET_COUNT_BITS; /* Offset 1: the shared prefix */
+        ptr_c[2]  = count_def >> 8;
+        ptr_c[3]  = count_def & 255;
+        ptr_c    += 4;
+      }
+
+      /* For each torrent in this bucket.. */
+      for (tor_offset = 0; tor_offset < torrents_list->size; ++tor_offset) {
+        /* Address torrents members */
+        ot_torrent  *torrent   = ((ot_torrent *)(torrents_list->data)) + tor_offset;
+        ot_peerlist *peer_list = torrent->peer_list;
+        ot_peer     *peers     = (ot_peer *)(peer_list->peers.data);
+        uint8_t    **dst;
+
+        /* Determine destination slot */
+        count_peers = peer_list->peer_count;
+        switch (count_peers) {
+        case 0:
+          continue;
+        case 1:
+          dst = mem_a ? &ptr_a : &ptr_c;
+          break;
+        case 2:
+          dst = mem_b ? &ptr_b : &ptr_c;
+          break;
+        default:
+          dst = &ptr_c;
+          break;
+        }
+
+        /* Copy tail of info_hash, advance pointer */
+        memcpy(*dst, ((uint8_t *)torrent->hash) + 1, sizeof(ot_hash) - 1);
+        *dst += sizeof(ot_hash) - 1;
+
+        /* Encode peer count */
+        if (dst == &ptr_c)
+          while (count_peers) {
+            if (count_peers <= 0x7f)
+              *(*dst)++ = count_peers;
+            else
+              *(*dst)++ = 0x80 | (count_peers & 0x7f);
+            count_peers >>= 7;
+          }
+
+        /* Copy peers */
+        count_peers = peer_list->peer_count;
+        while (count_peers--) {
+          memcpy(*dst, peers++, OT_IP_SIZE + 3);
+          *dst += OT_IP_SIZE + 3;
+        }
+        free_peerlist(peer_list);
+      }
+
+      free(torrents_list->data);
+      memset(torrents_list, 0, sizeof(*torrents_list));
+    unlock_continue:
+      mutex_bucket_unlock(bucket, 0);
+
+      if (ptr) {
+        int i;
+
+        if (ptr_b > ptr_c)
+          ptr_c = ptr_b;
+        if (ptr_a > ptr_c)
+          ptr_c = ptr_a;
+        mem = ptr_c - ptr;
+
+        for (i = 0; i < MAX_PEERS; ++i) {
+          if (PROXYPEER_ISCONNECTED(g_connections[i].state)) {
+            void *tmp = malloc(mem);
+            if (tmp) {
+              memcpy(tmp, ptr, mem);
+              iob_addbuf_free(&g_connections[i].outdata, tmp, mem);
+              io_wantwrite(g_connections[i].fd);
+            }
+          }
+        }
+
+        free(ptr);
+      }
+      usleep(OT_SYNC_SLEEP);
+    }
+  }
+  return 0;
+}
+
+static void livesync_issue_peersync() {
+  socket_send4(g_socket_out, (char *)g_peerbuffer_start, g_peerbuffer_pos - g_peerbuffer_start, groupip_1, LIVESYNC_PORT);
+  g_peerbuffer_pos   = g_peerbuffer_start + sizeof(g_tracker_id) + sizeof(uint32_t);
+  g_next_packet_time = time(NULL) + LIVESYNC_MAXDELAY;
+}
+
+void livesync_ticker() {
+  /* livesync_issue_peersync sets g_next_packet_time */
+  if (time(NULL) > g_next_packet_time && g_peerbuffer_pos > g_peerbuffer_start + sizeof(g_tracker_id))
+    livesync_issue_peersync();
+}
+
+static void livesync_proxytell(uint8_t prefix, uint8_t *info_hash, uint8_t *peer) {
+  //  unsigned int i;
+
+  *g_peerbuffer_pos = prefix;
+  memcpy(g_peerbuffer_pos + 1, info_hash, sizeof(ot_hash) - 1);
+  memcpy(g_peerbuffer_pos + sizeof(ot_hash), peer, sizeof(ot_peer) - 1);
+
+#if 0
+  /* Dump info_hash */
+  for( i=0; i<sizeof(ot_hash); ++i )
+    printf( "%02X", g_peerbuffer_pos[i] );
+  putchar( ':' );
+#endif
+  g_peerbuffer_pos += sizeof(ot_hash);
+#if 0
+  printf( "%hhu.%hhu.%hhu.%hhu:%hu (%02X %02X)\n", g_peerbuffer_pos[0], g_peerbuffer_pos[1], g_peerbuffer_pos[2], g_peerbuffer_pos[3],
+    g_peerbuffer_pos[4] | ( g_peerbuffer_pos[5] << 8 ), g_peerbuffer_pos[6], g_peerbuffer_pos[7] );
+#endif
+  g_peerbuffer_pos += sizeof(ot_peer);
+
+  if (g_peerbuffer_pos >= g_peerbuffer_highwater)
+    livesync_issue_peersync();
+}
+
+static void process_indata(proxy_peer *peer) {
+  size_t   consumed, peers;
+  uint8_t *data    = peer->indata, *hash;
+  uint8_t *dataend = data + peer->indata_length;
+
+  while (1) {
+    /* If we're not inside of a packet, make a new one */
+    if (!peer->packet_tcount) {
+      /* Ensure the header is complete or postpone processing */
+      if (data + 4 > dataend)
+        break;
+      peer->packet_type     = data[0];
+      peer->packet_tprefix  = data[1];
+      peer->packet_tcount   = data[2] * 256 + data[3];
+      data                 += 4;
+      printf("type: %hhu, prefix: %02X, torrentcount: %zd\n", peer->packet_type, peer->packet_tprefix, peer->packet_tcount);
+    }
+
+    /* Ensure size for a minimal torrent block */
+    if (data + sizeof(ot_hash) + OT_IP_SIZE + 3 > dataend)
+      break;
+
+    /* Advance pointer to peer count or peers */
+    hash   = data;
+    data  += sizeof(ot_hash) - 1;
+
+    /* Type 0 has peer count encoded before each peers */
+    peers  = peer->packet_type;
+    if (!peers) {
+      int shift = 0;
+      do
+        peers |= (0x7f & *data) << (7 * shift);
+      while (*(data++) & 0x80 && shift++ < 6);
+    }
+#if 0
+printf( "peers: %zd\n", peers );
+#endif
+    /* Ensure enough data being read to hold all peers */
+    if (data + (OT_IP_SIZE + 3) * peers > dataend) {
+      data = hash;
+      break;
+    }
+    while (peers--) {
+      livesync_proxytell(peer->packet_tprefix, hash, data);
+      data += OT_IP_SIZE + 3;
+    }
+    --peer->packet_tcount;
+  }
+
+  consumed = data - peer->indata;
+  memmove(peer->indata, data, peer->indata_length - consumed);
+  peer->indata_length -= consumed;
+}
+
+static void *livesync_worker(void *args) {
+  (void)args;
+  while (1) {
+    ot_ip6   in_ip;
+    uint16_t in_port;
+    size_t   datalen = socket_recv4(g_socket_in, (char *)g_inbuffer, LIVESYNC_INCOMING_BUFFSIZE, 12 + (char *)in_ip, &in_port);
+
+    /* Expect at least tracker id and packet type */
+    if (datalen <= (ssize_t)(sizeof(g_tracker_id) + sizeof(uint32_t)))
+      continue;
+    if (!memcmp(g_inbuffer, &g_tracker_id, sizeof(g_tracker_id))) {
+      /* drop packet coming from ourselves */
+      continue;
+    }
+    switch (uint32_read_big((char *)g_inbuffer + sizeof(g_tracker_id))) {
+    case OT_SYNC_PEER4:
+      livesync_handle_peersync(datalen, OT_PEER_SIZE4);
+      break;
+    case OT_SYNC_PEER6:
+      livesync_handle_peersync(datalen, OT_PEER_SIZE6);
+      break;
+    default:
+      // fprintf( stderr, "Received an unknown live sync packet type %u.\n", uint32_read_big( sizeof( g_tracker_id ) + (char*)g_inbuffer ) );
+      break;
+    }
+  }
+  return 0;
+}

scan_urlencoded_query.c

@@ -3,9 +3,15 @@
 
    $id$ */
 
-#include "scan.h"
+/* Opentracker */
 #include "scan_urlencoded_query.h"
 
+/* Libwofat */
+#include "scan.h"
+
+/* System */
+#include <string.h>
+
 /* Idea is to do a in place replacement or guarantee at least
    strlen( string ) bytes in deststring
    watch http://www.ietf.org/rfc/rfc2396.txt
@@ -28,7 +34,7 @@
   */
 static const unsigned char is_unreserved[256] = {
   8,0,0,0,0,0,0,0,0,0,8,0,0,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-  0,7,8,8,8,7,0,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,4,7,6,
+  8,7,8,8,8,7,0,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,4,7,6,
   4,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,8,8,8,8,7,
   8,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,8,8,8,7,0,
   0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
@@ -39,102 +45,120 @@ static const unsigned char is_unreserved[256] = {
 
 /* Do a fast nibble to hex representation conversion */
 static unsigned char fromhex(unsigned char x) {
-  x-='0'; if( x<=9) return x;
-  x&=~0x20; x-='A'-'0';
-  if( x<6 ) return x+10;
+  x -= '0';
+  if (x <= 9)
+    return x;
+  x &= ~0x20;
+  x -= 'A' - '0';
+  if (x < 6)
+    return x + 10;
   return 0xff;
 }
 
 /* Skip the value of a param=value pair */
-void scan_urlencoded_skipvalue( char **string ) {
-  const unsigned char* s=*(const unsigned char**) string;
-  unsigned char f;
+void scan_urlencoded_skipvalue(char **string) {
+  const unsigned char *s = *(const unsigned char **)string;
+  unsigned char        f;
 
   /* Since we are asked to skip the 'value', we assume to stop at
      terminators for a 'value' string position */
-  while( ( f = is_unreserved[ *s++ ] ) & SCAN_SEARCHPATH_VALUE );
+  while ((f = is_unreserved[*s++]) & SCAN_SEARCHPATH_VALUE)
+    ;
 
   /* If we stopped at a hard terminator like \0 or \n, make the
      next scan_urlencoded_query encounter it again */
-  if( f & SCAN_SEARCHPATH_TERMINATOR ) --s;
+  if (f & SCAN_SEARCHPATH_TERMINATOR)
+    --s;
 
-  *string = (char*)s;
+  *string = (char *)s;
+}
+
+int scan_find_keywords(const ot_keywords *keywords, char **string, SCAN_SEARCHPATH_FLAG flags) {
+  char   *deststring   = *string;
+  ssize_t match_length = scan_urlencoded_query(string, deststring, flags);
+
+  if (match_length < 0)
+    return match_length;
+  if (match_length == 0)
+    return -3;
+
+  while (keywords->key) {
+    if (!strncmp(keywords->key, deststring, match_length) && !keywords->key[match_length])
+      return keywords->value;
+    keywords++;
+  }
+
+  return -3;
 }
 
 ssize_t scan_urlencoded_query(char **string, char *deststring, SCAN_SEARCHPATH_FLAG flags) {
-  const unsigned char* s=*(const unsigned char**) string;
-  unsigned char *d = (unsigned char*)deststring;
-  unsigned char b, c;
+  const unsigned char *s = *(const unsigned char **)string;
+  unsigned char       *d = (unsigned char *)deststring;
+  unsigned char        b, c;
 
   /* This is the main decoding loop.
     'flag' determines, which characters are non-terminating in current context
     (ie. stop at '=' and '&' if scanning for a 'param'; stop at '?' if scanning for the path )
   */
-  while( is_unreserved[ c = *s++ ] & flags ) {
+  while (is_unreserved[c = *s++] & flags) {
 
     /* When encountering an url escaped character, try to decode */
-    if( c=='%') {
-      if( ( b = fromhex(*s++) ) == 0xff ) return -1;
-      if( ( c = fromhex(*s++) ) == 0xff ) return -1;
-      c|=(b<<4);
+    if (c == '%') {
+      if ((b = fromhex(*s++)) == 0xff)
+        return -1;
+      if ((c = fromhex(*s++)) == 0xff)
+        return -1;
+      c |= (b << 4);
     }
 
     /* Write (possibly decoded) character to output */
     *d++ = c;
   }
 
-  switch( c ) {
-  case 0: case '\r': case '\n': case ' ':
+  switch (c) {
+  case 0:
+  case '\r':
+  case '\n':
+  case ' ':
     /* If we started scanning on a hard terminator, indicate we've finished */
-    if( d == (unsigned char*)deststring ) return -2;
+    if (d == (unsigned char *)deststring)
+      return -2;
 
     /* Else make the next call to scan_urlencoded_param encounter it again */
     --s;
     break;
   case '?':
-    /* XXX to help us parse path?param=value?param=value?... sent by µTorrent 1600
-       do not return an error but silently terminate
-    if( flags != SCAN_PATH ) return -1; */
+    if (flags != SCAN_PATH)
+      return -1;
     break;
   case '=':
-    if( flags != SCAN_SEARCHPATH_PARAM ) return -1;
+    if (flags != SCAN_SEARCHPATH_PARAM)
+      return -1;
     break;
   case '&':
-    if( flags == SCAN_PATH ) return -1;
-    if( flags == SCAN_SEARCHPATH_PARAM ) --s;
+    if (flags == SCAN_PATH)
+      return -1;
+    if (flags == SCAN_SEARCHPATH_PARAM)
+      --s;
     break;
   default:
     return -1;
   }
 
   *string = (char *)s;
-  return d - (unsigned char*)deststring;
+  return d - (unsigned char *)deststring;
 }
 
-ssize_t scan_fixed_int( char *data, size_t len, int *tmp ) {
+ssize_t scan_fixed_int(char *data, size_t len, int *tmp) {
   int minus = 0;
-  *tmp = 0;
-  if( *data == '-' ) --len, ++data, ++minus;
-  while( (len > 0) && (*data >= '0') && (*data <= '9') ) { --len; *tmp = 10**tmp + *data++-'0'; }
-  if( minus ) *tmp = -*tmp;
-  return len;
-}
-
-ssize_t scan_fixed_ip( char *data, size_t len, unsigned char ip[4] ) {
-  int u, i;
-
-  for( i=0; i<4; ++i ) {
-    ssize_t j = scan_fixed_int( data, len, &u );
-    if( j == (ssize_t)len ) return len;
-    ip[i] = u;
-    data += len - j;
-    len = j;
-    if ( i<3 ) {
-      if( !len || *data != '.') return -1;
-      --len; ++data;
-    }
+  *tmp      = 0;
+  if (*data == '-')
+    --len, ++data, ++minus;
+  while ((len > 0) && (*data >= '0') && (*data <= '9')) {
+    --len;
+    *tmp = 10 * *tmp + *data++ - '0';
   }
+  if (minus)
+    *tmp = -*tmp;
   return len;
 }
-
-const char *g_version_scan_urlencoded_query_c = "$Source$: $Revision$\n";

scan_urlencoded_query.h

@@ -3,8 +3,15 @@
 
    $id$ */
 
-#ifndef __SCAN_URLENCODED_QUERY_H__
-#define __SCAN_URLENCODED_QUERY_H__
+#ifndef SCAN_URLENCODED_QUERY_H__
+#define SCAN_URLENCODED_QUERY_H__
+
+#include <sys/types.h>
+
+typedef struct {
+  char *key;
+  int   value;
+} ot_keywords;
 
 typedef enum {
   SCAN_PATH                  = 1,
@@ -19,26 +26,30 @@ typedef enum {
    flags      determines, what to parse
    returns    number of valid converted characters in deststring
               or -1 for parse error
+              or -2 for terminator found
 */
 ssize_t scan_urlencoded_query(char **string, char *deststring, SCAN_SEARCHPATH_FLAG flags);
 
+/* string     in: pointer to source
+              out: pointer to next scan position
+   flags      determines, what to parse
+   returns    value for matched keyword
+              or -1 for parse error
+              or -2 for terminator found
+              or -3 for no keyword matched
+ */
+int     scan_find_keywords(const ot_keywords *keywords, char **string, SCAN_SEARCHPATH_FLAG flags);
+
 /* string     in: pointer to value of a param=value pair to skip
               out: pointer to next scan position on return
 */
-void scan_urlencoded_skipvalue( char **string );
+void    scan_urlencoded_skipvalue(char **string);
 
 /* data       pointer to len chars of string
-   len        length of chars in data to parse
-   number     number to receive result
-   returns    number of bytes not parsed, mostly !=0 means fail
-*/
-ssize_t scan_fixed_int( char *data, size_t len, int *number );
-
-/* data       pointer to len chars of string
-   len        length of chars in data to parse
-   ip         buffer to receive result
-   returns    number of bytes not parsed, mostly !=0 means fail
-*/
-ssize_t scan_fixed_ip( char *data, size_t len, unsigned char ip[4] );
+ len        length of chars in data to parse
+ number     number to receive result
+ returns    number of bytes not parsed, mostly !=0 means fail
+ */
+ssize_t scan_fixed_int(char *data, size_t len, int *number);
 
 #endif

tests/testsuite.sh

@@ -1,15 +1,11 @@
 #!/bin/sh
 
 while true; do
-  request_string="GET /announce?info_hash=\
-%$(printf %02X $(( $RANDOM & 0xff )) )\
-%$(printf %02X $(( $RANDOM & 0xff )) )\
-2345678901234567\
-%$(printf %02X $(( $RANDOM & 0xff )) )\
-%$(printf %02X $(( $RANDOM & 0xff )) )\
-&ip=$(( $RANDOM & 0xff )).17.13.15&port=$(( $RANDOM & 0xff )) HTTP/1.0\n"
+  request_string="GET /announce?info_hash=0123456789012345678\
+%$(printf %02X $(( $RANDOM & 0xf )) )\
+&ip=$(( $RANDOM & 0xf )).$(( $RANDOM & 0xf )).13.16&port=$(( $RANDOM & 0xff )) HTTP/1.0\n"
 
-#  echo $request_string
+echo $request_string
 #  echo
   echo $request_string | nc 127.0.0.1 6969 >/dev/null
 #  echo

tests/testsuite2.sh

@@ -2,13 +2,21 @@
 
 while true; do
   request_string="GET /announce?info_hash=012345678901234567\
-%$(printf %02X $(( $RANDOM & 0xff )) )\
-%$(printf %02X $(( $RANDOM & 0xff )) )\
-&ip=$(( $RANDOM & 0xff )).17.13.15&port=$(( $RANDOM & 0xff )) HTTP/1.0\n"
+$(printf %02X $(( $RANDOM & 0xff )) )\
+&ip=$(( $RANDOM & 0xff )).17.13.15&port=$(( $RANDOM & 0xff )) HTTP/1.0"
 
-  echo $request_string
-  echo
-  echo $request_string | nc 10.0.1.3 6969 >/dev/null
-  echo
+#  echo $request_string
+#  echo
+  printf "%s\n\n" "$request_string" | nc 84.200.61.9 6969 | hexdump -C
+
+  request_string="GET /announce?info_hash=012345678901234567\
+$(printf %02X $(( $RANDOM & 0xff )) )\
+&ip=2001:1608:6:27::$(( $RANDOM & 0xff ))&port=$(( $RANDOM & 0xff )) HTTP/1.0"
+  printf "%s\n\n" "$request_string" | nc 2001:1608:6:27::9 6969 | hexdump -C
+  printf "%s\n\n" "$request_string"
+
+  request_string="GET /scrape?info_hash=012345678901234567\
+$(printf %02X $(( $RANDOM & 0xff )) ) HTTP/1.0"
+  printf "%s\n\n" "$request_string" | nc 2001:1608:6:27::9 6969 | hexdump -C
 
 done

trackerlogic.c

@@ -4,389 +4,595 @@
    $id$ */
 
 /* System */
+#include <arpa/inet.h>
+#include <errno.h>
+#include <stdint.h>
+#include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <stdio.h>
-#include <sys/uio.h>
-#include <arpa/inet.h>
-#include <sys/types.h>
-#include <sys/mman.h>
 #include <unistd.h>
-#include <time.h>
-#include <math.h>
-#include <errno.h>
 
 /* Libowfat */
-#include "scan.h"
+#include "array.h"
 #include "byte.h"
 #include "io.h"
+#include "iob.h"
+#include "ip6.h"
 
 /* Opentracker */
-#include "trackerlogic.h"
+#include "ot_accesslist.h"
+#include "ot_clean.h"
+#include "ot_fullscrape.h"
+#include "ot_http.h"
+#include "ot_livesync.h"
 #include "ot_mutex.h"
 #include "ot_stats.h"
-#include "ot_clean.h"
-#include "ot_accesslist.h"
-#include "ot_fullscrape.h"
-#include "ot_sync.h"
-#include "ot_livesync.h"
+#include "ot_vector.h"
+#include "trackerlogic.h"
 
-void free_peerlist( ot_peerlist *peer_list ) {
-  size_t i;
-  for( i=0; i<OT_POOLS_COUNT; ++i )
-    if( peer_list->peers[i].data )
-      free( peer_list->peers[i].data );
-#ifdef WANT_SYNC_BATCH
-  free( peer_list->changeset.data );
-#endif
-  free( peer_list );
+/* Forward declaration */
+size_t return_peers_for_torrent(struct ot_workstruct *ws, ot_torrent *torrent, size_t amount, char *reply, PROTO_FLAG proto);
+
+void   free_peerlist(ot_peerlist *peer_list) {
+  if (peer_list->peers.data) {
+    if (OT_PEERLIST_HASBUCKETS(peer_list))
+      vector_clean_list((ot_vector *)peer_list->peers.data, peer_list->peers.size);
+    else
+      free(peer_list->peers.data);
+  }
+  free(peer_list);
 }
 
-ot_torrent *add_peer_to_torrent( ot_hash *hash, ot_peer *peer  WANT_SYNC_PARAM( int from_changeset ) ) {
+void add_torrent_from_saved_state(ot_hash const hash, ot_time base, size_t down_count) {
   int         exactmatch;
   ot_torrent *torrent;
-  ot_peer    *peer_dest;
-  ot_vector  *torrents_list = mutex_bucket_lock_by_hash( hash ), *peer_pool;
-  int         base_pool = 0;
+  ot_vector  *torrents_list = mutex_bucket_lock_by_hash(hash);
 
-  if( !accesslist_hashisvalid( hash ) ) {
-    mutex_bucket_unlock_by_hash( hash );
-    return NULL;
+  if (!accesslist_hashisvalid(hash))
+    return mutex_bucket_unlock_by_hash(hash, 0);
+
+  torrent = vector_find_or_insert(torrents_list, (void *)hash, sizeof(ot_torrent), OT_HASH_COMPARE_SIZE, &exactmatch);
+  if (!torrent || exactmatch)
+    return mutex_bucket_unlock_by_hash(hash, 0);
+
+  /* Create a new torrent entry, then */
+  byte_zero(torrent, sizeof(ot_torrent));
+  memcpy(torrent->hash, hash, sizeof(ot_hash));
+
+  if (!(torrent->peer_list6 = malloc(sizeof(ot_peerlist))) || !(torrent->peer_list4 = malloc(sizeof(ot_peerlist)))) {
+    vector_remove_torrent(torrents_list, torrent);
+    return mutex_bucket_unlock_by_hash(hash, 0);
   }
 
-  torrent = vector_find_or_insert( torrents_list, (void*)hash, sizeof( ot_torrent ), OT_HASH_COMPARE_SIZE, &exactmatch );
-  if( !torrent ) {
-    mutex_bucket_unlock_by_hash( hash );
-    return NULL;
-  }
+  byte_zero(torrent->peer_list6, sizeof(ot_peerlist));
+  byte_zero(torrent->peer_list4, sizeof(ot_peerlist));
+  torrent->peer_list6->base       = base;
+  torrent->peer_list4->base       = base;
+  torrent->peer_list6->down_count = down_count;
+  torrent->peer_list4->down_count = down_count;
 
-  if( !exactmatch ) {
-    /* Create a new torrent entry, then */
-    memmove( &torrent->hash, hash, sizeof( ot_hash ) );
-
-    if( !( torrent->peer_list = malloc( sizeof (ot_peerlist) ) ) ) {
-      vector_remove_torrent( torrents_list, torrent );
-      mutex_bucket_unlock_by_hash( hash );
-      return NULL;
-    }
-
-    byte_zero( torrent->peer_list, sizeof( ot_peerlist ) );
-  } else
-    clean_single_torrent( torrent );
-
-  /* Timestamp our first pool */
-  torrent->peer_list->base = NOW;
-
-  /* Sanitize flags: Whoever claims to have completed download, must be a seeder */
-  if( ( OT_FLAG( peer ) & ( PEER_FLAG_COMPLETED | PEER_FLAG_SEEDING ) ) == PEER_FLAG_COMPLETED )
-    OT_FLAG( peer ) ^= PEER_FLAG_COMPLETED;
-
-#ifdef WANT_SYNC
-  if( from_changeset ) {
-    /* Check, whether peer already is in current pool, do nothing if so */
-    peer_pool = &torrent->peer_list->peers[0];
-    binary_search( peer, peer_pool->data, peer_pool->size, sizeof(ot_peer), OT_PEER_COMPARE_SIZE, &exactmatch );
-    if( exactmatch ) {
-      mutex_bucket_unlock_by_hash( hash );
-      return torrent;
-    }
-    base_pool = 1;
-    if( torrent->peer_list->base < NOW )
-      torrent->peer_list->base = NOW;
-  }
-#endif
-
-  peer_pool = &torrent->peer_list->peers[ base_pool ];
-  peer_dest = vector_find_or_insert( peer_pool, (void*)peer, sizeof( ot_peer ), OT_PEER_COMPARE_SIZE, &exactmatch );
-
-  /* If we hadn't had a match in current pool, create peer there and
-     remove it from all older pools */
-  if( !exactmatch ) {
-    int i;
-    memmove( peer_dest, peer, sizeof( ot_peer ) );
-    torrent->peer_list->peer_count++;
-
-#ifdef WANT_SYNC_LIVE
-    if( !from_changeset )
-      livesync_tell( hash, peer, PEER_FLAG_LEECHING );
-#endif
-
-    if( OT_FLAG( peer ) & PEER_FLAG_COMPLETED )
-      torrent->peer_list->down_count++;
-
-    if( OT_FLAG(peer) & PEER_FLAG_SEEDING ) {
-      torrent->peer_list->seed_counts[ base_pool ]++;
-      torrent->peer_list->seed_count++;
-    }
-
-    for( i= base_pool + 1; i<OT_POOLS_COUNT; ++i ) {
-      switch( vector_remove_peer( &torrent->peer_list->peers[i], peer, 0 ) ) {
-        case 0: continue;
-        case 2: torrent->peer_list->seed_counts[i]--;
-                torrent->peer_list->seed_count--;
-        case 1: default:
-                torrent->peer_list->peer_count--;
-                mutex_bucket_unlock_by_hash( hash );
-                stats_issue_event( EVENT_RENEW, 0, i );
-                return torrent;
-      }
-    }
-  } else {
-    if( (OT_FLAG(peer_dest) & PEER_FLAG_SEEDING ) && !(OT_FLAG(peer) & PEER_FLAG_SEEDING ) ) {
-      torrent->peer_list->seed_counts[ base_pool ]--;
-      torrent->peer_list->seed_count--;
-    }
-    if( !(OT_FLAG(peer_dest) & PEER_FLAG_SEEDING ) && (OT_FLAG(peer) & PEER_FLAG_SEEDING ) ) {
-      torrent->peer_list->seed_counts[ base_pool ]++;
-      torrent->peer_list->seed_count++;
-    }
-    if( !(OT_FLAG( peer_dest ) & PEER_FLAG_COMPLETED ) && (OT_FLAG( peer ) & PEER_FLAG_COMPLETED ) )
-      torrent->peer_list->down_count++;
-    if( OT_FLAG( peer_dest ) & PEER_FLAG_COMPLETED )
-      OT_FLAG( peer ) |= PEER_FLAG_COMPLETED;
-
-    stats_issue_event( EVENT_RENEW, 0, base_pool );
-    memmove( peer_dest, peer, sizeof( ot_peer ) );
-  }
-
-  mutex_bucket_unlock_by_hash( hash );
-  return torrent;
+  return mutex_bucket_unlock_by_hash(hash, 1);
 }
 
-/* Compiles a list of random peers for a torrent
-   * reply must have enough space to hold 92+6*amount bytes
-   * Selector function can be anything, maybe test for seeds, etc.
-   * RANDOM may return huge values
-   * does not yet check not to return self
-*/
-size_t return_peers_for_torrent( ot_hash *hash, size_t amount, char *reply, PROTO_FLAG proto ) {
-  char        *r = reply;
-  int          exactmatch;
-  ot_vector   *torrents_list = mutex_bucket_lock_by_hash( hash );
-  ot_torrent  *torrent = binary_search( hash, torrents_list->data, torrents_list->size, sizeof( ot_torrent ), OT_HASH_COMPARE_SIZE, &exactmatch );
-  ot_peerlist *peer_list = torrent->peer_list;
-  size_t       index;
+size_t add_peer_to_torrent_and_return_peers(PROTO_FLAG proto, struct ot_workstruct *ws, size_t amount) {
+  int            exactmatch, delta_torrentcount = 0;
+  ot_torrent    *torrent;
+  ot_peer       *peer_dest;
+  ot_vector     *torrents_list = mutex_bucket_lock_by_hash(*ws->hash);
+  ot_peerlist   *peer_list;
+  size_t         peer_size; /* initialized in next line */
+  ot_peer const *peer_src = peer_from_peer6(&ws->peer, &peer_size);
 
-  if( !torrent ) {
-    mutex_bucket_unlock_by_hash( hash );
+  if (!accesslist_hashisvalid(*ws->hash)) {
+    mutex_bucket_unlock_by_hash(*ws->hash, 0);
+    if (proto == FLAG_TCP) {
+      const char invalid_hash[] = "d14:failure reason63:Requested download is not authorized for use with this tracker.e";
+      memcpy(ws->reply, invalid_hash, strlen(invalid_hash));
+      return strlen(invalid_hash);
+    }
     return 0;
   }
 
-  if( peer_list->peer_count < amount )
-    amount = peer_list->peer_count;
-
-  if( proto == FLAG_TCP )
-    r += sprintf( r, "d8:completei%zde10:downloadedi%zde10:incompletei%zde8:intervali%ie5:peers%zd:", peer_list->seed_count, peer_list->down_count, peer_list->peer_count-peer_list->seed_count, OT_CLIENT_REQUEST_INTERVAL_RANDOM, 6*amount );
-  else {
-    *(uint32_t*)(r+0) = htonl( OT_CLIENT_REQUEST_INTERVAL_RANDOM );
-    *(uint32_t*)(r+4) = htonl( peer_list->peer_count );
-    *(uint32_t*)(r+8) = htonl( peer_list->seed_count );
-    r += 12;
+  torrent = vector_find_or_insert(torrents_list, (void *)ws->hash, sizeof(ot_torrent), OT_HASH_COMPARE_SIZE, &exactmatch);
+  if (!torrent) {
+    mutex_bucket_unlock_by_hash(*ws->hash, 0);
+    return 0;
   }
 
-  if( amount ) {
-    unsigned int pool_offset, pool_index = 0;;
-    unsigned int shifted_pc = peer_list->peer_count;
-    unsigned int shifted_step = 0;
-    unsigned int shift = 0;
+  if (!exactmatch) {
+    /* Create a new torrent entry, then */
+    byte_zero(torrent, sizeof(ot_torrent));
+    memcpy(torrent->hash, *ws->hash, sizeof(ot_hash));
 
-    /* Make fixpoint arithmetic as exact as possible */
-#define MAXPRECBIT (1<<(8*sizeof(int)-3))
-    while( !(shifted_pc & MAXPRECBIT ) ) { shifted_pc <<= 1; shift++; }
-    shifted_step = shifted_pc/amount;
-#undef MAXPRECBIT
+    if (!(torrent->peer_list6 = malloc(sizeof(ot_peerlist))) || !(torrent->peer_list4 = malloc(sizeof(ot_peerlist)))) {
+      vector_remove_torrent(torrents_list, torrent);
+      mutex_bucket_unlock_by_hash(*ws->hash, 0);
+      return 0;
+    }
 
-    /* Initialize somewhere in the middle of peers so that
-       fixpoint's aliasing doesn't alway miss the same peers */
-    pool_offset = random() % peer_list->peer_count;
+    byte_zero(torrent->peer_list6, sizeof(ot_peerlist));
+    byte_zero(torrent->peer_list4, sizeof(ot_peerlist));
+    delta_torrentcount = 1;
+  } else
+    clean_single_torrent(torrent);
 
-    for( index = 0; index < amount; ++index ) {
-      /* This is the aliased, non shifted range, next value may fall into */
-      unsigned int diff = ( ( ( index + 1 ) * shifted_step ) >> shift ) -
-                          ( (   index       * shifted_step ) >> shift );
-      pool_offset += 1 + random() % diff;
+  torrent->peer_list6->base = g_now_minutes;
+  torrent->peer_list4->base = g_now_minutes;
 
-      while( pool_offset >= peer_list->peers[pool_index].size ) {
-        pool_offset -= peer_list->peers[pool_index].size;
-        pool_index = ( pool_index + 1 ) % OT_POOLS_COUNT;
+  peer_list = peer_size == OT_PEER_SIZE6 ? torrent->peer_list6 : torrent->peer_list4;
+
+  /* Check for peer in torrent */
+  peer_dest = vector_find_or_insert_peer(&(peer_list->peers), peer_src, peer_size, &exactmatch);
+  if (!peer_dest) {
+    mutex_bucket_unlock_by_hash(*ws->hash, delta_torrentcount);
+    return 0;
+  }
+
+  /* Tell peer that it's fresh */
+  OT_PEERTIME(ws->peer, OT_PEER_SIZE6) = 0;
+
+  /* Sanitize flags: Whoever claims to have completed download, must be a seeder */
+  if ((OT_PEERFLAG(ws->peer) & (PEER_FLAG_COMPLETED | PEER_FLAG_SEEDING)) == PEER_FLAG_COMPLETED)
+    OT_PEERFLAG(ws->peer) ^= PEER_FLAG_COMPLETED;
+
+  /* If we hadn't had a match create peer there */
+  if (!exactmatch) {
+
+#ifdef WANT_SYNC_LIVE
+    if (proto == FLAG_MCA)
+      OT_PEERFLAG(ws->peer) |= PEER_FLAG_FROM_SYNC;
+    else
+      livesync_tell(ws);
+#endif
+
+    peer_list->peer_count++;
+    if (OT_PEERFLAG(ws->peer) & PEER_FLAG_COMPLETED) {
+      peer_list->down_count++;
+      stats_issue_event(EVENT_COMPLETED, 0, (uintptr_t)ws);
+    }
+    if (OT_PEERFLAG(ws->peer) & PEER_FLAG_SEEDING)
+      peer_list->seed_count++;
+
+  } else {
+    stats_issue_event(EVENT_RENEW, 0, OT_PEERTIME(peer_dest, peer_size));
+#ifdef WANT_SPOT_WOODPECKER
+    if ((OT_PEERTIME(peer_dest, peer_size) > 0) && (OT_PEERTIME(peer_dest, peer_size) < 20))
+      stats_issue_event(EVENT_WOODPECKER, 0, (uintptr_t)&ws->peer);
+#endif
+#ifdef WANT_SYNC_LIVE
+    /* Won't live sync peers that come back too fast. Only exception:
+       fresh "completed" reports */
+    if (proto != FLAG_MCA) {
+      if (OT_PEERTIME(peer_dest, peer_size) > OT_CLIENT_SYNC_RENEW_BOUNDARY ||
+          (!(OT_PEERFLAG_D(peer_dest, peer_size) & PEER_FLAG_COMPLETED) && (OT_PEERFLAG(ws->peer) & PEER_FLAG_COMPLETED)))
+        livesync_tell(ws);
+    }
+#endif
+
+    if ((OT_PEERFLAG_D(peer_dest, peer_size) & PEER_FLAG_SEEDING) && !(OT_PEERFLAG(ws->peer) & PEER_FLAG_SEEDING))
+      peer_list->seed_count--;
+    if (!(OT_PEERFLAG_D(peer_dest, peer_size) & PEER_FLAG_SEEDING) && (OT_PEERFLAG(ws->peer) & PEER_FLAG_SEEDING))
+      peer_list->seed_count++;
+    if (!(OT_PEERFLAG_D(peer_dest, peer_size) & PEER_FLAG_COMPLETED) && (OT_PEERFLAG(ws->peer) & PEER_FLAG_COMPLETED)) {
+      peer_list->down_count++;
+      stats_issue_event(EVENT_COMPLETED, 0, (uintptr_t)ws);
+    }
+    if (OT_PEERFLAG_D(peer_dest, peer_size) & PEER_FLAG_COMPLETED)
+      OT_PEERFLAG(ws->peer) |= PEER_FLAG_COMPLETED;
+  }
+
+  memcpy(peer_dest, peer_src, peer_size);
+#ifdef WANT_SYNC
+  if (proto == FLAG_MCA) {
+    mutex_bucket_unlock_by_hash(*ws->hash, delta_torrentcount);
+    return 0;
+  }
+#endif
+
+  ws->reply_size = return_peers_for_torrent(ws, torrent, amount, ws->reply, proto);
+  mutex_bucket_unlock_by_hash(*ws->hash, delta_torrentcount);
+  return ws->reply_size;
+}
+
+static size_t return_peers_all(ot_peerlist *peer_list, size_t peer_size, char *reply) {
+  unsigned int bucket, num_buckets = 1;
+  ot_vector   *bucket_list  = &peer_list->peers;
+  size_t       compare_size = OT_PEER_COMPARE_SIZE_FROM_PEER_SIZE(peer_size);
+  size_t       result       = compare_size * peer_list->peer_count;
+  char        *r_end        = reply + result;
+
+  if (OT_PEERLIST_HASBUCKETS(peer_list)) {
+    num_buckets = bucket_list->size;
+    bucket_list = (ot_vector *)bucket_list->data;
+  }
+
+  for (bucket = 0; bucket < num_buckets; ++bucket) {
+    ot_peer *peers      = bucket_list[bucket].data;
+    size_t   peer_count = bucket_list[bucket].size;
+    while (peer_count--) {
+      if (OT_PEERFLAG_D(peers, peer_size) & PEER_FLAG_SEEDING) {
+        r_end -= compare_size;
+        memcpy(r_end, peers, compare_size);
+      } else {
+        memcpy(reply, peers, compare_size);
+        reply += compare_size;
       }
-
-      memmove( r, ((ot_peer*)peer_list->peers[pool_index].data) + pool_offset, 6 );
-      r += 6;
+      peers += peer_size;
     }
   }
-  if( proto == FLAG_TCP )
-    *r++ = 'e';
+  return result;
+}
 
-  mutex_bucket_unlock_by_hash( hash );
+static size_t return_peers_selection(struct ot_workstruct *ws, ot_peerlist *peer_list, size_t peer_size, size_t amount, char *reply) {
+  unsigned int bucket_offset, bucket_index = 0, num_buckets = 1;
+  ot_vector   *bucket_list  = &peer_list->peers;
+  unsigned int shifted_pc   = peer_list->peer_count;
+  unsigned int shifted_step = 0;
+  unsigned int shift        = 0;
+  size_t       compare_size = OT_PEER_COMPARE_SIZE_FROM_PEER_SIZE(peer_size);
+  size_t       result       = compare_size * amount;
+  char        *r_end        = reply + result;
+
+  if (OT_PEERLIST_HASBUCKETS(peer_list)) {
+    num_buckets = bucket_list->size;
+    bucket_list = (ot_vector *)bucket_list->data;
+  }
+
+  /* Make fixpoint arithmetic as exact as possible */
+#define MAXPRECBIT (1 << (8 * sizeof(int) - 3))
+  while (!(shifted_pc & MAXPRECBIT)) {
+    shifted_pc <<= 1;
+    shift++;
+  }
+  shifted_step = shifted_pc / amount;
+#undef MAXPRECBIT
+
+  /* Initialize somewhere in the middle of peers so that
+   fixpoint's aliasing doesn't alway miss the same peers */
+  bucket_offset = nrand48(ws->rand48_state) % peer_list->peer_count;
+
+  while (amount--) {
+    ot_peer *peer;
+
+    /* This is the aliased, non shifted range, next value may fall into */
+    unsigned int diff  = (((amount + 1) * shifted_step) >> shift) - ((amount * shifted_step) >> shift);
+    bucket_offset     += 1 + nrand48(ws->rand48_state) % diff;
+
+    while (bucket_offset >= bucket_list[bucket_index].size) {
+      bucket_offset -= bucket_list[bucket_index].size;
+      bucket_index   = (bucket_index + 1) % num_buckets;
+    }
+    peer = bucket_list[bucket_index].data + peer_size * bucket_offset;
+    if (OT_PEERFLAG_D(peer, peer_size) & PEER_FLAG_SEEDING) {
+      r_end -= compare_size;
+      memcpy(r_end, peer, compare_size);
+    } else {
+      memcpy(reply, peer, compare_size);
+      reply += compare_size;
+    }
+  }
+  return result;
+}
+
+static size_t return_peers_for_torrent_udp(struct ot_workstruct *ws, ot_torrent *torrent, size_t amount, char *reply) {
+  char        *r          = reply;
+  size_t       peer_size  = peer_size_from_peer6(&ws->peer);
+  ot_peerlist *peer_list  = peer_size == OT_PEER_SIZE6 ? torrent->peer_list6 : torrent->peer_list4;
+  size_t       peer_count = torrent->peer_list6->peer_count + torrent->peer_list4->peer_count;
+  size_t       seed_count = torrent->peer_list6->seed_count + torrent->peer_list4->seed_count;
+
+  if (amount > peer_list->peer_count)
+    amount = peer_list->peer_count;
+
+  *(uint32_t *)(r + 0)  = htonl(OT_CLIENT_REQUEST_INTERVAL_RANDOM);
+  *(uint32_t *)(r + 4)  = htonl(peer_count - seed_count);
+  *(uint32_t *)(r + 8)  = htonl(seed_count);
+  r                    += 12;
+
+  if (amount) {
+    if (amount == peer_list->peer_count)
+      r += return_peers_all(peer_list, peer_size, r);
+    else
+      r += return_peers_selection(ws, peer_list, peer_size, amount, r);
+  }
   return r - reply;
 }
 
+static size_t return_peers_for_torrent_tcp(struct ot_workstruct *ws, ot_torrent *torrent, size_t amount, char *reply) {
+  char  *r          = reply;
+  int    erval      = OT_CLIENT_REQUEST_INTERVAL_RANDOM;
+  size_t seed_count = torrent->peer_list6->seed_count + torrent->peer_list4->seed_count;
+  size_t down_count = torrent->peer_list6->down_count + torrent->peer_list4->down_count;
+  size_t peer_count = torrent->peer_list6->peer_count + torrent->peer_list4->peer_count - seed_count;
+
+  /* Simple case: amount of peers in both lists is less than requested, here we return all results */
+  size_t amount_v4  = torrent->peer_list4->peer_count;
+  size_t amount_v6  = torrent->peer_list6->peer_count;
+
+  /* Complex case: both lists have more than enough entries and we need to split between v4 and v6 clients */
+  if (amount_v4 + amount_v6 > amount) {
+    size_t       amount_left, percent_v6 = 0, percent_v4 = 0, left_v6, left_v4;
+    const size_t SCALE = 1024;
+
+    /* If possible, fill at least a quarter of peer from each family */
+    if (amount / 4 <= amount_v4)
+      amount_v4 = amount / 4;
+    if (amount / 4 <= amount_v6)
+      amount_v6 = amount / 4;
+
+    /* Fill the rest according to which family's pool provides more peers */
+    amount_left = amount - (amount_v4 + amount_v6);
+
+    left_v4     = torrent->peer_list4->peer_count - amount_v4;
+    left_v6     = torrent->peer_list6->peer_count - amount_v6;
+
+    if (left_v4 + left_v6) {
+      percent_v4 = (SCALE * left_v4) / (left_v4 + left_v6);
+      percent_v6 = (SCALE * left_v6) / (left_v4 + left_v6);
+    }
+
+    amount_v4 += (amount_left * percent_v4) / SCALE;
+    amount_v6 += (amount_left * percent_v6) / SCALE;
+
+    /* Integer division rounding can leave out a peer */
+    if (amount_v4 + amount_v6 < amount && amount_v6 < torrent->peer_list6->peer_count)
+      ++amount_v6;
+    if (amount_v4 + amount_v6 < amount && amount_v4 < torrent->peer_list4->peer_count)
+      ++amount_v4;
+  }
+
+  r +=
+      sprintf(r, "d8:completei%zde10:downloadedi%zde10:incompletei%zde8:intervali%ie12:min intervali%ie", seed_count, down_count, peer_count, erval, erval / 2);
+
+  if (amount_v4) {
+    r += sprintf(r, PEERS_BENCODED4 "%zd:", OT_PEER_COMPARE_SIZE4 * amount_v4);
+    if (amount_v4 == torrent->peer_list4->peer_count)
+      r += return_peers_all(torrent->peer_list4, OT_PEER_SIZE4, r);
+    else
+      r += return_peers_selection(ws, torrent->peer_list4, OT_PEER_SIZE4, amount_v4, r);
+  }
+
+  if (amount_v6) {
+    r += sprintf(r, PEERS_BENCODED6 "%zd:", OT_PEER_COMPARE_SIZE6 * amount_v6);
+    if (amount_v6 == torrent->peer_list6->peer_count)
+      r += return_peers_all(torrent->peer_list6, OT_PEER_SIZE6, r);
+    else
+      r += return_peers_selection(ws, torrent->peer_list6, OT_PEER_SIZE6, amount_v6, r);
+  }
+
+  *r++ = 'e';
+
+  return r - reply;
+}
+
+/* Compiles a list of random peers for a torrent
+ * Reply must have enough space to hold:
+ * 92 + 6 * amount bytes for TCP/IPv4
+ * 92 + 18 * amount bytes for TCP/IPv6
+ * 12 + 6 * amount bytes for UDP/IPv4
+ * 12 + 18 * amount bytes for UDP/IPv6
+ * Does not yet check not to return self
+ */
+size_t return_peers_for_torrent(struct ot_workstruct *ws, ot_torrent *torrent, size_t amount, char *reply, PROTO_FLAG proto) {
+  return proto == FLAG_TCP ? return_peers_for_torrent_tcp(ws, torrent, amount, reply) : return_peers_for_torrent_udp(ws, torrent, amount, reply);
+}
+
 /* Fetches scrape info for a specific torrent */
-size_t return_udp_scrape_for_torrent( ot_hash *hash, char *reply ) {
-  int          exactmatch;
-  ot_vector   *torrents_list = mutex_bucket_lock_by_hash( hash );
-  ot_torrent  *torrent = binary_search( hash, torrents_list->data, torrents_list->size, sizeof( ot_torrent ), OT_HASH_COMPARE_SIZE, &exactmatch );
+size_t return_udp_scrape_for_torrent(ot_hash const hash, char *reply) {
+  int         exactmatch, delta_torrentcount = 0;
+  ot_vector  *torrents_list = mutex_bucket_lock_by_hash(hash);
+  ot_torrent *torrent       = binary_search(hash, torrents_list->data, torrents_list->size, sizeof(ot_torrent), OT_HASH_COMPARE_SIZE, &exactmatch);
 
-  if( !exactmatch ) {
-    memset( reply, 0, 12);
+  if (!exactmatch) {
+    memset(reply, 0, 12);
   } else {
-    uint32_t *r = (uint32_t*) reply;
+    uint32_t *r = (uint32_t *)reply;
 
-    if( clean_single_torrent( torrent ) ) {
-      vector_remove_torrent( torrents_list, torrent );
-      memset( reply, 0, 12);
+    if (clean_single_torrent(torrent)) {
+      vector_remove_torrent(torrents_list, torrent);
+      memset(reply, 0, 12);
+      delta_torrentcount = -1;
     } else {
-      r[0] = htonl( torrent->peer_list->seed_count );
-      r[1] = htonl( torrent->peer_list->down_count );
-      r[2] = htonl( torrent->peer_list->peer_count-torrent->peer_list->seed_count );
+      r[0] = htonl(torrent->peer_list6->seed_count + torrent->peer_list4->seed_count);
+      r[1] = htonl(torrent->peer_list6->down_count + torrent->peer_list4->down_count);
+      r[2] = htonl(torrent->peer_list6->peer_count + torrent->peer_list4->peer_count - torrent->peer_list6->seed_count - torrent->peer_list4->seed_count);
     }
   }
-  mutex_bucket_unlock_by_hash( hash );
+  mutex_bucket_unlock_by_hash(hash, delta_torrentcount);
   return 12;
 }
 
 /* Fetches scrape info for a specific torrent */
-size_t return_tcp_scrape_for_torrent( ot_hash *hash_list, int amount, char *reply ) {
-  char        *r = reply;
-  int          exactmatch, i;
+size_t return_tcp_scrape_for_torrent(ot_hash const *hash_list, int amount, char *reply) {
+  char *r = reply;
+  int   exactmatch, i;
 
-  r += sprintf( r, "d5:filesd" );
+  r += sprintf(r, "d5:filesd");
 
-  for( i=0; i<amount; ++i ) {
-    ot_hash     *hash = hash_list + i;
-    ot_vector   *torrents_list = mutex_bucket_lock_by_hash( hash );
-    ot_torrent  *torrent = binary_search( hash, torrents_list->data, torrents_list->size, sizeof( ot_torrent ), OT_HASH_COMPARE_SIZE, &exactmatch );
+  for (i = 0; i < amount; ++i) {
+    int            delta_torrentcount = 0;
+    ot_hash const *hash               = hash_list + i;
+    ot_vector     *torrents_list      = mutex_bucket_lock_by_hash(*hash);
+    ot_torrent    *torrent            = binary_search(hash, torrents_list->data, torrents_list->size, sizeof(ot_torrent), OT_HASH_COMPARE_SIZE, &exactmatch);
 
-    if( exactmatch ) {
-      if( clean_single_torrent( torrent ) ) {
-        vector_remove_torrent( torrents_list, torrent );
+    if (exactmatch) {
+      if (clean_single_torrent(torrent)) {
+        vector_remove_torrent(torrents_list, torrent);
+        delta_torrentcount = -1;
       } else {
-        memmove( r, "20:", 3 ); memmove( r+3, hash, 20 );
-        r += sprintf( r+23, "d8:completei%zde10:downloadedi%zde10:incompletei%zdee",
-          torrent->peer_list->seed_count, torrent->peer_list->down_count, torrent->peer_list->peer_count-torrent->peer_list->seed_count ) + 23;
+        *r++ = '2';
+        *r++ = '0';
+        *r++ = ':';
+        memcpy(r, hash, sizeof(ot_hash));
+        r += sizeof(ot_hash);
+        r += sprintf(r, "d8:completei%zde10:downloadedi%zde10:incompletei%zdee", torrent->peer_list6->seed_count + torrent->peer_list4->seed_count,
+                     torrent->peer_list6->down_count + torrent->peer_list4->down_count,
+                     torrent->peer_list6->peer_count + torrent->peer_list4->peer_count - torrent->peer_list6->seed_count - torrent->peer_list4->seed_count);
       }
     }
-    mutex_bucket_unlock_by_hash( hash );
+    mutex_bucket_unlock_by_hash(*hash, delta_torrentcount);
   }
 
-  *r++ = 'e'; *r++ = 'e';
+  *r++ = 'e';
+  *r++ = 'e';
   return r - reply;
 }
 
-size_t remove_peer_from_torrent( ot_hash *hash, ot_peer *peer, char *reply, PROTO_FLAG proto ) {
-  int          exactmatch;
-  size_t       index;
-  ot_vector   *torrents_list = mutex_bucket_lock_by_hash( hash );
-  ot_torrent  *torrent = binary_search( hash, torrents_list->data, torrents_list->size, sizeof( ot_torrent ), OT_HASH_COMPARE_SIZE, &exactmatch );
-  ot_peerlist *peer_list;
+static ot_peerlist dummy_list;
+size_t remove_peer_from_torrent(PROTO_FLAG proto, struct ot_workstruct *ws) {
+  int            exactmatch;
+  ot_vector     *torrents_list = mutex_bucket_lock_by_hash(*ws->hash);
+  ot_torrent    *torrent = binary_search(ws->hash, torrents_list->data, torrents_list->size, sizeof(ot_torrent), OT_HASH_COMPARE_SIZE, &exactmatch);
+  ot_peerlist   *peer_list = &dummy_list;
+  size_t         peer_size; /* initialized in next line */
+  ot_peer const *peer_src   = peer_from_peer6(&ws->peer, &peer_size);
+  size_t         peer_count = 0, seed_count = 0;
 
 #ifdef WANT_SYNC_LIVE
-  if( proto != FLAG_MCA )
-    livesync_tell( hash, peer, PEER_FLAG_STOPPED );
+  if (proto != FLAG_MCA) {
+    OT_PEERFLAG(ws->peer) |= PEER_FLAG_STOPPED;
+    livesync_tell(ws);
+  }
 #endif
 
-  if( !exactmatch ) {
-    mutex_bucket_unlock_by_hash( hash );
-
-    if( proto == FLAG_TCP )
-      return sprintf( reply, "d8:completei0e10:incompletei0e8:intervali%ie5:peers0:e", OT_CLIENT_REQUEST_INTERVAL_RANDOM );
-
-    /* Create fake packet to satisfy parser on the other end */
-    if( proto == FLAG_UDP ) {
-      ((uint32_t*)reply)[2] = htonl( OT_CLIENT_REQUEST_INTERVAL_RANDOM );
-      ((uint32_t*)reply)[3] = ((uint32_t*)reply)[4] = 0;
-      return (size_t)20;
+  if (exactmatch) {
+    peer_list = peer_size == OT_PEER_SIZE6 ? torrent->peer_list6 : torrent->peer_list4;
+    switch (vector_remove_peer(&peer_list->peers, peer_src, peer_size)) {
+    case 2:
+      peer_list->seed_count--; /* Intentional fallthrough */
+    case 1:
+      peer_list->peer_count--; /* Intentional fallthrough */
+    default:
+      break;
     }
 
-    if( proto == FLAG_MCA )
-      return 0;
+    peer_count = torrent->peer_list6->peer_count + torrent->peer_list4->peer_count;
+    seed_count = torrent->peer_list6->seed_count + torrent->peer_list4->seed_count;
   }
 
-  peer_list = torrent->peer_list;
-  for( index = 0; index<OT_POOLS_COUNT; ++index ) {
-    switch( vector_remove_peer( &peer_list->peers[index], peer, index == 0 ) ) {
-      case 0: continue;
-      case 2: peer_list->seed_counts[index]--;
-              peer_list->seed_count--;
-      case 1: default:
-              peer_list->peer_count--;
-              goto exit_loop;
-    }
-  }
-
-exit_loop:
-
-  if( proto == FLAG_TCP ) {
-    size_t reply_size = sprintf( reply, "d8:completei%zde10:incompletei%zde8:intervali%ie5:peers0:e", peer_list->seed_count, peer_list->peer_count - peer_list->seed_count, OT_CLIENT_REQUEST_INTERVAL_RANDOM );
-    mutex_bucket_unlock_by_hash( hash );
-    return reply_size;
+  if (proto == FLAG_TCP) {
+    int erval      = OT_CLIENT_REQUEST_INTERVAL_RANDOM;
+    ws->reply_size = sprintf(ws->reply, "d8:completei%zde10:incompletei%zde8:intervali%ie12:min intervali%ie%s0:e", seed_count, peer_count - seed_count, erval,
+                             erval / 2, peer_size == OT_PEER_SIZE6 ? PEERS_BENCODED6 : PEERS_BENCODED4);
   }
 
   /* Handle UDP reply */
-  if( proto == FLAG_UDP ) {
-    ((uint32_t*)reply)[2] = htonl( OT_CLIENT_REQUEST_INTERVAL_RANDOM );
-    ((uint32_t*)reply)[3] = htonl( peer_list->peer_count - peer_list->seed_count );
-    ((uint32_t*)reply)[4] = htonl( peer_list->seed_count);
+  if (proto == FLAG_UDP) {
+    ((uint32_t *)ws->reply)[2] = htonl(OT_CLIENT_REQUEST_INTERVAL_RANDOM);
+    ((uint32_t *)ws->reply)[3] = htonl(peer_count - seed_count);
+    ((uint32_t *)ws->reply)[4] = htonl(seed_count);
+    ws->reply_size             = 20;
   }
 
-  mutex_bucket_unlock_by_hash( hash );
-  return (size_t)20;
+  mutex_bucket_unlock_by_hash(*ws->hash, 0);
+  return ws->reply_size;
 }
 
-void exerr( char * message ) {
-  fprintf( stderr, "%s\n", message );
-  exit( 111 );
-}
-
-int trackerlogic_init( const char * const serverdir ) {
-  if( serverdir && chdir( serverdir ) ) {
-    fprintf( stderr, "Could not chdir() to %s, because %s\n", serverdir, strerror(errno) );
-    return -1;
-  }
-
-  srandom( time(NULL) );
-  g_tracker_id = random();
-
-  /* Initialise background worker threads */
-  mutex_init( );
-  clean_init( );
-  fullscrape_init( );
-  accesslist_init( );
-  livesync_init( );
-  sync_init( );
-  stats_init( );
-
-  return 0;
-}
-
-void trackerlogic_deinit( void ) {
-  int bucket;
+void iterate_all_torrents(int (*for_each)(ot_torrent *torrent, uintptr_t data), uintptr_t data) {
+  int    bucket;
   size_t j;
 
-  /* Deinitialise background worker threads */
-  stats_deinit( );
-  sync_deinit( );
-  livesync_init( );
-  accesslist_init( );
-  fullscrape_deinit( );
-  clean_deinit( );
+  for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) {
+    ot_vector  *torrents_list = mutex_bucket_lock(bucket);
+    ot_torrent *torrents      = (ot_torrent *)(torrents_list->data);
 
-  /* Free all torrents... */
-  for(bucket=0; bucket<OT_BUCKET_COUNT; ++bucket ) {
-    ot_vector *torrents_list = mutex_bucket_lock( bucket );
-    if( torrents_list->size ) {
-      for( j=0; j<torrents_list->size; ++j ) {
-        ot_torrent *torrent = ((ot_torrent*)(torrents_list->data)) + j;
-        free_peerlist( torrent->peer_list );
-      }
-      free( torrents_list->data );
-    }
-    mutex_bucket_unlock( bucket );
+    for (j = 0; j < torrents_list->size; ++j)
+      if (for_each(torrents + j, data))
+        break;
+
+    mutex_bucket_unlock(bucket, 0);
+    if (!g_opentracker_running)
+      return;
   }
-
-  /* Release mutexes */
-  mutex_deinit( );
 }
 
-const char *g_version_trackerlogic_c = "$Source$: $Revision$\n";
+ot_peer *peer_from_peer6(ot_peer6 *peer, size_t *peer_size) {
+  ot_ip6 *ip = (ot_ip6 *)peer;
+  if (!ip6_isv4mapped(ip)) {
+    *peer_size = OT_PEER_SIZE6;
+    return (ot_peer *)peer;
+  }
+  *peer_size = OT_PEER_SIZE4;
+  return (ot_peer *)(((uint8_t *)peer) + 12);
+}
+
+size_t peer_size_from_peer6(ot_peer6 *peer) {
+  ot_ip6 *ip = (ot_ip6 *)peer;
+  if (!ip6_isv4mapped(ip))
+    return OT_PEER_SIZE6;
+  return OT_PEER_SIZE4;
+}
+
+#ifdef _DEBUG_RANDOMTORRENTS
+void trackerlogic_add_random_torrents(size_t amount) {
+  struct ot_workstruct ws;
+  memset(&ws, 0, sizeof(ws));
+
+  ws.inbuf  = malloc(G_INBUF_SIZE);
+  ws.outbuf = malloc(G_OUTBUF_SIZE);
+  ws.reply  = ws.outbuf;
+  ws.hash   = (ot_hash *)ws.inbuf;
+
+  while (amount--) {
+    arc4random_buf(ws.hash, sizeof(ot_hash));
+    arc4random_buf(&ws.peer, sizeof(ws.peer));
+
+    OT_PEERFLAG(ws.peer) &= PEER_FLAG_SEEDING | PEER_FLAG_COMPLETED | PEER_FLAG_STOPPED;
+
+    add_peer_to_torrent_and_return_peers(FLAG_TCP, &ws, 1);
+  }
+
+  free(ws.inbuf);
+  free(ws.outbuf);
+}
+#endif
+
+void exerr(char *message) {
+  fprintf(stderr, "%s\n", message);
+  exit(111);
+}
+
+void trackerlogic_init() {
+  g_tracker_id = random();
+
+  if (!g_stats_path)
+    g_stats_path = "stats";
+  g_stats_path_len = strlen(g_stats_path);
+
+  /* Initialise background worker threads */
+  mutex_init();
+  clean_init();
+  fullscrape_init();
+  accesslist_init();
+  livesync_init();
+  stats_init();
+}
+
+void trackerlogic_deinit(void) {
+  int    bucket, delta_torrentcount = 0;
+  size_t j;
+
+  /* Free all torrents... */
+  for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) {
+    ot_vector *torrents_list = mutex_bucket_lock(bucket);
+    if (torrents_list->size) {
+      for (j = 0; j < torrents_list->size; ++j) {
+        ot_torrent *torrent = ((ot_torrent *)(torrents_list->data)) + j;
+        free_peerlist(torrent->peer_list6);
+        free_peerlist(torrent->peer_list4);
+        delta_torrentcount -= 1;
+      }
+      free(torrents_list->data);
+    }
+    mutex_bucket_unlock(bucket, delta_torrentcount);
+  }
+
+  /* Deinitialise background worker threads */
+  stats_deinit();
+  livesync_deinit();
+  accesslist_deinit();
+  fullscrape_deinit();
+  clean_deinit();
+  /* Release mutexes */
+  mutex_deinit();
+}

trackerlogic.h

@@ -3,110 +3,208 @@
 
    $id$ */
 
-#ifndef __OT_TRACKERLOGIC_H__
-#define __OT_TRACKERLOGIC_H__
+#ifndef OT_TRACKERLOGIC_H__
+#define OT_TRACKERLOGIC_H__
 
-#include <sys/types.h>
-#include <sys/time.h>
-#include <time.h>
 #include <stdint.h>
+#include <stdlib.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <time.h>
 
-typedef uint8_t        ot_hash[20];
-typedef time_t         ot_time;
+#if defined(__linux__) && defined(WANT_ARC4RANDOM)
+#include <bsd/stdlib.h>
+#endif
+#ifdef __FreeBSD__
+#define WANT_ARC4RANDOM
+#endif
+
+typedef uint8_t ot_hash[20];
+typedef time_t  ot_time;
+typedef char    ot_ip6[16];
+typedef struct {
+  ot_ip6 address;
+  int    bits;
+} ot_net;
+/* List of peers should fit in a single UDP packet (around 1200 bytes) */
+#define OT_MAX_PEERS_UDP6               66
+#define OT_MAX_PEERS_UDP4               200
+
+#define OT_IP_SIZE6                     16
+#define OT_IP_SIZE4                     4
+#define OT_PORT_SIZE                    2
+#define OT_FLAG_SIZE                    1
+#define OT_TIME_SIZE                    1
 
 /* Some tracker behaviour tunable */
-#define OT_CLIENT_TIMEOUT 30
+#define OT_CLIENT_TIMEOUT               30
 #define OT_CLIENT_TIMEOUT_CHECKINTERVAL 10
-#define OT_CLIENT_TIMEOUT_SEND (60*15)
-#define OT_CLIENT_REQUEST_INTERVAL (60*30)
-#define OT_CLIENT_REQUEST_VARIATION (60*6)
+#define OT_CLIENT_TIMEOUT_SEND          (60 * 15)
+#define OT_CLIENT_REQUEST_INTERVAL      (60 * 30)
+#define OT_CLIENT_REQUEST_VARIATION     (60 * 6)
 
-#define OT_TORRENT_TIMEOUT_HOURS 24
-#define OT_TORRENT_TIMEOUT ((60*60*OT_TORRENT_TIMEOUT_HOURS)/OT_POOLS_TIMEOUT)
+#define OT_TORRENT_TIMEOUT_HOURS        24
+#define OT_TORRENT_TIMEOUT              (60 * OT_TORRENT_TIMEOUT_HOURS)
 
-#define OT_CLIENT_REQUEST_INTERVAL_RANDOM ( OT_CLIENT_REQUEST_INTERVAL - OT_CLIENT_REQUEST_VARIATION/2 + (int)( random( ) % OT_CLIENT_REQUEST_VARIATION ) )
+#define OT_CLIENT_REQUEST_INTERVAL_RANDOM                                                                                                                      \
+  (OT_CLIENT_REQUEST_INTERVAL - OT_CLIENT_REQUEST_VARIATION / 2 + (int)(nrand48(ws->rand48_state) % OT_CLIENT_REQUEST_VARIATION))
 
-/* We maintain a list of 1024 pointers to sorted list of ot_torrent structs
-   Sort key is, of course, its hash */
-#define OT_BUCKET_COUNT 1024
+/* If WANT_MODEST_FULLSCRAPES is on, ip addresses may not
+   fullscrape more frequently than this amount in seconds */
+#define OT_MODEST_PEER_TIMEOUT        (60 * 5)
+
+/* If peers come back before 10 minutes, don't live sync them */
+#define OT_CLIENT_SYNC_RENEW_BOUNDARY 10
 
 /* Number of tracker admin ip addresses allowed */
-#define OT_ADMINIP_MAX 64
-#define OT_MAX_THREADS 16
+#define OT_ADMINIP_MAX                64
+#define OT_MAX_THREADS                64
 
-/* This list points to 9 pools of peers each grouped in five-minute-intervals
-   thus achieving a timeout of 2700s or 45 minutes
-   These pools are sorted by its binary content */
-#define OT_POOLS_COUNT   9
-#define OT_POOLS_TIMEOUT (60*5)
+/* Number of minutes after announce before peer is removed */
+#define OT_PEER_TIMEOUT               45
+
+/* We maintain a list of 1024 pointers to sorted list of ot_torrent structs
+ Sort key is, of course, its hash */
+#define OT_BUCKET_COUNT_BITS          10
+
+#define OT_BUCKET_COUNT               (1 << OT_BUCKET_COUNT_BITS)
+#define OT_BUCKET_COUNT_SHIFT         (32 - OT_BUCKET_COUNT_BITS)
+
+/* if _DEBUG_RANDOMTORRENTS is set, this is the amount of torrents to create
+   on startup */
+#define RANDOMTORRENTS                (1024 * 1024 * 1)
 
 /* From opentracker.c */
-extern time_t g_now;
-#define NOW              (g_now/OT_POOLS_TIMEOUT)
-extern uint32_t g_tracker_id;
-typedef enum { FLAG_TCP, FLAG_UDP, FLAG_MCA } PROTO_FLAG;
+extern time_t       g_now_seconds;
+extern volatile int g_opentracker_running;
+#define g_now_minutes (g_now_seconds / 60)
 
-typedef struct {
-  uint8_t data[8];
-} ot_peer;
+extern uint32_t g_tracker_id;
+typedef enum { FLAG_TCP, FLAG_UDP, FLAG_MCA, FLAG_SELFPIPE } PROTO_FLAG;
+
+#define OT_PEER_COMPARE_SIZE6                          ((OT_IP_SIZE6) + (OT_PORT_SIZE))
+#define OT_PEER_COMPARE_SIZE4                          ((OT_IP_SIZE4) + (OT_PORT_SIZE))
+#define OT_PEER_COMPARE_SIZE_FROM_PEER_SIZE(PEER_SIZE) ((PEER_SIZE) - (OT_TIME_SIZE) - (OT_FLAG_SIZE))
+
+#define OT_PEER_SIZE6                                  ((OT_TIME_SIZE) + (OT_FLAG_SIZE) + (OT_PEER_COMPARE_SIZE6))
+#define OT_PEER_SIZE4                                  ((OT_TIME_SIZE) + (OT_FLAG_SIZE) + (OT_PEER_COMPARE_SIZE4))
+
+typedef uint8_t      ot_peer; /* Generic pointer to a v6 or v4 peer */
+typedef uint8_t      ot_peer6[OT_PEER_SIZE6];
+typedef uint8_t      ot_peer4[OT_PEER_SIZE4];
 static const uint8_t PEER_FLAG_SEEDING   = 0x80;
 static const uint8_t PEER_FLAG_COMPLETED = 0x40;
 static const uint8_t PEER_FLAG_STOPPED   = 0x20;
+static const uint8_t PEER_FLAG_FROM_SYNC = 0x10;
 static const uint8_t PEER_FLAG_LEECHING  = 0x00;
 
-#define OT_SETIP( peer, ip ) memmove((peer),(ip),4);
-#define OT_SETPORT( peer, port ) memmove(((uint8_t*)peer)+4,(port),2);
-#define OT_FLAG(peer) (((uint8_t*)(peer))[6])
+/* Takes an ot_peer6 and returns the proper pointer to the peer and sets peer_size */
+ot_peer             *peer_from_peer6(ot_peer6 *peer, size_t *peer_size);
+size_t               peer_size_from_peer6(ot_peer6 *peer);
 
-#define OT_PEER_COMPARE_SIZE ((size_t)6)
-#define OT_HASH_COMPARE_SIZE (sizeof(ot_hash))
+/* New style */
+#define OT_SETIP(peer, ip)            memcpy((uint8_t *)(peer), (ip), OT_IP_SIZE6)
+#define OT_SETPORT(peer, port)        memcpy(((uint8_t *)(peer)) + (OT_IP_SIZE6), (port), 2)
+#define OT_PEERFLAG(peer)             (((uint8_t *)(peer))[(OT_IP_SIZE6) + 2])
+#define OT_PEERFLAG_D(peer, peersize) (((uint8_t *)(peer))[(peersize) - 2])
+#define OT_PEERTIME(peer, peersize)   (((uint8_t *)(peer))[(peersize) - 1])
+
+#define PEERS_BENCODED6               "6:peers6"
+#define PEERS_BENCODED4               "5:peers"
+
+#define OT_HASH_COMPARE_SIZE          (sizeof(ot_hash))
 
 struct ot_peerlist;
 typedef struct ot_peerlist ot_peerlist;
 typedef struct {
   ot_hash      hash;
-  ot_peerlist *peer_list;
+  ot_peerlist *peer_list6;
+  ot_peerlist *peer_list4;
 } ot_torrent;
 
 #include "ot_vector.h"
 
 struct ot_peerlist {
-  ot_time        base;
-  size_t         seed_count;
-  size_t         peer_count;
-  size_t         down_count;
-  size_t         seed_counts[ OT_POOLS_COUNT ];
-  ot_vector      peers[ OT_POOLS_COUNT ];
-#ifdef WANT_SYNC_BATCH
-  ot_vector      changeset;
+  ot_time   base;
+  size_t    seed_count;
+  size_t    peer_count;
+  size_t    down_count;
+  /* normal peers vector or
+     pointer to ot_vector[32] buckets if data != NULL and space == 0
+  */
+  ot_vector peers;
+};
+#define OT_PEERLIST_HASBUCKETS(peer_list) ((peer_list)->peers.size > (peer_list)->peers.space)
+
+struct ot_workstruct {
+  /* Thread specific, static */
+  char *inbuf;
+#define G_INBUF_SIZE 8192
+  char *outbuf;
+#define G_OUTBUF_SIZE 8192
+#ifdef _DEBUG_HTTPERROR
+  char *debugbuf;
+#define G_DEBUGBUF_SIZE 8192
 #endif
+
+  /* The peer currently in the working */
+  ot_peer6 peer; /* Can fit v6 and v4 peers */
+
+  /* Pointers into the request buffer */
+  ot_hash *hash;
+  char    *peer_id;
+
+  /* HTTP specific, non static */
+  char    *request;
+  ssize_t  request_size;
+  ssize_t  header_size;
+  char    *reply;
+  ssize_t  reply_size;
+
+  /* Entropy state for rand48 function so that threads don't need to acquire mutexes for
+     global random() or arc4random() state, which causes heavy load on linuxes */
+  uint16_t rand48_state[3];
+
+  int      keep_alive;
 };
 
 /*
    Exported functions
 */
 
-#if defined( WANT_SYNC_BATCH ) || defined( WANT_SYNC_LIVE )
+#ifdef WANT_SYNC_LIVE
 #define WANT_SYNC
 #endif
 
 #ifdef WANT_SYNC
-#define WANT_SYNC_PARAM( param ) , param
+#define WANT_SYNC_PARAM(param) , param
 #else
-#define WANT_SYNC_PARAM( param )
+#define WANT_SYNC_PARAM(param)
 #endif
 
-int  trackerlogic_init( const char * const serverdir );
-void trackerlogic_deinit( void );
-void exerr( char * message );
+#ifdef WANT_LOG_NETWORKS
+#error Live logging networks disabled at the moment.
+#endif
 
-ot_torrent *add_peer_to_torrent( ot_hash *hash, ot_peer *peer  WANT_SYNC_PARAM( int from_changeset ) );
-size_t remove_peer_from_torrent( ot_hash *hash, ot_peer *peer, char *reply, PROTO_FLAG proto );
-size_t return_peers_for_torrent( ot_hash *hash, size_t amount, char *reply, PROTO_FLAG proto );
-size_t return_tcp_scrape_for_torrent( ot_hash *hash, int amount, char *reply );
-size_t return_udp_scrape_for_torrent( ot_hash *hash, char *reply );
+void   trackerlogic_init(void);
+void   trackerlogic_deinit(void);
+void   exerr(char *message);
+
+/* add_peer_to_torrent does only release the torrent bucket if from_sync is set,
+   otherwise it is released in return_peers_for_torrent */
+size_t add_peer_to_torrent_and_return_peers(PROTO_FLAG proto, struct ot_workstruct *ws, size_t amount);
+size_t remove_peer_from_torrent(PROTO_FLAG proto, struct ot_workstruct *ws);
+size_t return_tcp_scrape_for_torrent(ot_hash const *hash_list, int amount, char *reply);
+size_t return_udp_scrape_for_torrent(ot_hash const hash, char *reply);
+void   add_torrent_from_saved_state(ot_hash const hash, ot_time base, size_t down_count);
+#ifdef _DEBUG_RANDOMTORRENTS
+void trackerlogic_add_random_torrents(size_t amount);
+#endif
+
+/* torrent iterator */
+void iterate_all_torrents(int (*for_each)(ot_torrent *torrent, uintptr_t data), uintptr_t data);
 
 /* Helper, before it moves to its own object */
-void free_peerlist( ot_peerlist *peer_list );
+void free_peerlist(ot_peerlist *peer_list);
 
 #endif