commit 51a98929148e9ca08b33735fb0542759380c57a9 Author: Nick Mathewson <nickm@torproject.org> Date: Mon Nov 4 10:19:41 2019 -0500 doxygen: Take "lib" descriptions from doc/HACKING/design. This commit takes descriptions for src/lib and moves them into our doxygen hierarchy. I've covered everything from lib/cc through lib/sandbox here. --- src/lib/arch/lib_arch.dox | 2 +- src/lib/cc/lib_cc.dox | 2 +- src/lib/conf/lib_conf.dox | 3 +- src/lib/container/lib_container.dox | 3 +- src/lib/ctime/lib_ctime.dox | 14 ++- src/lib/defs/lib_defs.dox | 2 +- src/lib/dispatch/lib_dispatch.dox | 14 ++- src/lib/encoding/lib_encoding.dox | 6 +- src/lib/err/lib_err.dox | 13 ++- src/lib/fdio/lib_fdio.dox | 5 +- src/lib/intmath/lib_intmath.dox | 2 +- src/lib/lib.dox | 149 +++++++++++++++++++++++++- src/lib/lock/lib_lock.dox | 6 +- src/lib/log/lib_log.dox | 10 +- src/lib/malloc/lib_malloc.dox | 3 +- src/lib/memarea/lib_memarea.dox | 8 +- src/lib/osinfo/lib_osinfo.dox | 8 +- src/lib/sandbox/lib_sandbox.dox | 15 ++- src/lib/smartlist_core/lib_smartlist_core.dox | 10 +- src/lib/string/lib_string.dox | 13 ++- src/lib/subsys/lib_subsys.dox | 32 +++++- src/lib/testsupport/lib_testsupport.dox | 2 +- src/lib/trace/lib_trace.dox | 6 +- src/lib/version/lib_version.dox | 2 +- src/lib/wallclock/lib_wallclock.dox | 10 +- 25 files changed, 313 insertions(+), 27 deletions(-) diff --git a/src/lib/arch/lib_arch.dox b/src/lib/arch/lib_arch.dox index 60b5fafeb..f7cfdbf0e 100644 --- a/src/lib/arch/lib_arch.dox +++ b/src/lib/arch/lib_arch.dox @@ -1,4 +1,4 @@ /** @dir lib/arch -@brief lib/arch +@brief lib/arch: Compatibility code for handling different CPU architectures. **/ diff --git a/src/lib/cc/lib_cc.dox b/src/lib/cc/lib_cc.dox index 804260cb2..880cfc44f 100644 --- a/src/lib/cc/lib_cc.dox +++ b/src/lib/cc/lib_cc.dox @@ -1,4 +1,4 @@ /** @dir lib/cc -@brief lib/cc +@brief lib/cc: Macros for managing the C compiler and language. **/ diff --git a/src/lib/conf/lib_conf.dox b/src/lib/conf/lib_conf.dox index 40a1d9f90..2ad051dcf 100644 --- a/src/lib/conf/lib_conf.dox +++ b/src/lib/conf/lib_conf.dox @@ -1,4 +1,5 @@ /** @dir lib/conf -@brief lib/conf +@brief lib/conf: Types and macros for declaring configuration options. + **/ diff --git a/src/lib/container/lib_container.dox b/src/lib/container/lib_container.dox index 6ee719f47..9599cce6e 100644 --- a/src/lib/container/lib_container.dox +++ b/src/lib/container/lib_container.dox @@ -1,4 +1,5 @@ /** @dir lib/container -@brief lib/container +@brief lib/container: Hash tables, dynamic arrays, bit arrays, etc. + **/ diff --git a/src/lib/ctime/lib_ctime.dox b/src/lib/ctime/lib_ctime.dox index 476c95991..bf95b0316 100644 --- a/src/lib/ctime/lib_ctime.dox +++ b/src/lib/ctime/lib_ctime.dox @@ -1,4 +1,16 @@ /** @dir lib/ctime -@brief lib/ctime +@brief lib/ctime: Constant-time code to avoid side-channels. + +This module contains constant-time implementations of various +data comparison and table lookup functions. We use these in preference to +memcmp() and so forth, since memcmp() can leak information about its inputs +based on how fast it returns. In general, your code should call tor_memeq() +and tor_memneq(), not memcmp(). + +We also define some _non_-constant-time wrappers for memcmp() here: Since we +consider calls to memcmp() to be in error, we require that code that actually +doesn't need to be constant-time to use the fast_memeq() / fast_memneq() / +fast_memcmp() aliases instead. + **/ diff --git a/src/lib/defs/lib_defs.dox b/src/lib/defs/lib_defs.dox index 5adb527fc..cd39414bf 100644 --- a/src/lib/defs/lib_defs.dox +++ b/src/lib/defs/lib_defs.dox @@ -1,4 +1,4 @@ /** @dir lib/defs -@brief lib/defs +@brief lib/defs: Lowest-level constants, used in many places. **/ diff --git a/src/lib/dispatch/lib_dispatch.dox b/src/lib/dispatch/lib_dispatch.dox index f194eff48..9f3a1fc7d 100644 --- a/src/lib/dispatch/lib_dispatch.dox +++ b/src/lib/dispatch/lib_dispatch.dox @@ -1,4 +1,16 @@ /** @dir lib/dispatch -@brief lib/dispatch +@brief lib/dispatch: In-process message delivery. + +This module provides a general in-process "message dispatch" system in which +typed messages are sent on channels. The dispatch.h header has far more +information. + +It is used by by \ref src/lib/pubsub "lib/pubsub" to implement our general +inter-module publish/subscribe system. + +This is not a fancy multi-threaded many-to-many dispatcher as you may be used +to from more sophisticated architectures: this dispatcher is intended only +for use in improving Tor's architecture. + **/ diff --git a/src/lib/encoding/lib_encoding.dox b/src/lib/encoding/lib_encoding.dox index 4a5fad927..19aca645f 100644 --- a/src/lib/encoding/lib_encoding.dox +++ b/src/lib/encoding/lib_encoding.dox @@ -1,4 +1,8 @@ /** @dir lib/encoding -@brief lib/encoding +@brief lib/encoding: Encoding data in various forms, types, and transformations + +Here we have time formats (timefmt.c), quoted strings (qstring.c), C strings +(string.c) base-16/32/64 (binascii.c), and more. + **/ diff --git a/src/lib/err/lib_err.dox b/src/lib/err/lib_err.dox index 8994fa5fd..23f9d9d3a 100644 --- a/src/lib/err/lib_err.dox +++ b/src/lib/err/lib_err.dox @@ -1,4 +1,15 @@ /** @dir lib/err -@brief lib/err +@brief lib/err: Lowest-level error handling code. + +This module is responsible for generating stack traces, handling raw +assertion failures, and otherwise reporting problems that might not be +safe to report via the regular logging module. + +There are three kinds of users for the functions in this module: + * Code that needs a way to assert(), but which cannot use the regular + `tor_assert()` macros in logging module. + * Code that needs signal-safe error reporting. + * Higher-level error handling code. + **/ diff --git a/src/lib/fdio/lib_fdio.dox b/src/lib/fdio/lib_fdio.dox index b868d28aa..c3ca3b9c8 100644 --- a/src/lib/fdio/lib_fdio.dox +++ b/src/lib/fdio/lib_fdio.dox @@ -1,4 +1,7 @@ /** @dir lib/fdio -@brief lib/fdio +@brief lib/fdio Code to read/write on file descriptors. + +(This module also handles sockets, on platforms where a socket is not a kind +of fd.) **/ diff --git a/src/lib/intmath/lib_intmath.dox b/src/lib/intmath/lib_intmath.dox index ce71e455d..0d5d711ee 100644 --- a/src/lib/intmath/lib_intmath.dox +++ b/src/lib/intmath/lib_intmath.dox @@ -1,4 +1,4 @@ /** @dir lib/intmath -@brief lib/intmath +@brief lib/intmath Integer mathematics. **/ diff --git a/src/lib/lib.dox b/src/lib/lib.dox index f1b2291c7..1d3bc3a3f 100644 --- a/src/lib/lib.dox +++ b/src/lib/lib.dox @@ -1,8 +1,151 @@ /** -@dir lib +@dir /lib @brief lib: low-level functionality. -The "lib" directory contains low-level functionality, most of it not -necessarily Tor-specific. +The "lib" directory contains low-level functionality. In general, this +code is not necessarily Tor-specific, but is instead possibly useful for +other applications. + +The modules in `lib` are currently well-factored: each one depends +only on lower-level modules. You can see an up-to-date list of the +modules, sorted from lowest to highest level, by running +`./scripts/maint/practracker/includes.py --toposort`. + +As of this writing, the library modules are (from lowest to highest +level): + + - \ref src/lib/cc "lib/cc" -- Macros for managing the C compiler and + language. + + - \ref src/lib/version "lib/version" -- Holds the current version of Tor. + + - \ref src/lib/testsupport "lib/testsupport" -- Helpers for making + test-only code, and test mocking support. + + - \ref src/lib/defs "lib/defs" -- Lowest-level constants. + + - \ref src/lib/subsys "lib/subsys" -- Types used for declaring a + "subsystem". (_A subsystem is a module with support for initialization, + shutdown, configuration, and so on._) + + - \ref src/lib/conf "lib/conf" -- For declaring configuration options. + + - \ref src/lib/arch "lib/arch" -- For handling differences in CPU + architecture. + + - \ref src/lib/err "lib/err" -- Lowest-level error handling code. + + - \ref src/lib/malloc "lib/malloc" -- Memory management. + management. + + - \ref src/lib/intmath "lib/intmath" -- Integer mathematics. + + - \ref src/lib/fdio "lib/fdio" -- For + reading and writing n file descriptors. + + - \ref src/lib/lock "lib/lock" -- Simple locking support. + (_Lower-level than the rest of the threading code._) + + - \ref src/lib/ctime "lib/ctime" -- Constant-time code to avoid + side-channels. + + - \ref src/lib/string "lib/string" -- Low-level string manipulation. + + - \ref src/lib/wallclock "lib/wallclock" -- + For inspecting and manipulating the current (UTC) time. + + - \ref src/lib/osinfo "lib/osinfo" -- For inspecting the OS version + and capabilities. + + - \ref src/lib/smartlist_core "lib/smartlist_core" -- The bare-bones + pieces of our dynamic array ("smartlist") implementation. + + - \ref src/lib/log "lib/log" -- Log messages to files, syslogs, etc. + + - \ref src/lib/container "lib/container" -- General purpose containers, + including dynamic arrays ("smartlists"), hashtables, bit arrays, + etc. + + - \ref src/lib/trace "lib/trace" -- A general-purpose API + function-tracing functionality Tor. (_Currently not much used._) + + - \ref src/lib/thread "lib/thread" -- Threading compatibility and utility + functionality, other than low-level locks (which are in \ref + src/lib/lock "lib/lock") and workqueue/threadpool code (which belongs + in \ref src/lib/evloop "lib/evloop"). + + - \ref src/lib/term "lib/term" -- Terminal manipulation + (like reading a password from the user). + + - \ref src/lib/memarea "lib/memarea" -- A fast + "arena" style allocator, where the data is freed all at once. + + - \ref src/lib/encoding "lib/encoding" -- Encoding + data in various formats, datatypes, and transformations. + + - \ref src/lib/dispatch "lib/dispatch" -- A general-purpose in-process + message delivery system. + + - \ref src/lib/sandbox "lib/sandbox" -- Our Linux seccomp2 sandbox + implementation. + + - \ref src/lib/pubsub "lib/pubsub" -- Code and macros to implement our + publish/subscribe message passing system. + + - \ref src/lib/fs "lib/fs" -- Utility and compatibility code for + manipulating files, filenames, directories, and so on. + + - \ref src/lib/confmgt "lib/confmgt" -- Code to parse, encode, and + manipulate our configuration files, state files, and so forth. + + - \ref src/lib/crypt_ops "lib/crypt_ops" -- Cryptographic operations. This + module contains wrappers around the cryptographic libraries that we + support, and implementations for some higher-level cryptographic + constructions that we use. + + - \ref src/lib/meminfo "lib/meminfo" -- Functions for inspecting our + memory usage, if the malloc implementation exposes that to us. + + - \ref src/lib/time "lib/time" -- Higher level time functions, including + fine-gained and monotonic timers. + + - \ref src/lib/math "lib/math" -- Floating-point mathematical utilities, + including compatibility code, and probability distributions. + + - \ref src/lib/buf "lib/buf" -- A general purpose queued buffer + implementation, similar to the BSD kernel's "mbuf" structure. + + - \ref src/lib/net "lib/net" -- Networking code, including address + manipulation, compatibility wrappers, + + - \ref src/lib/compress "lib/compress" -- A compatibility wrapper around + several compression libraries, currently including zlib, zstd, and lzma. + + - \ref src/lib/geoip "lib/geoip" -- Utilities to manage geoip (IP to + country) lookups and formats. + + - \ref src/lib/tls "lib/tls" -- Compatibility wrappers around the library + (NSS or OpenSSL, depending on configuration) that Tor uses to implement + the TLS link security protocol. + + - \ref src/lib/evloop "lib/evloop" -- Tools to manage the event loop and + related functionality, in order to implement asynchronous networking, + timers, periodic events, and other scheduling tasks. + + - \ref src/lib/process "lib/process" -- Utilities and compatibility code + to launch and manage subprocesses. + +### What belongs in lib? + +In general, if you can imagine some program wanting the functionality +you're writing, even if that program had nothing to do with Tor, your +functionality belongs in lib. + +If it falls into one of the existing "lib" categories, your +functionality belongs in lib. + +If you are using platform-specific `ifdef`s to manage compatibility +issues among platforms, you should probably consider whether you can +put your code into lib. **/ diff --git a/src/lib/lock/lib_lock.dox b/src/lib/lock/lib_lock.dox index 44693e7a6..f248e12c8 100644 --- a/src/lib/lock/lib_lock.dox +++ b/src/lib/lock/lib_lock.dox @@ -1,4 +1,8 @@ /** @dir lib/lock -@brief lib/lock +@brief lib/lock: Simple locking support. + +This module is more low-level than the rest of the threading code, since it +is needed by more intermediate-level modules. + **/ diff --git a/src/lib/log/lib_log.dox b/src/lib/log/lib_log.dox index 915d65240..5059f3ef7 100644 --- a/src/lib/log/lib_log.dox +++ b/src/lib/log/lib_log.dox @@ -1,4 +1,12 @@ /** @dir lib/log -@brief lib/log +@brief lib/log: Log messages to files, syslogs, etc. + +You can think of this as the logical "midpoint" of the +\ref src/lib "library code": much of the higher-level code is higher-level +_because_ it uses the logging module, and much of the lower-level code is +specifically written to avoid having to log, because the logging module +depends on it. + + **/ diff --git a/src/lib/malloc/lib_malloc.dox b/src/lib/malloc/lib_malloc.dox index 4923f1446..6408d7ce3 100644 --- a/src/lib/malloc/lib_malloc.dox +++ b/src/lib/malloc/lib_malloc.dox @@ -1,4 +1,5 @@ /** @dir lib/malloc -@brief lib/malloc +@brief lib/malloc: Wrappers and utilities for memory management. + **/ diff --git a/src/lib/memarea/lib_memarea.dox b/src/lib/memarea/lib_memarea.dox index dbd98de5e..f5b44f97c 100644 --- a/src/lib/memarea/lib_memarea.dox +++ b/src/lib/memarea/lib_memarea.dox @@ -1,4 +1,10 @@ /** @dir lib/memarea -@brief lib/memarea +@brief lib/memarea A fast arena-style allocator. + +This module has a fast "arena" style allocator, where memory is freed all at +once. This kind of allocation is very fast and avoids fragmentation, at the +expense of requiring all the data to be freed at the same time. We use this +for parsing and diff calculations. + **/ diff --git a/src/lib/osinfo/lib_osinfo.dox b/src/lib/osinfo/lib_osinfo.dox index 7733755f2..c78615937 100644 --- a/src/lib/osinfo/lib_osinfo.dox +++ b/src/lib/osinfo/lib_osinfo.dox @@ -1,4 +1,10 @@ /** @dir lib/osinfo -@brief lib/osinfo +@brief lib/osinfo: For inspecting the OS version and capabilities. + +In general, we use this module when we're telling the user what operating +system they are running. We shouldn't make decisions based on the output of +these checks: instead, we should have more specific checks, either at compile +time or run time, based on the observed system behavior. + **/ diff --git a/src/lib/sandbox/lib_sandbox.dox b/src/lib/sandbox/lib_sandbox.dox index eb42d9758..8b33fc255 100644 --- a/src/lib/sandbox/lib_sandbox.dox +++ b/src/lib/sandbox/lib_sandbox.dox @@ -1,4 +1,17 @@ /** @dir lib/sandbox -@brief lib/sandbox +@brief lib/sandbox Linux seccomp2-based sandbox. + +This module uses Linux's seccomp2 facility via the +[`libseccomp` library](https://github.com/seccomp/libseccomp), to restrict +the set of system calls that Tor is allowed to invoke while it is running. + +Because there are many libc versions that invoke different system calls, and +because handling strings is quite complex, this module is more complex and +less portable than it needs to be. + +A better architecture would put the responsibility for invoking tricky system +calls (like open()) in another, less restricted process, and give that +process responsibility for enforcing our sandbox rules. + **/ diff --git a/src/lib/smartlist_core/lib_smartlist_core.dox b/src/lib/smartlist_core/lib_smartlist_core.dox index 507d0fe92..68b824935 100644 --- a/src/lib/smartlist_core/lib_smartlist_core.dox +++ b/src/lib/smartlist_core/lib_smartlist_core.dox @@ -1,4 +1,12 @@ /** @dir lib/smartlist_core -@brief lib/smartlist_core +@brief lib/smartlist_core: Minimal dynamic array implementation + +A `smartlist_t` is a dynamic array type for holding `void *`. We use it +throughout the rest of the codebase. + +There are higher-level pieces in \ref src/lib/container "lib/container", but +the ones in lib/smartlist_core are used by the logging code, and therefore +cannot use the logging code. + **/ diff --git a/src/lib/string/lib_string.dox b/src/lib/string/lib_string.dox index 3e038ea07..95645ab36 100644 --- a/src/lib/string/lib_string.dox +++ b/src/lib/string/lib_string.dox @@ -1,4 +1,15 @@ /** @dir lib/string -@brief lib/string +@brief lib/string: Low-level string manipulation. + +We have a number of compatibility functions here: some are for handling +functionality that is not implemented (or not implemented the same) on every +platform; some are for providing locale-independent versions of libc +functions that would otherwise be defined differently for different users. + +Other functions here are for common string-manipulation operations that we do +in the rest of the codebase. + +Any string function high-level enough to need logging belongs in a +higher-level module. **/ diff --git a/src/lib/subsys/lib_subsys.dox b/src/lib/subsys/lib_subsys.dox index f9cd5eeb8..4d98ff577 100644 --- a/src/lib/subsys/lib_subsys.dox +++ b/src/lib/subsys/lib_subsys.dox @@ -1,4 +1,34 @@ /** @dir lib/subsys -@brief lib/subsys +@brief lib/subsys: Types for declaring a "subsystem". + +## Subsystems in Tor + +A subsystem is a module with support for initialization, shutdown, +configuration, and so on. + +Many parts of Tor can be initialized, cleaned up, and configured somewhat +independently through a table-driven mechanism. Each such part is called a +"subsystem". + +To declare a subsystem, make a global `const` instance of the `subsys_fns_t` +type, filling in the function pointer fields that you require with ones +corresponding to your subsystem. Any function pointers left as "NULL" will +be a no-op. Each system must have a name and a "level", which corresponds to +the order in which it is initialized. (See `app/main/subsystem_list.c` for a +list of current subsystems and their levels.) + +Then, insert your subsystem in the list in `app/main/subsystem_list.c`. It +will need to occupy a position corresponding to its level. + +At this point, your subsystem will be handled like the others: it will get +initialized at startup, torn down at exit, and so on. + +Historical note: Not all of Tor's code is currently handled as +subsystems. As you work with older code, you may see some parts of the code +that are initialized from `tor_init()` or `run_tor_main_loop()` or +`tor_run_main()`; and torn down from `tor_cleanup()`. We aim to migrate +these to subsystems over time; please don't add any new code that follows +this pattern. + **/ diff --git a/src/lib/testsupport/lib_testsupport.dox b/src/lib/testsupport/lib_testsupport.dox index 63ccc47d3..665b10e1f 100644 --- a/src/lib/testsupport/lib_testsupport.dox +++ b/src/lib/testsupport/lib_testsupport.dox @@ -1,4 +1,4 @@ /** @dir lib/testsupport -@brief lib/testsupport +@brief lib/testsupport: Helpers for test-only code and for function mocking. **/ diff --git a/src/lib/trace/lib_trace.dox b/src/lib/trace/lib_trace.dox index a1ae25650..a8ac09d87 100644 --- a/src/lib/trace/lib_trace.dox +++ b/src/lib/trace/lib_trace.dox @@ -1,4 +1,8 @@ /** @dir lib/trace -@brief lib/trace +@brief lib/trace: Function-tracing functionality API. + +This module is used for adding "trace" support (low-granularity function +logging) to Tor. Right now it doesn't have many users. + **/ diff --git a/src/lib/version/lib_version.dox b/src/lib/version/lib_version.dox index 213e1a1ae..d3d2481ba 100644 --- a/src/lib/version/lib_version.dox +++ b/src/lib/version/lib_version.dox @@ -1,4 +1,4 @@ /** @dir lib/version -@brief lib/version +@brief lib/version: holds the current version of Tor. **/ diff --git a/src/lib/wallclock/lib_wallclock.dox b/src/lib/wallclock/lib_wallclock.dox index 7bb2b075d..a024425ea 100644 --- a/src/lib/wallclock/lib_wallclock.dox +++ b/src/lib/wallclock/lib_wallclock.dox @@ -1,4 +1,12 @@ /** @dir lib/wallclock -@brief lib/wallclock +@brief lib/wallclock: Inspect and manipulate the current time. + +This module handles our concept of "what time is it" or "what time does the +world agree it is?" Generally, if you want something derived from UTC, this +is the module for you. + +For versions of the time that are more local, more monotonic, or more +accurate, see \ref src/lib/time "lib/time". + **/