commit 40dc25f60a6d192b29d701d10c6f970bfbe4d4eb Author: Damian Johnson atagar@torproject.org Date: Thu Mar 22 12:55:28 2018 -0700
Update manual cache
Recaching information from tor's manual. Ran into a couple interesting wrinkles while doing this...
https://trac.torproject.org/projects/tor/ticket/25581 https://trac.torproject.org/projects/tor/ticket/25582 --- stem/cached_tor_manual.sqlite | Bin 227328 -> 238592 bytes stem/manual.py | 3 ++- stem/settings.cfg | 32 ++++++++++++++++++++++++-------- test/integ/control/controller.py | 18 +++++++++++------- test/integ/manual.py | 13 ++++++++----- 5 files changed, 45 insertions(+), 21 deletions(-)
diff --git a/stem/cached_tor_manual.sqlite b/stem/cached_tor_manual.sqlite index 86050fe8..e8fe44cb 100644 Binary files a/stem/cached_tor_manual.sqlite and b/stem/cached_tor_manual.sqlite differ diff --git a/stem/manual.py b/stem/manual.py index 5e628ead..0bff9b68 100644 --- a/stem/manual.py +++ b/stem/manual.py @@ -79,7 +79,7 @@ try: except ImportError: import urllib2 as urllib
-Category = stem.util.enum.Enum('GENERAL', 'CLIENT', 'RELAY', 'DIRECTORY', 'AUTHORITY', 'HIDDEN_SERVICE', 'TESTING', 'UNKNOWN') +Category = stem.util.enum.Enum('GENERAL', 'CLIENT', 'RELAY', 'DIRECTORY', 'AUTHORITY', 'HIDDEN_SERVICE', 'DENIAL_OF_SERVICE', 'TESTING', 'UNKNOWN') GITWEB_MANUAL_URL = 'https://gitweb.torproject.org/tor.git/plain/doc/tor.1.txt' CACHE_PATH = os.path.join(os.path.dirname(__file__), 'cached_tor_manual.sqlite') DATABASE = None # cache database connections @@ -104,6 +104,7 @@ CATEGORY_SECTIONS = OrderedDict(( ('DIRECTORY SERVER OPTIONS', Category.DIRECTORY), ('DIRECTORY AUTHORITY SERVER OPTIONS', Category.AUTHORITY), ('HIDDEN SERVICE OPTIONS', Category.HIDDEN_SERVICE), + ('DENIAL OF SERVICE MITIGATION OPTIONS', Category.DENIAL_OF_SERVICE), ('TESTING NETWORK OPTIONS', Category.TESTING), ))
diff --git a/stem/settings.cfg b/stem/settings.cfg index 59d6650b..5ce1cfa0 100644 --- a/stem/settings.cfg +++ b/stem/settings.cfg @@ -73,6 +73,8 @@ manual.summary.ControlPortWriteToFile Path for a file tor writes containing its manual.summary.ControlPortFileGroupReadable Group read permissions for the control port file manual.summary.DataDirectory Location for storing runtime data (state, keys, etc) manual.summary.DataDirectoryGroupReadable Group read permissions for the data directory +manual.summary.CacheDirectory Directory where information is cached +manual.summary.CacheDirectoryGroupReadable Group read permissions for the cache directory manual.summary.FallbackDir Fallback when unable to retrieve descriptor information manual.summary.UseDefaultFallbackDirs Use hard-coded fallback directory authorities when needed manual.summary.DirAuthority Alternative directory authorities @@ -95,7 +97,7 @@ manual.summary.Socks4Proxy SOCKS 4 proxy for connecting to tor manual.summary.Socks5Proxy SOCKS 5 for connecting to tor manual.summary.Socks5ProxyUsername Username for connecting to the Socks5Proxy manual.summary.Socks5ProxyPassword Password for connecting to the Socks5Proxy -manual.summary.SocksSocketsGroupWritable Group write permissions for the socks socket +manual.summary.UnixSocksGroupWritable Group write permissions for the socks socket manual.summary.KeepalivePeriod Rate at which to send keepalive packets manual.summary.Log Runlevels and location for tor logging manual.summary.LogMessageDomains Includes a domain when logging messages @@ -109,6 +111,7 @@ manual.summary.RunAsDaemon Toggles if tor runs as a daemon process manual.summary.LogTimeGranularity limits granularity of log message timestamps manual.summary.TruncateLogFile Overwrites log file rather than appending when restarted manual.summary.SyslogIdentityTag Tag logs appended to the syslog as being from tor +manual.summary.AndroidIdentityTag Tag when logging to android subsystem manual.summary.SafeLogging Toggles if logs are scrubbed of sensitive information manual.summary.User UID for the process when started manual.summary.KeepBindCapabilities Retain permission for binding to low valued ports @@ -179,6 +182,7 @@ manual.summary.NATDPort Port for forwarding ipfw NATD connections manual.summary.AutomapHostsOnResolve Map addresses ending with special suffixes to virtual addresses manual.summary.AutomapHostsSuffixes Address suffixes recognized by AutomapHostsOnResolve manual.summary.DNSPort Port from which DNS responses are fetched instead of tor +manual.summary.ClientDNSRejectInternalAddresses Disregards anonymous DNS responses for internal addresses manual.summary.ClientRejectInternalAddresses Disables use of Tor for internal connections manual.summary.DownloadExtraInfo Toggles fetching of extra information about relays manual.summary.WarnPlaintextPorts Toggles warnings for using risky ports @@ -186,6 +190,8 @@ manual.summary.RejectPlaintextPorts Prevents connections on risky ports manual.summary.OptimisticData Use exits without confirmation that prior connections succeeded manual.summary.Tor2webMode Establish non-anonymous hidden service connections manual.summary.Tor2webRendezvousPoints Rendezvous points to use for hidden services when in Tor2webMode +manual.summary._HSLayer2Nodes # TODO: https://trac.torproject.org/projects/tor/ticket/25581 +manual.summary._HSLayer3Nodes # TODO: https://trac.torproject.org/projects/tor/ticket/25581 manual.summary.UseMicrodescriptors Retrieve microdescriptors rather than server descriptors manual.summary.PathBiasCircThreshold Number of circuits through a guard before applying bias checks manual.summary.PathBiasNoticeRate Fraction of circuits that must succeed before logging a notice @@ -205,8 +211,6 @@ manual.summary.PathsNeededToBuildCircuits Portion of relays to require informati manual.summary.ClientBootstrapConsensusAuthorityDownloadSchedule Schedule when bootstrapping for when to download resources from authorities manual.summary.ClientBootstrapConsensusFallbackDownloadSchedule Schedule when bootstrapping for when to download resources from fallback authorities manual.summary.ClientBootstrapConsensusAuthorityOnlyDownloadSchedule Schedule when bootstrapping for when to download resources from authorities when fallbacks unavailable -manual.summary.ClientBootstrapConsensusMaxDownloadTries Number of times to attempt downloading consensus -manual.summary.ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries Number of times to attempt downloading consensus from authorities manual.summary.ClientBootstrapConsensusMaxInProgressTries Number of consensus download requests to allow in-flight at once
# Server Config Options @@ -218,6 +222,7 @@ manual.summary.BridgeDistribution Distribution method BrideDB should provide our manual.summary.ContactInfo Contact information for this relay manual.summary.ExitRelay Allow relaying of exit traffic manual.summary.ExitPolicy Traffic destinations that can exit from this relay +manual.summary.ExitPolicyDefault # TODO: https://trac.torproject.org/projects/tor/ticket/25582 manual.summary.ExitPolicyRejectPrivate Prevent exiting on the local network manual.summary.ExitPolicyRejectLocalInterfaces More extensive prevention of exiting on the local network manual.summary.ReducedExitPolicy Customized reduced exit policy @@ -233,6 +238,7 @@ manual.summary.PublishServerDescriptor Types of descriptors published manual.summary.ShutdownWaitLength Delay before quitting after receiving a SIGINT signal manual.summary.SSLKeyLifetime Lifetime for our link certificate manual.summary.HeartbeatPeriod Rate at which an INFO level heartbeat message is sent +manual.summary.MainloopStats Include development information from the main loop with heartbeats manual.summary.AccountingMax Amount of traffic before hibernating manual.summary.AccountingRule Method to determine when the accounting limit is reached manual.summary.AccountingStart Duration of an accounting period @@ -260,6 +266,8 @@ manual.summary.MaxMemInQueues Threshold at which tor will terminate circuits to manual.summary.DisableOOSCheck Don't close connections when running out of sockets manual.summary.SigningKeyLifetime Duration the Ed25519 signing key is valid for manual.summary.OfflineMasterKey Don't generate the master secret key +manual.summary.KeyDirectory Directory where secret keys reside +manual.summary.KeyDirectoryGroupReadable Group read permissions for the secret key directory
# Directory Server Options
@@ -322,6 +330,19 @@ manual.summary.HiddenServiceNumIntroductionPoints Number of introduction points manual.summary.HiddenServiceSingleHopMode Allow non-anonymous single hop hidden services manual.summary.HiddenServiceNonAnonymousMode Enables HiddenServiceSingleHopMode to be set
+# DoS Mitigation Options + +manual.summary.DoSCircuitCreationEnabled Enables circuit creation DoS mitigation +manual.summary.DoSCircuitCreationMinConnections Connection rate when clients are a suspected DoS +manual.summary.DoSCircuitCreationRate Acceptable rate for circuit creation +manual.summary.DoSCircuitCreationBurst Accept burst of circuit creation up to this rate +manual.summary.DoSCircuitCreationDefenseType Method for mitigating circuit creation DoS +manual.summary.DoSCircuitCreationDefenseTimePeriod Duration of DoS mitigation +manual.summary.DoSConnectionEnabled Enables connection DoS mitigation +manual.summary.DoSConnectionMaxConcurrentCount Acceptable number of connections +manual.summary.DoSConnectionDefenseType Method for mitigating connection DoS +manual.summary.DoSRefuseSingleHopClientRendezvous Prevent establishment of single hop rendezvous points + # Testing Network Options
manual.summary.TestingTorNetwork Overrides other options to be a testing network @@ -340,10 +361,6 @@ manual.summary.TestingBridgeDownloadSchedule Schedule for when we should downloa manual.summary.TestingBridgeBootstrapDownloadSchedule Schedule for downloading bridge descriptors when started manual.summary.TestingClientMaxIntervalWithoutRequest Maximum time to wait to batch requests for missing descriptors manual.summary.TestingDirConnectionMaxStall Duration to let directory connections stall before timing out -manual.summary.TestingConsensusMaxDownloadTries Retries for downloading the consensus -manual.summary.TestingDescriptorMaxDownloadTries Retries for downloading server descriptors -manual.summary.TestingMicrodescMaxDownloadTries Retries for downloading microdescriptors -manual.summary.TestingCertMaxDownloadTries Retries for downloading authority certificates manual.summary.TestingDirAuthVoteExit Relays to give the Exit flag to manual.summary.TestingDirAuthVoteExitIsStrict Only grant the Exit flag to relays listed by TestingDirAuthVoteExit manual.summary.TestingDirAuthVoteGuard Relays to give the Guard flag to @@ -359,7 +376,6 @@ manual.summary.TestingAuthKeyLifetime Duration for our ed25519 signing key manual.summary.TestingLinkKeySlop Time before expiration that we replace our ed25519 link key manual.summary.TestingAuthKeySlop Time before expiration that we replace our ed25519 authentication key manual.summary.TestingSigningKeySlop Time before expiration that we replace our ed25519 signing key -manual.summary.TestingClientDNSRejectInternalAddresses Skips DNS resolutions of internal addresses
# Brief description of tor events
diff --git a/test/integ/control/controller.py b/test/integ/control/controller.py index 87d6e970..8042e858 100644 --- a/test/integ/control/controller.py +++ b/test/integ/control/controller.py @@ -1395,16 +1395,20 @@ class TestController(unittest.TestCase): """
with test.runner.get_runner().get_tor_controller() as controller: - self.assertEqual(None, controller.get_conf('OrPort')) + try: + controller.reset_conf('OrPort', 'DisableNetwork') + self.assertEqual(None, controller.get_conf('OrPort'))
- # DisableNetwork ensures no port is actually opened - controller.set_options({'OrPort': '9090', 'DisableNetwork': '1'}) + # DisableNetwork ensures no port is actually opened + controller.set_options({'OrPort': '9090', 'DisableNetwork': '1'})
- # TODO once tor 0.2.7.x exists, test that we can generate a descriptor on demand. + # TODO once tor 0.2.7.x exists, test that we can generate a descriptor on demand.
- self.assertEqual('9090', controller.get_conf('OrPort')) - controller.reset_conf('OrPort', 'DisableNetwork') - self.assertEqual(None, controller.get_conf('OrPort')) + self.assertEqual('9090', controller.get_conf('OrPort')) + controller.reset_conf('OrPort', 'DisableNetwork') + self.assertEqual(None, controller.get_conf('OrPort')) + finally: + controller.set_conf('OrPort', test.runner.ORPORT)
def _get_router_status_entry(self, controller): """ diff --git a/test/integ/manual.py b/test/integ/manual.py index 1eb4fb76..b08179db 100644 --- a/test/integ/manual.py +++ b/test/integ/manual.py @@ -27,6 +27,7 @@ EXPECTED_CATEGORIES = set([ 'DIRECTORY SERVER OPTIONS', 'DIRECTORY AUTHORITY SERVER OPTIONS', 'HIDDEN SERVICE OPTIONS', + 'DENIAL OF SERVICE MITIGATION OPTIONS', 'TESTING NETWORK OPTIONS', 'NON-PERSISTENT OPTIONS', 'SIGNALS', @@ -66,7 +67,11 @@ Private addresses are rejected by default (at the beginning of your exit policy)
This directive can be specified multiple times so you don't have to put it all on one line.
-Policies are considered first to last, and the first match wins. If you want to allow the same ports on IPv4 and IPv6, write your rules using accept/reject *. If you want to allow different ports on IPv4 and IPv6, write your IPv6 rules using accept6/reject6 *6, and your IPv4 rules using accept/reject *4. If you want to _replace_ the default exit policy, end your exit policy with either a reject *:* or an accept *:*. Otherwise, you're _augmenting_ (prepending to) the default exit policy. The default exit policy is: +Policies are considered first to last, and the first match wins. If you want to allow the same ports on IPv4 and IPv6, write your rules using accept/reject *. If you want to allow different ports on IPv4 and IPv6, write your IPv6 rules using accept6/reject6 *6, and your IPv4 rules using accept/reject *4. If you want to _replace_ the default exit policy, end your exit policy with either a reject *:* or an accept *:*. Otherwise, you're _augmenting_ (prepending to) the default exit policy. + +If you want to use a reduced exit policy rather than the default exit policy, set "ReducedExitPolicy 1". If you want to replace the default exit policy with your custom exit policy, end your exit policy with either a reject : or an accept :. Otherwise, you're augmenting (prepending to) the default or reduced exit policy. + +The default exit policy is:
reject *:25 reject *:119 @@ -79,8 +84,6 @@ Policies are considered first to last, and the first match wins. If you want to reject *:6699 reject *:6881-6999 accept *:* - -Since the default exit policy uses accept/reject *, it applies to both IPv4 and IPv6 addresses. """.strip()
@@ -203,7 +206,7 @@ class TestManual(unittest.TestCase): assert_equal('signals', EXPECTED_SIGNALS, set(manual.signals.keys())) assert_equal('sighup description', 'Tor will catch this, clean up and sync to disk if necessary, and exit.', manual.signals['SIGTERM'])
- assert_equal('number of files', 50, len(manual.files)) + assert_equal('number of files', 48, len(manual.files)) assert_equal('lib path description', 'The tor process stores keys and other data here.', manual.files['@LOCALSTATEDIR@/lib/tor/'])
for category in Category: @@ -213,7 +216,7 @@ class TestManual(unittest.TestCase): unknown_options = [entry for entry in manual.config_options.values() if entry.category == Category.UNKNOWN]
if unknown_options: - self.fail("We don't recognize the category for the %s options. Maybe a new man page section? If so then please update the Category enum in stem/manual.py." % ', '.join(unknown_options)) + self.fail("We don't recognize the category for the %s options. Maybe a new man page section? If so then please update the Category enum in stem/manual.py." % ', '.join([option.name for option in unknown_options]))
option = manual.config_options['BandwidthRate'] self.assertEqual(Category.GENERAL, option.category)