[tor-commits] [stem/master] Sync manual and update assertions
atagar at torproject.org
atagar at torproject.org
Fri Dec 27 23:50:23 UTC 2019
commit f34212fb0c91715d7a9f3ab3764033e3c962682b
Author: Damian Johnson <atagar at torproject.org>
Date: Wed Dec 25 16:14:51 2019 -0800
Sync manual and update assertions
The manual has changed quite a bit, requiring small parser adjustments.
---
stem/cached_manual.sqlite | Bin 249856 -> 252928 bytes
stem/manual.py | 53 ++++++++++++++++++++++++++--------------------
stem/settings.cfg | 3 +++
test/integ/manual.py | 27 +++++++++++------------
test/unit/manual.py | 2 +-
5 files changed, 46 insertions(+), 39 deletions(-)
diff --git a/stem/cached_manual.sqlite b/stem/cached_manual.sqlite
index 515e894b..a2d9b02c 100644
Binary files a/stem/cached_manual.sqlite and b/stem/cached_manual.sqlite differ
diff --git a/stem/manual.py b/stem/manual.py
index 24596851..5b502a4d 100644
--- a/stem/manual.py
+++ b/stem/manual.py
@@ -730,7 +730,9 @@ def _get_indented_descriptions(lines):
options, last_arg = OrderedDict(), None
for line in lines:
- if line and not line.startswith(' '):
+ if line == ' Note':
+ last_arg = None # manual has several indented 'Note' blocks
+ elif line and not line.startswith(' '):
options[line], last_arg = [], line
elif last_arg and line.startswith(' '):
options[last_arg].append(line[4:])
@@ -756,43 +758,48 @@ def _add_config_options(config_options, category, lines):
since that platform lacks getrlimit(). (Default: 1000)
"""
- last_option, usage, description = None, None, []
+ def add_option(title, description):
+ if 'PER INSTANCE OPTIONS' in title:
+ return # skip, unfortunately amid the options
+
+ if ', ' in title:
+ # Line actually had multiple options with the same description. For
+ # example...
+ #
+ # AlternateBridgeAuthority [nickname], AlternateDirAuthority [nickname]
- # Drop the section description. Each ends with a paragraph saying 'The
- # following options...'.
+ for subtitle in title.split(', '):
+ add_option(subtitle, description)
+ else:
+ name, usage = title.split(' ', 1) if ' ' in title else (title, '')
+ summary = _config().get('manual.summary.%s' % name.lower(), '')
+ config_options[name] = ConfigOption(name, category, usage, summary, _join_lines(description).strip())
- desc_paragraph_index = None
+ # Remove the section's description by finding the sentence the section
+ # ends with.
- for i, line in enumerate(lines):
- if 'The following options' in line:
- desc_paragraph_index = i
- break
+ end_indices = [i for (i, line) in enumerate(lines) if ('The following options' in line or 'PER SERVICE OPTIONS' in line)]
- if desc_paragraph_index is not None:
- lines = lines[desc_paragraph_index:] # trim to the description paragrah
+ if end_indices:
+ lines = lines[max(end_indices):] # trim to the description paragrah
lines = lines[lines.index(''):] # drop the paragraph
+ last_title, description = None, []
+
for line in lines:
if line and not line.startswith(' '):
- if last_option:
- summary = _config().get('manual.summary.%s' % last_option.lower(), '')
- config_options[last_option] = ConfigOption(last_option, category, usage, summary, _join_lines(description).strip())
-
- if ' ' in line:
- last_option, usage = line.split(' ', 1)
- else:
- last_option, usage = line, ''
+ if last_title:
+ add_option(last_title, description)
- description = []
+ last_title, description = line, []
else:
if line.startswith(' '):
line = line[4:]
description.append(line)
- if last_option:
- summary = _config().get('manual.summary.%s' % last_option.lower(), '')
- config_options[last_option] = ConfigOption(last_option, category, usage, summary, _join_lines(description).strip())
+ if last_title:
+ add_option(last_title, description)
def _join_lines(lines):
diff --git a/stem/settings.cfg b/stem/settings.cfg
index b7be1ee1..c8cb29c9 100644
--- a/stem/settings.cfg
+++ b/stem/settings.cfg
@@ -331,6 +331,9 @@ manual.summary.HiddenServiceMaxStreamsCloseCircuit Closes rendezvous circuits th
manual.summary.RendPostPeriod Period at which the rendezvous service descriptors are refreshed
manual.summary.HiddenServiceDirGroupReadable Group read permissions for the hidden service directory
manual.summary.HiddenServiceNumIntroductionPoints Number of introduction points the hidden service will have
+manual.summary.HiddenServiceEnableIntroDoSDefense Introduction point DoS protection
+manual.summary.HiddenServiceEnableIntroDoSRatePerSec Request rate allowed for the introduction point
+manual.summary.HiddenServiceEnableIntroDoSBurstPerSec Burst rate allowed for the introduction point
manual.summary.HiddenServiceSingleHopMode Allow non-anonymous single hop hidden services
manual.summary.HiddenServiceNonAnonymousMode Enables HiddenServiceSingleHopMode to be set
diff --git a/test/integ/manual.py b/test/integ/manual.py
index de86f695..ae0b2b16 100644
--- a/test/integ/manual.py
+++ b/test/integ/manual.py
@@ -38,20 +38,21 @@ EXPECTED_CATEGORIES = set([
'AUTHORS',
])
-EXPECTED_CLI_OPTIONS = set(['-f FILE', '--hash-password PASSWORD', '--ignore-missing-torrc', '--defaults-torrc FILE', '--key-expiration [purpose]', '--list-fingerprint', '--list-deprecated-options', '--allow-missing-torrc', '--nt-service', '--verify-config', '--service remove|start|stop', '--passphrase-fd FILEDES', '--keygen [--newpass]', '--list-torrc-options', '--service install [--options command-line options]', '--list-modules', '--quiet|--hush', '--version', '-h, --help'])
+EXPECTED_CLI_OPTIONS = set(['-f FILE', '--hash-password PASSWORD', '--ignore-missing-torrc', '--defaults-torrc FILE', '--key-expiration [purpose]', '--list-fingerprint', '--list-deprecated-options', '--allow-missing-torrc', '--nt-service', '--verify-config', '--dump-config short|full|non-builtin', '--service remove|start|stop', '--passphrase-fd FILEDES', '--keygen [--newpass]', '--list-torrc-options', '--service install [--options command-line options]', '--list-modules', '--quiet|--hush', '--version', '-h, --help'])
EXPECTED_SIGNALS = set(['SIGTERM', 'SIGINT', 'SIGHUP', 'SIGUSR1', 'SIGUSR2', 'SIGCHLD', 'SIGPIPE', 'SIGXFSZ'])
EXPECTED_DESCRIPTION = """
-Tor is a connection-oriented anonymizing communication service. Users choose a source-routed path through a set of nodes, and negotiate a "virtual circuit" through the network, in which each node knows its predecessor and successor, but no others. Traffic flowing down the circuit is unwrapped by a symmetric key at each node, which reveals the downstream node.
+Tor is a connection-oriented anonymizing communication service. Users choose a source-routed path through a set of nodes, and negotiate a "virtual circuit" through the network. Each node in a virtual circuit knows its predecessor and successor nodes, but no other nodes. Traffic flowing down the circuit is unwrapped by a symmetric key at each node, which reveals the downstream node.
-Basically, Tor provides a distributed network of servers or relays ("onion routers"). Users bounce their TCP streams -- web traffic, ftp, ssh, etc. -- around the network, and recipients, observers, and even the relays themselves have difficulty tracking the source of the stream.
+Basically, Tor provides a distributed network of servers or relays ("onion routers"). Users bounce their TCP streams, including web traffic, ftp, ssh, etc., around the network, so that recipients, observers, and even the relays themselves have difficulty tracking the source of the stream.
-By default, tor will act as a client only. To help the network by providing bandwidth as a relay, change the ORPort configuration option -- see below. Please also consult the documentation on the Tor Project's website.
+ Note
+ By default, tor acts as a client only. To help the network by providing bandwidth as a relay, change the ORPort configuration option as mentioned below. Please also consult the documentation on the Tor Project's website.
""".strip()
-EXPECTED_FILE_DESCRIPTION = 'Specify a new configuration file to contain further Tor configuration options OR pass - to make Tor read its configuration from standard input. (Default: @CONFDIR@/torrc, or $HOME/.torrc if that file is not found)'
+EXPECTED_FILE_DESCRIPTION = 'Specify a new configuration file to contain further Tor configuration options, or pass - to make Tor read its configuration from standard input. (Default: @CONFDIR@/torrc, or $HOME/.torrc if that file is not found)'
-EXPECTED_BANDWIDTH_RATE_DESCRIPTION = 'A token bucket limits the average incoming bandwidth usage on this node to the specified number of bytes per second, and the average outgoing bandwidth usage to that same value. If you want to run a relay in the public network, this needs to be at the very least 75 KBytes for a relay (that is, 600 kbits) or 50 KBytes for a bridge (400 kbits) -- but of course, more is better; we recommend at least 250 KBytes (2 mbits) if possible. (Default: 1 GByte)\n\nNote that this option, and other bandwidth-limiting options, apply to TCP data only: They do not count TCP headers or DNS traffic.\n\nWith this option, and in other options that take arguments in bytes, KBytes, and so on, other formats are also supported. Notably, "KBytes" can also be written as "kilobytes" or "kb"; "MBytes" can be written as "megabytes" or "MB"; "kbits" can be written as "kilobits"; and so forth. Tor also accepts "byte" and "bit" in the singular. The prefixes "tera" and "T" are a
lso recognized. If no units are given, we default to bytes. To avoid confusion, we recommend writing "bytes" or "bits" explicitly, since it\'s easy to forget that "B" means bytes, not bits.'
+EXPECTED_BANDWIDTH_RATE_DESCRIPTION = 'A token bucket limits the average incoming bandwidth usage on this node to the specified number of bytes per second, and the average outgoing bandwidth usage to that same value. If you want to run a relay in the public network, this needs to be at the very least 75 KBytes for a relay (that is, 600 kbits) or 50 KBytes for a bridge (400 kbits) -- but of course, more is better; we recommend at least 250 KBytes (2 mbits) if possible. (Default: 1 GByte)\n\nNote that this option, and other bandwidth-limiting options, apply to TCP data only: They do not count TCP headers or DNS traffic.\n\nTor uses powers of two, not powers of ten, so 1 GByte is 1024*1024*1024 bytes as opposed to 1 billion bytes.\n\nWith this option, and in other options that take arguments in bytes, KBytes, and so on, other formats are also supported. Notably, "KBytes" can also be written as "kilobytes" or "kb"; "MBytes" can be written as "megabytes" or "MB"; "kbits" can be written a
s "kilobits"; and so forth. Case doesn\'t matter. Tor also accepts "byte" and "bit" in the singular. The prefixes "tera" and "T" are also recognized. If no units are given, we default to bytes. To avoid confusion, we recommend writing "bytes" or "bits" explicitly, since it\'s easy to forget that "B" means bytes, not bits.'
EXPECTED_EXIT_POLICY_DESCRIPTION_START = 'Set an exit policy for this server. Each policy'
EXPECTED_EXIT_POLICY_DESCRIPTION_END = 'it applies to both IPv4 and IPv6 addresses.'
@@ -162,7 +163,11 @@ class TestManual(unittest.TestCase):
def assert_equal(category, expected, actual):
if expected != actual:
- self.fail("Changed tor's man page? The %s changed as follows...\n\nexpected: %s\n\nactual: %s" % (category, sorted(expected), sorted(actual)))
+ if isinstance(expected, (set, tuple, list)):
+ expected = sorted(expected)
+ actual = sorted(actual)
+
+ self.fail("Changed tor's man page? The %s changed as follows...\n\nexpected: %s\n\nactual: %s" % (category, expected, actual))
manual = stem.manual.Manual.from_man(self.man_path)
@@ -237,14 +242,6 @@ class TestManual(unittest.TestCase):
if name.startswith('_'):
config_options_in_tor.remove(name)
- # TODO: Looks like options we should remove from tor...
- #
- # https://trac.torproject.org/projects/tor/ticket/17665
-
- for option in ('SchedulerMaxFlushCells__', 'SchedulerLowWaterMark__', 'SchedulerHighWaterMark__'):
- if option in config_options_in_tor:
- config_options_in_tor.remove(option)
-
manual = stem.manual.Manual.from_man(self.man_path)
config_options_in_manual = set(manual.config_options.keys())
diff --git a/test/unit/manual.py b/test/unit/manual.py
index cd2fce94..72f847e5 100644
--- a/test/unit/manual.py
+++ b/test/unit/manual.py
@@ -234,7 +234,7 @@ class TestManual(unittest.TestCase):
self.assertEqual('tor - The second-generation onion router', manual.name)
self.assertEqual('tor [OPTION value]...', manual.synopsis)
- self.assertTrue(manual.description.startswith(EXPECTED_DESCRIPTION))
+ self.assertTrue(manual.description.startswith('Tor is a connection-oriented anonymizing communication service.'))
self.assertTrue(len(manual.commandline_options) > 10)
self.assertTrue(len(manual.signals) > 5)
self.assertTrue(len(manual.files) > 20)
More information about the tor-commits
mailing list