commit f34212fb0c91715d7a9f3ab3764033e3c962682b Author: Damian Johnson atagar@torproject.org Date: Wed Dec 25 16:14:51 2019 -0800

Sync manual and update assertions

The manual has changed quite a bit, requiring small parser adjustments. --- stem/cached_manual.sqlite | Bin 249856 -> 252928 bytes stem/manual.py | 53 ++++++++++++++++++++++++++-------------------- stem/settings.cfg | 3 +++ test/integ/manual.py | 27 +++++++++++------------ test/unit/manual.py | 2 +- 5 files changed, 46 insertions(+), 39 deletions(-)

diff --git a/stem/cached_manual.sqlite b/stem/cached_manual.sqlite index 515e894b..a2d9b02c 100644 Binary files a/stem/cached_manual.sqlite and b/stem/cached_manual.sqlite differ diff --git a/stem/manual.py b/stem/manual.py index 24596851..5b502a4d 100644 --- a/stem/manual.py +++ b/stem/manual.py @@ -730,7 +730,9 @@ def _get_indented_descriptions(lines): options, last_arg = OrderedDict(), None

for line in lines: - if line and not line.startswith(' '): + if line == ' Note': + last_arg = None # manual has several indented 'Note' blocks + elif line and not line.startswith(' '): options[line], last_arg = [], line elif last_arg and line.startswith(' '): options[last_arg].append(line[4:]) @@ -756,43 +758,48 @@ def _add_config_options(config_options, category, lines): since that platform lacks getrlimit(). (Default: 1000) """

- last_option, usage, description = None, None, [] + def add_option(title, description): + if 'PER INSTANCE OPTIONS' in title: + return # skip, unfortunately amid the options + + if ', ' in title: + # Line actually had multiple options with the same description. For + # example... + # + # AlternateBridgeAuthority [nickname], AlternateDirAuthority [nickname]

- # Drop the section description. Each ends with a paragraph saying 'The - # following options...'. + for subtitle in title.split(', '): + add_option(subtitle, description) + else: + name, usage = title.split(' ', 1) if ' ' in title else (title, '') + summary = _config().get('manual.summary.%s' % name.lower(), '') + config_options[name] = ConfigOption(name, category, usage, summary, _join_lines(description).strip())

- desc_paragraph_index = None + # Remove the section's description by finding the sentence the section + # ends with.

- for i, line in enumerate(lines): - if 'The following options' in line: - desc_paragraph_index = i - break + end_indices = [i for (i, line) in enumerate(lines) if ('The following options' in line or 'PER SERVICE OPTIONS' in line)]

- if desc_paragraph_index is not None: - lines = lines[desc_paragraph_index:] # trim to the description paragrah + if end_indices: + lines = lines[max(end_indices):] # trim to the description paragrah lines = lines[lines.index(''):] # drop the paragraph

+ last_title, description = None, [] + for line in lines: if line and not line.startswith(' '): - if last_option: - summary = _config().get('manual.summary.%s' % last_option.lower(), '') - config_options[last_option] = ConfigOption(last_option, category, usage, summary, _join_lines(description).strip()) - - if ' ' in line: - last_option, usage = line.split(' ', 1) - else: - last_option, usage = line, '' + if last_title: + add_option(last_title, description)

- description = [] + last_title, description = line, [] else: if line.startswith(' '): line = line[4:]

description.append(line)

- if last_option: - summary = _config().get('manual.summary.%s' % last_option.lower(), '') - config_options[last_option] = ConfigOption(last_option, category, usage, summary, _join_lines(description).strip()) + if last_title: + add_option(last_title, description)

def _join_lines(lines): diff --git a/stem/settings.cfg b/stem/settings.cfg index b7be1ee1..c8cb29c9 100644 --- a/stem/settings.cfg +++ b/stem/settings.cfg @@ -331,6 +331,9 @@ manual.summary.HiddenServiceMaxStreamsCloseCircuit Closes rendezvous circuits th manual.summary.RendPostPeriod Period at which the rendezvous service descriptors are refreshed manual.summary.HiddenServiceDirGroupReadable Group read permissions for the hidden service directory manual.summary.HiddenServiceNumIntroductionPoints Number of introduction points the hidden service will have +manual.summary.HiddenServiceEnableIntroDoSDefense Introduction point DoS protection +manual.summary.HiddenServiceEnableIntroDoSRatePerSec Request rate allowed for the introduction point +manual.summary.HiddenServiceEnableIntroDoSBurstPerSec Burst rate allowed for the introduction point manual.summary.HiddenServiceSingleHopMode Allow non-anonymous single hop hidden services manual.summary.HiddenServiceNonAnonymousMode Enables HiddenServiceSingleHopMode to be set

diff --git a/test/integ/manual.py b/test/integ/manual.py index de86f695..ae0b2b16 100644 --- a/test/integ/manual.py +++ b/test/integ/manual.py @@ -38,20 +38,21 @@ EXPECTED_CATEGORIES = set([ 'AUTHORS', ])

-EXPECTED_CLI_OPTIONS = set(['-f FILE', '--hash-password PASSWORD', '--ignore-missing-torrc', '--defaults-torrc FILE', '--key-expiration [purpose]', '--list-fingerprint', '--list-deprecated-options', '--allow-missing-torrc', '--nt-service', '--verify-config', '--service remove|start|stop', '--passphrase-fd FILEDES', '--keygen [--newpass]', '--list-torrc-options', '--service install [--options command-line options]', '--list-modules', '--quiet|--hush', '--version', '-h, --help']) +EXPECTED_CLI_OPTIONS = set(['-f FILE', '--hash-password PASSWORD', '--ignore-missing-torrc', '--defaults-torrc FILE', '--key-expiration [purpose]', '--list-fingerprint', '--list-deprecated-options', '--allow-missing-torrc', '--nt-service', '--verify-config', '--dump-config short|full|non-builtin', '--service remove|start|stop', '--passphrase-fd FILEDES', '--keygen [--newpass]', '--list-torrc-options', '--service install [--options command-line options]', '--list-modules', '--quiet|--hush', '--version', '-h, --help']) EXPECTED_SIGNALS = set(['SIGTERM', 'SIGINT', 'SIGHUP', 'SIGUSR1', 'SIGUSR2', 'SIGCHLD', 'SIGPIPE', 'SIGXFSZ'])

EXPECTED_DESCRIPTION = """ -Tor is a connection-oriented anonymizing communication service. Users choose a source-routed path through a set of nodes, and negotiate a "virtual circuit" through the network, in which each node knows its predecessor and successor, but no others. Traffic flowing down the circuit is unwrapped by a symmetric key at each node, which reveals the downstream node. +Tor is a connection-oriented anonymizing communication service. Users choose a source-routed path through a set of nodes, and negotiate a "virtual circuit" through the network. Each node in a virtual circuit knows its predecessor and successor nodes, but no other nodes. Traffic flowing down the circuit is unwrapped by a symmetric key at each node, which reveals the downstream node.

-Basically, Tor provides a distributed network of servers or relays ("onion routers"). Users bounce their TCP streams -- web traffic, ftp, ssh, etc. -- around the network, and recipients, observers, and even the relays themselves have difficulty tracking the source of the stream. +Basically, Tor provides a distributed network of servers or relays ("onion routers"). Users bounce their TCP streams, including web traffic, ftp, ssh, etc., around the network, so that recipients, observers, and even the relays themselves have difficulty tracking the source of the stream.

-By default, tor will act as a client only. To help the network by providing bandwidth as a relay, change the ORPort configuration option -- see below. Please also consult the documentation on the Tor Project's website. + Note + By default, tor acts as a client only. To help the network by providing bandwidth as a relay, change the ORPort configuration option as mentioned below. Please also consult the documentation on the Tor Project's website. """.strip()

-EXPECTED_FILE_DESCRIPTION = 'Specify a new configuration file to contain further Tor configuration options OR pass - to make Tor read its configuration from standard input. (Default: @CONFDIR@/torrc, or $HOME/.torrc if that file is not found)' +EXPECTED_FILE_DESCRIPTION = 'Specify a new configuration file to contain further Tor configuration options, or pass - to make Tor read its configuration from standard input. (Default: @CONFDIR@/torrc, or $HOME/.torrc if that file is not found)'

-EXPECTED_BANDWIDTH_RATE_DESCRIPTION = 'A token bucket limits the average incoming bandwidth usage on this node to the specified number of bytes per second, and the average outgoing bandwidth usage to that same value. If you want to run a relay in the public network, this needs to be at the very least 75 KBytes for a relay (that is, 600 kbits) or 50 KBytes for a bridge (400 kbits) -- but of course, more is better; we recommend at least 250 KBytes (2 mbits) if possible. (Default: 1 GByte)\n\nNote that this option, and other bandwidth-limiting options, apply to TCP data only: They do not count TCP headers or DNS traffic.\n\nWith this option, and in other options that take arguments in bytes, KBytes, and so on, other formats are also supported. Notably, "KBytes" can also be written as "kilobytes" or "kb"; "MBytes" can be written as "megabytes" or "MB"; "kbits" can be written as "kilobits"; and so forth. Tor also accepts "byte" and "bit" in the singular. The prefixes "tera" and "T" are a lso recognized. If no units are given, we default to bytes. To avoid confusion, we recommend writing "bytes" or "bits" explicitly, since it's easy to forget that "B" means bytes, not bits.' +EXPECTED_BANDWIDTH_RATE_DESCRIPTION = 'A token bucket limits the average incoming bandwidth usage on this node to the specified number of bytes per second, and the average outgoing bandwidth usage to that same value. If you want to run a relay in the public network, this needs to be at the very least 75 KBytes for a relay (that is, 600 kbits) or 50 KBytes for a bridge (400 kbits) -- but of course, more is better; we recommend at least 250 KBytes (2 mbits) if possible. (Default: 1 GByte)\n\nNote that this option, and other bandwidth-limiting options, apply to TCP data only: They do not count TCP headers or DNS traffic.\n\nTor uses powers of two, not powers of ten, so 1 GByte is 1024*1024*1024 bytes as opposed to 1 billion bytes.\n\nWith this option, and in other options that take arguments in bytes, KBytes, and so on, other formats are also supported. Notably, "KBytes" can also be written as "kilobytes" or "kb"; "MBytes" can be written as "megabytes" or "MB"; "kbits" can be written a s "kilobits"; and so forth. Case doesn't matter. Tor also accepts "byte" and "bit" in the singular. The prefixes "tera" and "T" are also recognized. If no units are given, we default to bytes. To avoid confusion, we recommend writing "bytes" or "bits" explicitly, since it's easy to forget that "B" means bytes, not bits.'

EXPECTED_EXIT_POLICY_DESCRIPTION_START = 'Set an exit policy for this server. Each policy' EXPECTED_EXIT_POLICY_DESCRIPTION_END = 'it applies to both IPv4 and IPv6 addresses.' @@ -162,7 +163,11 @@ class TestManual(unittest.TestCase):

def assert_equal(category, expected, actual): if expected != actual: - self.fail("Changed tor's man page? The %s changed as follows...\n\nexpected: %s\n\nactual: %s" % (category, sorted(expected), sorted(actual))) + if isinstance(expected, (set, tuple, list)): + expected = sorted(expected) + actual = sorted(actual) + + self.fail("Changed tor's man page? The %s changed as follows...\n\nexpected: %s\n\nactual: %s" % (category, expected, actual))

manual = stem.manual.Manual.from_man(self.man_path)

@@ -237,14 +242,6 @@ class TestManual(unittest.TestCase): if name.startswith('_'): config_options_in_tor.remove(name)

- # TODO: Looks like options we should remove from tor... - # - # https://trac.torproject.org/projects/tor/ticket/17665 - - for option in ('SchedulerMaxFlushCells__', 'SchedulerLowWaterMark__', 'SchedulerHighWaterMark__'): - if option in config_options_in_tor: - config_options_in_tor.remove(option) - manual = stem.manual.Manual.from_man(self.man_path) config_options_in_manual = set(manual.config_options.keys())

diff --git a/test/unit/manual.py b/test/unit/manual.py index cd2fce94..72f847e5 100644 --- a/test/unit/manual.py +++ b/test/unit/manual.py @@ -234,7 +234,7 @@ class TestManual(unittest.TestCase):

self.assertEqual('tor - The second-generation onion router', manual.name) self.assertEqual('tor [OPTION value]...', manual.synopsis) - self.assertTrue(manual.description.startswith(EXPECTED_DESCRIPTION)) + self.assertTrue(manual.description.startswith('Tor is a connection-oriented anonymizing communication service.')) self.assertTrue(len(manual.commandline_options) > 10) self.assertTrue(len(manual.signals) > 5) self.assertTrue(len(manual.files) > 20)