[tor-commits] [stem/master] Synchronize manual cache

[atagar at torproject.org]

commit 950895c9b35d6f6ea3ec24d3b9cb078bf4976a98
Author: Damian Johnson <atagar at torproject.org>
Date:   Sat Jul 18 16:14:13 2020 -0700

    Synchronize manual cache
    
    Re-sync our manual cache with tor and correct a few issues with regard to
    python 3 and asyncio compatibility.
---
 cache_manual.py           |   4 ++--
 stem/cached_manual.sqlite | Bin 252928 -> 253952 bytes
 stem/manual.py            |   6 +++++-
 test/integ/manual.py      |  42 ++++++++++++++++++++++++++++++++++++------
 4 files changed, 43 insertions(+), 9 deletions(-)

diff --git a/cache_manual.py b/cache_manual.py
index 5bc68b57..803197f1 100755
--- a/cache_manual.py
+++ b/cache_manual.py
@@ -14,12 +14,12 @@ import stem.manual
 import stem.util.system
 
 GITWEB_MAN_LOG = 'https://gitweb.torproject.org/tor.git/log/doc/tor.1.txt'
-MAN_LOG_LINK = "href='/tor.git/commit/doc/tor.1.txt\\?id=([^']*)'"
+MAN_LOG_LINK = b"href='/tor.git/commit/doc/tor.1.txt\\?id=([^']*)'"
 
 if __name__ == '__main__':
   try:
     man_log_page = urllib.request.urlopen(GITWEB_MAN_LOG).read()
-    man_commit = re.search(MAN_LOG_LINK, man_log_page).group(1)
+    man_commit = re.search(MAN_LOG_LINK, man_log_page).group(1).decode('utf-8')
   except:
     print("Unable to determine the latest commit to edit tor's man page: %s" % sys.exc_info()[1])
     sys.exit(1)
diff --git a/stem/cached_manual.sqlite b/stem/cached_manual.sqlite
index a2d9b02c..c967dd32 100644
Binary files a/stem/cached_manual.sqlite and b/stem/cached_manual.sqlite differ
diff --git a/stem/manual.py b/stem/manual.py
index 2ce5a6ff..0376d112 100644
--- a/stem/manual.py
+++ b/stem/manual.py
@@ -66,7 +66,7 @@ import stem.util.system
 
 from typing import Any, BinaryIO, Dict, List, Mapping, Optional, Sequence, Tuple, Union
 
-Category = stem.util.enum.Enum('GENERAL', 'CLIENT', 'RELAY', 'DIRECTORY', 'AUTHORITY', 'HIDDEN_SERVICE', 'DENIAL_OF_SERVICE', 'TESTING', 'UNKNOWN')
+Category = stem.util.enum.Enum('GENERAL', 'CLIENT', 'CIRCUIT_TIMEOUT', 'DORMANT_MODE', 'NODE_SELECTION', 'RELAY', 'STATISTIC', 'DIRECTORY', 'AUTHORITY', 'HIDDEN_SERVICE', 'DENIAL_OF_SERVICE', 'TESTING', 'UNKNOWN')
 GITWEB_MANUAL_URL = 'https://gitweb.torproject.org/tor.git/plain/doc/man/tor.1.txt'
 CACHE_PATH = os.path.join(os.path.dirname(__file__), 'cached_manual.sqlite')
 DATABASE = None  # cache database connections
@@ -87,7 +87,11 @@ SCHEMA = (
 CATEGORY_SECTIONS = collections.OrderedDict((
   ('GENERAL OPTIONS', Category.GENERAL),
   ('CLIENT OPTIONS', Category.CLIENT),
+  ('CIRCUIT TIMEOUT OPTIONS', Category.CIRCUIT_TIMEOUT),
+  ('DORMANT MODE OPTIONS', Category.DORMANT_MODE),
+  ('NODE SELECTION OPTIONS', Category.NODE_SELECTION),
   ('SERVER OPTIONS', Category.RELAY),
+  ('STATISTICS OPTIONS', Category.STATISTIC),
   ('DIRECTORY SERVER OPTIONS', Category.DIRECTORY),
   ('DIRECTORY AUTHORITY SERVER OPTIONS', Category.AUTHORITY),
   ('HIDDEN SERVICE OPTIONS', Category.HIDDEN_SERVICE),
diff --git a/test/integ/manual.py b/test/integ/manual.py
index 3e721ac6..c80e517c 100644
--- a/test/integ/manual.py
+++ b/test/integ/manual.py
@@ -14,6 +14,7 @@ import test
 import test.runner
 
 from stem.manual import Category
+from stem.util.test_tools import async_test
 
 EXPECTED_CATEGORIES = set([
   'NAME',
@@ -23,7 +24,11 @@ EXPECTED_CATEGORIES = set([
   'THE CONFIGURATION FILE FORMAT',
   'GENERAL OPTIONS',
   'CLIENT OPTIONS',
+  'CIRCUIT TIMEOUT OPTIONS',
+  'DORMANT MODE OPTIONS',
+  'NODE SELECTION OPTIONS',
   'SERVER OPTIONS',
+  'STATISTICS OPTIONS',
   'DIRECTORY SERVER OPTIONS',
   'DIRECTORY AUTHORITY SERVER OPTIONS',
   'HIDDEN SERVICE OPTIONS',
@@ -38,7 +43,30 @@ EXPECTED_CATEGORIES = set([
   'AUTHORS',
 ])
 
-EXPECTED_CLI_OPTIONS = set(['-f FILE', '--hash-password PASSWORD', '--ignore-missing-torrc', '--defaults-torrc FILE', '--key-expiration [purpose]', '--list-fingerprint', '--list-deprecated-options', '--allow-missing-torrc', '--nt-service', '--verify-config', '--dump-config short|full|non-builtin', '--service remove|start|stop', '--passphrase-fd FILEDES', '--keygen [--newpass]', '--list-torrc-options', '--service install [--options command-line options]', '--list-modules', '--quiet|--hush', '--version', '-h, --help'])
+EXPECTED_CLI_OPTIONS = set([
+  '--allow-missing-torrc',
+  '--dbg-...',
+  '--defaults-torrc FILE',
+  '--dump-config short|full',
+  '--hash-password PASSWORD',
+  '--ignore-missing-torrc',
+  '--key-expiration [purpose]',
+  '--keygen [--newpass]',
+  '--list-deprecated-options',
+  '--list-fingerprint',
+  '--list-modules',
+  '--list-torrc-options',
+  '--nt-service',
+  '--passphrase-fd FILEDES',
+  '--quiet|--hush',
+  '--service install [--options command-line options]',
+  '--service remove|start|stop',
+  '--verify-config',
+  '--version',
+  '-f FILE',
+  '-h, --help',
+])
+
 EXPECTED_SIGNALS = set(['SIGTERM', 'SIGINT', 'SIGHUP', 'SIGUSR1', 'SIGUSR2', 'SIGCHLD', 'SIGPIPE', 'SIGXFSZ'])
 
 EXPECTED_DESCRIPTION = """
@@ -216,20 +244,22 @@ class TestManual(unittest.TestCase):
     self.assertEqual(['tor - The second-generation onion router'], categories['NAME'])
     self.assertEqual(['tor [OPTION value]...'], categories['SYNOPSIS'])
 
-  def test_has_all_tor_config_options(self):
+  @async_test
+  async def test_has_all_tor_config_options(self):
     """
     Check that all the configuration options tor supports are in the man page.
     """
 
     self.requires_downloaded_manual()
 
-    with test.runner.get_runner().get_tor_controller() as controller:
-      config_options_in_tor = set([line.split()[0] for line in controller.get_info('config/names').splitlines() if line.split()[1] != 'Virtual'])
+    async with await test.runner.get_runner().get_tor_controller() as controller:
+      config_options_in_tor = set([line.split()[0] for line in (await controller.get_info('config/names')).splitlines() if line.split()[1] != 'Virtual'])
 
-      # options starting with an underscore are hidden by convention
+      # options starting with an underscore are hidden by convention, but the
+      # manual includes OwningControllerProcess
 
       for name in list(config_options_in_tor):
-        if name.startswith('_'):
+        if name.startswith('_') and name != '__OwningControllerProcess':
           config_options_in_tor.remove(name)
 
     manual = stem.manual.Manual.from_man(self.man_path)