[tor-commits] [stem/master] Use man arguments to help with parsing

[atagar at torproject.org]

commit 72786c6edaeb4a6de955bf7c0a459746181222a1
Author: Damian Johnson <atagar at torproject.org>
Date:   Fri Nov 27 20:55:39 2015 -0800

    Use man arguments to help with parsing
    
    Using a couple arguments to simplify parsing...
    
      * MANWIDTH so we don't need to contend with line wrapping.
    
      * --encoding=ascii to normalize our output. Seems on my netbook this is the
        default, causing some long dashes to be replaced with '--'.
---
 stem/manual.py       |   21 ++++-----------------
 stem/util/system.py  |    8 ++++++--
 test/integ/manual.py |   12 ++++++------
 test/unit/manual.py  |    4 ++--
 4 files changed, 18 insertions(+), 27 deletions(-)

diff --git a/stem/manual.py b/stem/manual.py
index be25353..138380e 100644
--- a/stem/manual.py
+++ b/stem/manual.py
@@ -244,9 +244,9 @@ class Manual(object):
     """
 
     try:
-      man_output = stem.util.system.call('man -P cat %s' % man_path)
+      man_output = stem.util.system.call('man --encoding=ascii -P cat %s' % man_path, env = {'MANWIDTH': '10000000'})
     except OSError as exc:
-      raise IOError("Unable to run 'man -P cat %s': %s" % (man_path, exc))
+      raise IOError("Unable to run 'man --encoding=ascii -P cat %s': %s" % (man_path, exc))
 
     categories, config_options = _get_categories(man_output), OrderedDict()
 
@@ -425,20 +425,7 @@ def _add_config_options(config_options, category, lines):
 
 def _join_lines(lines):
   """
-  The man page provides line-wrapped content. Attempting to undo that. This is
-  close to a simple join, but we still want empty lines to provide newlines.
+  Simple join, except we want empty lines to still provide a newline.
   """
 
-  content = []
-
-  for line in lines:
-    if line:
-      if content and content[-1][-1] != '\n':
-        line = ' ' + line
-
-      content.append(line)
-    else:
-      if content and content[-1][-1] != '\n':
-        content.append('\n\n')
-
-  return ''.join(content)
+  return ''.join([line if line else '\n\n' for line in lines])
diff --git a/stem/util/system.py b/stem/util/system.py
index 4f74f81..7976e48 100644
--- a/stem/util/system.py
+++ b/stem/util/system.py
@@ -951,7 +951,7 @@ def files_with_suffix(base_path, suffix):
           yield os.path.join(root, filename)
 
 
-def call(command, default = UNDEFINED, ignore_exit_status = False):
+def call(command, default = UNDEFINED, ignore_exit_status = False, env = None):
   """
   call(command, default = UNDEFINED, ignore_exit_status = False)
 
@@ -959,10 +959,14 @@ def call(command, default = UNDEFINED, ignore_exit_status = False):
   results. This is not actually ran in a shell so pipes and other shell syntax
   are not permitted.
 
+  .. versionchanged:: 1.5.0
+     Added env argument.
+
   :param str,list command: command to be issued
   :param object default: response if the query fails
   :param bool ignore_exit_status: reports failure if our command's exit status
     was non-zero
+  :param dict env: environment variables
 
   :returns: **list** with the lines of output from the command
 
@@ -978,7 +982,7 @@ def call(command, default = UNDEFINED, ignore_exit_status = False):
     is_shell_command = command_list[0] in SHELL_COMMANDS
 
     start_time = time.time()
-    process = subprocess.Popen(command_list, stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell = is_shell_command)
+    process = subprocess.Popen(command_list, stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell = is_shell_command, env = env)
 
     stdout, stderr = process.communicate()
     stdout, stderr = stdout.strip(), stderr.strip()
diff --git a/test/integ/manual.py b/test/integ/manual.py
index 3fbf89a..d9cf171 100644
--- a/test/integ/manual.py
+++ b/test/integ/manual.py
@@ -40,14 +40,14 @@ EXPECTED_SIGNALS = set(['SIGTERM', 'SIGINT', 'SIGHUP', 'SIGUSR1', 'SIGUSR2', 'SI
 EXPECTED_DESCRIPTION = """
 Tor is a connection-oriented anonymizing communication service. Users choose a source-routed path through a set of nodes, and negotiate a "virtual circuit" through the network, in which each node knows its predecessor and successor, but no others. Traffic flowing down the circuit is unwrapped by a symmetric key at each node, which reveals the downstream node.
 
-Basically, Tor provides a distributed network of servers or relays ("onion routers"). Users bounce their TCP streams - web traffic, ftp, ssh, etc. - around the network, and recipients, observers, and even the relays themselves have difficulty tracking the source of the stream.
+Basically, Tor provides a distributed network of servers or relays ("onion routers"). Users bounce their TCP streams -- web traffic, ftp, ssh, etc. -- around the network, and recipients, observers, and even the relays themselves have difficulty tracking the source of the stream.
 
-By default, tor will only act as a client only. To help the network by providing bandwidth as a relay, change the ORPort configuration option - see below. Please also consult the documentation on the Tor Project's website.
+By default, tor will only act as a client only. To help the network by providing bandwidth as a relay, change the ORPort configuration option -- see below. Please also consult the documentation on the Tor Project's website.
 """.strip()
 
 EXPECTED_FILE_DESCRIPTION = 'Specify a new configuration file to contain further Tor configuration options OR pass - to make Tor read its configuration from standard input. (Default: @CONFDIR@/torrc, or $HOME/.torrc if that file is not found)'
 
-EXPECTED_BANDWIDTH_RATE_DESCRIPTION = 'A token bucket limits the average incoming bandwidth usage on this node to the specified number of bytes per second, and the average outgoing bandwidth usage to that same value. If you want to run a relay in the public network, this needs to be at the very least 75 KBytes for a relay (that is, 600 kbits) or 50 KBytes for a bridge (400 kbits) - but of course, more is better; we recommend at least 250 KBytes (2 mbits) if possible. (Default: 1 GByte)\n\nWith this option, and in other options that take arguments in bytes, KBytes, and so on, other formats are also supported. Notably, "KBytes" can also be written as "kilobytes" or "kb"; "MBytes" can be written as "megabytes" or "MB"; "kbits" can be written as "kilobits"; and so forth. Tor also accepts "byte" and "bit" in the singular. The prefixes "tera" and "T" are also recognized. If no units are given, we default to bytes. To avoid confusion, we recommend writing "bytes" or "bits" explicitly, sinc
 e it\'s easy to forget that "B" means bytes, not bits.'
+EXPECTED_BANDWIDTH_RATE_DESCRIPTION = 'A token bucket limits the average incoming bandwidth usage on this node to the specified number of bytes per second, and the average outgoing bandwidth usage to that same value. If you want to run a relay in the public network, this needs to be at the very least 75 KBytes for a relay (that is, 600 kbits) or 50 KBytes for a bridge (400 kbits) -- but of course, more is better; we recommend at least 250 KBytes (2 mbits) if possible. (Default: 1 GByte)\n\nWith this option, and in other options that take arguments in bytes, KBytes, and so on, other formats are also supported. Notably, "KBytes" can also be written as "kilobytes" or "kb"; "MBytes" can be written as "megabytes" or "MB"; "kbits" can be written as "kilobits"; and so forth. Tor also accepts "byte" and "bit" in the singular. The prefixes "tera" and "T" are also recognized. If no units are given, we default to bytes. To avoid confusion, we recommend writing "bytes" or "bits" explicitly, sin
 ce it\'s easy to forget that "B" means bytes, not bits.'
 
 
 class TestManual(unittest.TestCase):
@@ -259,9 +259,9 @@ class TestManual(unittest.TestCase):
       #
       # https://trac.torproject.org/projects/tor/ticket/17665
 
-      config_options_in_tor.remove('SchedulerMaxFlushCells__')
-      config_options_in_tor.remove('SchedulerLowWaterMark__')
-      config_options_in_tor.remove('SchedulerHighWaterMark__')
+      for option in ('SchedulerMaxFlushCells__', 'SchedulerLowWaterMark__', 'SchedulerHighWaterMark__'):
+        if option in config_options_in_tor:
+          config_options_in_tor.remove(option)
 
     manual = stem.manual.Manual.from_man(self.man_path)
     config_options_in_manual = set(manual.config_options.keys())
diff --git a/test/unit/manual.py b/test/unit/manual.py
index 7ca69da..1f5ea74 100644
--- a/test/unit/manual.py
+++ b/test/unit/manual.py
@@ -104,13 +104,13 @@ class TestManual(unittest.TestCase):
     self.assertEqual(b'a2x output', output.getvalue())
     call_mock.assert_called_once_with('a2x -f manpage /no/such/path/tor.1.txt')
 
-  @patch('stem.util.system.call', Mock(side_effect = OSError('man -P cat tor returned exit status 16')))
+  @patch('stem.util.system.call', Mock(side_effect = OSError('man --encoding=ascii -P cat tor returned exit status 16')))
   def test_from_man_when_manual_is_unavailable(self):
     try:
       stem.manual.Manual.from_man()
       self.fail("fetching the manual should fail when it's unavailable")
     except IOError as exc:
-      self.assertEqual("Unable to run 'man -P cat tor': man -P cat tor returned exit status 16", str(exc))
+      self.assertEqual("Unable to run 'man --encoding=ascii -P cat tor': man --encoding=ascii -P cat tor returned exit status 16", str(exc))
 
   @patch('stem.util.system.call', Mock(return_value = []))
   def test_when_man_is_empty(self):