[tor-commits] [nyx/master] Use stem.manual module for retrieving manual information

commit 2382eac70ba8f08c62463b04041ad05a1d842951 Author: Damian Johnson atagar@torproject.org Date: Sun Dec 6 12:58:52 2015 -0800

Use stem.manual module for retrieving manual information

Stem's shiny new stem.manual module is faster, tested, more up to date and... oh, right. It works.

Removing our code for parsing tor's man page and replacing it was a very simple call to Stem which does that all now and *much* better than nyx ever did. --- nyx/config/torrc.cfg | 260 ------------------------------ nyx/config_panel.py | 45 ++++-- nyx/starter.py | 12 +- nyx/util/tor_config.py | 409 +----------------------------------------------- 4 files changed, 31 insertions(+), 695 deletions(-)

diff --git a/nyx/config/torrc.cfg b/nyx/config/torrc.cfg index 5f5d993..71d4608 100644 --- a/nyx/config/torrc.cfg +++ b/nyx/config/torrc.cfg @@ -4,8 +4,6 @@ # # * torrc.alias Aliases for configuration options tor will accept. # * torrc.units Labels accepted by tor for various units. -# * torrc.important Important configuration options which are shown by default. -# * torrc.summary Short summary describing the option. # ################################################################################

@@ -53,261 +51,3 @@ torrc.units.time.min minute, minutes torrc.units.time.hour hour, hours torrc.units.time.day day, days torrc.units.time.week week, weeks - -# Especially important tor configuration options. - -torrc.important BandwidthRate -torrc.important BandwidthBurst -torrc.important RelayBandwidthRate -torrc.important RelayBandwidthBurst -torrc.important ControlPort -torrc.important HashedControlPassword -torrc.important CookieAuthentication -torrc.important DataDirectory -torrc.important Log -torrc.important RunAsDaemon -torrc.important User - -torrc.important Bridge -torrc.important ExcludeNodes -torrc.important MaxCircuitDirtiness -torrc.important SocksPort -torrc.important UseBridges - -torrc.important BridgeRelay -torrc.important ContactInfo -torrc.important ExitPolicy -torrc.important MyFamily -torrc.important Nickname -torrc.important ORPort -torrc.important PortForwarding -torrc.important AccountingMax -torrc.important AccountingStart - -torrc.important DirPortFrontPage -torrc.important DirPort - -torrc.important HiddenServiceDir -torrc.important HiddenServicePort - -# General Config Options - -torrc.summary.BandwidthRate Average bandwidth usage limit -torrc.summary.BandwidthBurst Maximum bandwidth usage limit -torrc.summary.MaxAdvertisedBandwidth Limit for the bandwidth we advertise as being available for relaying -torrc.summary.RelayBandwidthRate Average bandwidth usage limit for relaying -torrc.summary.RelayBandwidthBurst Maximum bandwidth usage limit for relaying -torrc.summary.PerConnBWRate Average relayed bandwidth limit per connection -torrc.summary.PerConnBWBurst Maximum relayed bandwidth limit per connection -torrc.summary.ConnLimit Minimum number of file descriptors for Tor to start -torrc.summary.ConstrainedSockets Shrinks sockets to ConstrainedSockSize -torrc.summary.ConstrainedSockSize Limit for the received and transmit buffers of sockets -torrc.summary.ControlPort Port providing access to tor controllers (nyx, vidalia, etc) -torrc.summary.ControlListenAddress Address providing controller access -torrc.summary.ControlSocket Socket providing controller access -torrc.summary.HashedControlPassword Hash of the password for authenticating to the control port -torrc.summary.CookieAuthentication If set, authenticates controllers via a cookie -torrc.summary.CookieAuthFile Location of the authentication cookie -torrc.summary.CookieAuthFileGroupReadable Group read permissions for the authentication cookie -torrc.summary.ControlPortWriteToFile Path for a file tor writes containing its control port -torrc.summary.ControlPortFileGroupReadable Group read permissions for the control port file -torrc.summary.DataDirectory Location for storing runtime data (state, keys, etc) -torrc.summary.DirServer Alternative directory authorities -torrc.summary.AlternateDirAuthority Alternative directory authorities (consensus only) -torrc.summary.AlternateHSAuthority Alternative directory authorities (hidden services only) -torrc.summary.AlternateBridgeAuthority Alternative directory authorities (bridges only) -torrc.summary.DisableAllSwap Locks all allocated memory so they can't be paged out -torrc.summary.FetchDirInfoEarly Keeps consensus information up to date, even if unnecessary -torrc.summary.FetchDirInfoExtraEarly Updates consensus information when it's first available -torrc.summary.FetchHidServDescriptors Toggles if hidden service descriptors are fetched automatically or not -torrc.summary.FetchServerDescriptors Toggles if the consensus is fetched automatically or not -torrc.summary.FetchUselessDescriptors Toggles if relay descriptors are fetched when they aren't strictly necessary -torrc.summary.Group GID for the process when started -torrc.summary.HttpProxy HTTP proxy for connecting to tor -torrc.summary.HttpProxyAuthenticator Authentication credentials for HttpProxy -torrc.summary.HttpsProxy SSL proxy for connecting to tor -torrc.summary.HttpsProxyAuthenticator Authentication credentials for HttpsProxy -torrc.summary.Socks4Proxy SOCKS 4 proxy for connecting to tor -torrc.summary.Socks5Proxy SOCKS 5 for connecting to tor -torrc.summary.Socks5ProxyUsername Username for connecting to the Socks5Proxy -torrc.summary.Socks5ProxyPassword Password for connecting to the Socks5Proxy -torrc.summary.KeepalivePeriod Rate at which to send keepalive packets -torrc.summary.Log Runlevels and location for tor logging -torrc.summary.LogMessageDomains Includes a domain when logging messages -torrc.summary.OutboundBindAddress Sets the IP used for connecting to tor -torrc.summary.PidFile Path for a file tor writes containing its process id -torrc.summary.ProtocolWarnings Toggles if protocol errors give warnings or not -torrc.summary.RunAsDaemon Toggles if tor runs as a daemon process -torrc.summary.LogTimeGranularity limits granularity of log message timestamps -torrc.summary.SafeLogging Toggles if logs are scrubbed of sensitive information -torrc.summary.User UID for the process when started -torrc.summary.HardwareAccel Toggles if tor attempts to use hardware acceleration -torrc.summary.AccelName OpenSSL engine name for crypto acceleration -torrc.summary.AccelDir Crypto acceleration library path -torrc.summary.AvoidDiskWrites Toggles if tor avoids frequently writing to disk -torrc.summary.TunnelDirConns Toggles if directory requests can be made over the ORPort -torrc.summary.PreferTunneledDirConns Avoids directory requests that can't be made over the ORPort if set -torrc.summary.CircuitPriorityHalflife Overwrite method for prioritizing traffic among relayed connections -torrc.summary.DisableIOCP Disables use of the Windows IOCP networking API -torrc.summary.CountPrivateBandwidth Applies rate limiting to private IP addresses - -# Client Config Options - -torrc.summary.AllowInvalidNodes Permits use of relays flagged as invalid by authorities -torrc.summary.ExcludeSingleHopRelays Permits use of relays that allow single hop connections -torrc.summary.Bridge Available bridges -torrc.summary.LearnCircuitBuildTimeout Toggles adaptive timeouts for circuit creation -torrc.summary.CircuitBuildTimeout Initial timeout for circuit creation -torrc.summary.CircuitIdleTimeout Timeout for closing circuits that have never been used -torrc.summary.CircuitStreamTimeout Timeout for shifting streams among circuits -torrc.summary.ClientOnly Ensures that we aren't used as a relay or directory mirror -torrc.summary.ExcludeNodes Relays or locales never to be used in circuits -torrc.summary.ExcludeExitNodes Relays or locales never to be used for exits -torrc.summary.ExitNodes Preferred final hop for circuits -torrc.summary.EntryNodes Preferred first hops for circuits -torrc.summary.StrictNodes Never uses notes outside of Entry/ExitNodes -torrc.summary.FascistFirewall Only make outbound connections on FirewallPorts -torrc.summary.FirewallPorts Ports used by FascistFirewall -torrc.summary.HidServAuth Authentication credentials for connecting to a hidden service -torrc.summary.ReachableAddresses Rules for bypassing the local firewall -torrc.summary.ReachableDirAddresses Rules for bypassing the local firewall (directory fetches) -torrc.summary.ReachableORAddresses Rules for bypassing the local firewall (OR connections) -torrc.summary.LongLivedPorts Ports requiring highly reliable relays -torrc.summary.MapAddress Alias mappings for address requests -torrc.summary.NewCircuitPeriod Period for considering the creation of new circuits -torrc.summary.MaxCircuitDirtiness Duration for reusing constructed circuits -torrc.summary.NodeFamily Define relays as belonging to a family -torrc.summary.EnforceDistinctSubnets Prevent use of multiple relays from the same subnet on a circuit -torrc.summary.SocksPort Port for using tor as a Socks proxy -torrc.summary.SocksListenAddress Address from which Socks connections can be made -torrc.summary.SocksPolicy Access policy for the pocks port -torrc.summary.SocksTimeout Time until idle or unestablished socks connections are closed -torrc.summary.TrackHostExits Maintains use of the same exit whenever connecting to this destination -torrc.summary.TrackHostExitsExpire Time until use of an exit for tracking expires -torrc.summary.UpdateBridgesFromAuthority Toggles fetching bridge descriptors from the authorities -torrc.summary.UseBridges Make use of configured bridges -torrc.summary.UseEntryGuards Use guard relays for first hop -torrc.summary.NumEntryGuards Pool size of guard relays we'll select from -torrc.summary.SafeSocks Toggles rejecting unsafe variants of the socks protocol -torrc.summary.TestSocks Provide notices for if socks connections are of the safe or unsafe variants -torrc.summary.WarnUnsafeSocks Toggle warning of unsafe socks connection -torrc.summary.VirtualAddrNetwork Address range used with MAPADDRESS -torrc.summary.AllowNonRFC953Hostnames Toggles blocking invalid characters in hostname resolution -torrc.summary.AllowDotExit Toggles allowing exit notation in addresses -torrc.summary.FastFirstHopPK Toggle public key usage for the first hop -torrc.summary.TransPort Port for transparent proxying if the OS supports it -torrc.summary.TransListenAddress Address from which transparent proxy connections can be made -torrc.summary.NATDPort Port for forwarding ipfw NATD connections -torrc.summary.NATDListenAddress Address from which NATD forwarded connections can be made -torrc.summary.AutomapHostsOnResolve Map addresses ending with special suffixes to virtual addresses -torrc.summary.AutomapHostsSuffixes Address suffixes recognized by AutomapHostsOnResolve -torrc.summary.DNSPort Port from which DNS responses are fetched instead of tor -torrc.summary.DNSListenAddress Address for performing DNS resolution -torrc.summary.ClientDNSRejectInternalAddresses Ignores DNS responses for internal addresses -torrc.summary.ClientRejectInternalAddresses Disables use of Tor for internal connections -torrc.summary.DownloadExtraInfo Toggles fetching of extra information about relays -torrc.summary.FallbackNetworkstatusFile Path for a fallback cache of the consensus -torrc.summary.WarnPlaintextPorts Toggles warnings for using risky ports -torrc.summary.RejectPlaintextPorts Prevents connections on risky ports -torrc.summary.AllowSingleHopCircuits Makes use of single hop exits if able - -# Server Config Options - -torrc.summary.Address Overwrites address others will use to reach this relay -torrc.summary.AllowSingleHopExits Toggles permitting use of this relay as a single hop proxy -torrc.summary.AssumeReachable Skips reachability test at startup -torrc.summary.BridgeRelay Act as a bridge -torrc.summary.ContactInfo Contact information for this relay -torrc.summary.ExitPolicy Traffic destinations that can exit from this relay -torrc.summary.ExitPolicyRejectPrivate Prevent exiting connection on the local network -torrc.summary.MaxOnionsPending Decryption queue size -torrc.summary.MyFamily Other relays this operator administers -torrc.summary.Nickname Identifier for this relay -torrc.summary.NumCPUs Number of processes spawned for decryption -torrc.summary.ORPort Port used to accept relay traffic -torrc.summary.ORListenAddress Address for relay connections -torrc.summary.PortForwarding Use UPnP or NAT-PMP if needed to relay -torrc.summary.PortForwardingHelper Executable for configuring port forwarding -torrc.summary.PublishServerDescriptor Types of descriptors published -torrc.summary.ShutdownWaitLength Delay before quitting after receiving a SIGINT signal -torrc.summary.HeartbeatPeriod Rate at which an INFO level heartbeat message is sent -torrc.summary.AccountingMax Amount of traffic before hibernating -torrc.summary.AccountingStart Duration of an accounting period -torrc.summary.RefuseUnknownExits Prevents relays not in the consensus from using us as an exit -torrc.summary.ServerDNSResolvConfFile Overriding resolver config for DNS queries we provide -torrc.summary.ServerDNSAllowBrokenConfig Toggles if we persist despite configuration parsing errors or not -torrc.summary.ServerDNSSearchDomains Toggles if our DNS queries search for addresses in the local domain -torrc.summary.ServerDNSDetectHijacking Toggles testing for DNS hijacking -torrc.summary.ServerDNSTestAddresses Addresses to test to see if valid DNS queries are being hijacked -torrc.summary.ServerDNSAllowNonRFC953Hostnames Toggles if we reject DNS queries with invalid characters -torrc.summary.BridgeRecordUsageByCountry Tracks geoip information on bridge usage -torrc.summary.ServerDNSRandomizeCase Toggles DNS query case randomization -torrc.summary.GeoIPFile Path to file containing geoip information -torrc.summary.CellStatistics Toggles storing circuit queue duration to disk -torrc.summary.DirReqStatistics Toggles storing network status counts and performance to disk -torrc.summary.EntryStatistics Toggles storing client connection counts to disk -torrc.summary.ExitPortStatistics Toggles storing traffic and port usage data to disk -torrc.summary.ConnDirectionStatistics Toggles storing connection use to disk -torrc.summary.ExtraInfoStatistics Publishes statistic data in the extra-info documents - -# Directory Server Options - -torrc.summary.AuthoritativeDirectory Act as a directory authority -torrc.summary.DirPortFrontPage Publish this html file on the DirPort -torrc.summary.V1AuthoritativeDirectory Generates a version 1 consensus -torrc.summary.V2AuthoritativeDirectory Generates a version 2 consensus -torrc.summary.V3AuthoritativeDirectory Generates a version 3 consensus -torrc.summary.VersioningAuthoritativeDirectory Provides opinions on recommended versions of tor -torrc.summary.NamingAuthoritativeDirectory Provides opinions on fingerprint to nickname bindings -torrc.summary.HSAuthoritativeDir Toggles accepting hidden service descriptors -torrc.summary.HidServDirectoryV2 Toggles accepting version 2 hidden service descriptors -torrc.summary.BridgeAuthoritativeDir Acts as a bridge authority -torrc.summary.MinUptimeHidServDirectoryV2 Required uptime before accepting hidden service directory -torrc.summary.DirPort Port for directory connections -torrc.summary.DirListenAddress Address the directory service is bound to -torrc.summary.DirPolicy Access policy for the DirPort -torrc.summary.FetchV2Networkstatus Get the obsolete V2 consensus - -# Directory Authority Server Options - -torrc.summary.RecommendedVersions Tor versions believed to be safe -torrc.summary.RecommendedClientVersions Tor versions believed to be safe for clients -torrc.summary.RecommendedServerVersions Tor versions believed to be safe for relays -torrc.summary.ConsensusParams Params entry of the networkstatus vote -torrc.summary.DirAllowPrivateAddresses Toggles allowing arbitrary input or non-public IPs in descriptors -torrc.summary.AuthDirBadDir Relays to be flagged as bad directory caches -torrc.summary.AuthDirBadExit Relays to be flagged as bad exits -torrc.summary.AuthDirInvalid Relays from which the valid flag is withheld -torrc.summary.AuthDirReject Relays to be dropped from the consensus -torrc.summary.AuthDirListBadDirs Toggles if we provide an opinion on bad directory caches -torrc.summary.AuthDirListBadExits Toggles if we provide an opinion on bad exits -torrc.summary.AuthDirRejectUnlisted Rejects further relay descriptors -torrc.summary.AuthDirMaxServersPerAddr Limit on the number of relays accepted per ip -torrc.summary.AuthDirMaxServersPerAuthAddr Limit on the number of relays accepted per an authority's ip -torrc.summary.BridgePassword Password for requesting bridge information -torrc.summary.V3AuthVotingInterval Consensus voting interval -torrc.summary.V3AuthVoteDelay Wait time to collect votes of other authorities -torrc.summary.V3AuthDistDelay Wait time to collect the signatures of other authorities -torrc.summary.V3AuthNIntervalsValid Number of voting intervals a consensus is valid for -torrc.summary.V3BandwidthsFile Path to a file containing measured relay bandwidths -torrc.summary.V3AuthUseLegacyKey Signs consensus with both the current and legacy keys -torrc.summary.RephistTrackTime Discards old, unchanged reliability informaition - -# Hidden Service Options - -torrc.summary.HiddenServiceDir Directory contents for the hidden service -torrc.summary.HiddenServicePort Port the hidden service is provided on -torrc.summary.PublishHidServDescriptors Toggles automated publishing of the hidden service to the rendezvous directory -torrc.summary.HiddenServiceVersion Version for published hidden service descriptors -torrc.summary.HiddenServiceAuthorizeClient Restricts access to the hidden service -torrc.summary.RendPostPeriod Period at which the rendezvous service descriptors are refreshed - -# Testing Network Options - -torrc.summary.TestingTorNetwork Overrides other options to be a testing network -torrc.summary.TestingV3AuthInitialVotingInterval Overrides V3AuthVotingInterval for the first consensus -torrc.summary.TestingV3AuthInitialVoteDelay Overrides TestingV3AuthInitialVoteDelay for the first consensus -torrc.summary.TestingV3AuthInitialDistDelay Overrides TestingV3AuthInitialDistDelay for the first consensus -torrc.summary.TestingAuthDirTimeToLearnReachability Delay until opinions are given about which relays are running or not -torrc.summary.TestingEstimatedDescriptorPropagationTime Delay before clients attempt to fetch descriptors from directory caches - diff --git a/nyx/config_panel.py b/nyx/config_panel.py index 36688a9..1a3a836 100644 --- a/nyx/config_panel.py +++ b/nyx/config_panel.py @@ -8,10 +8,18 @@ import curses import nyx.controller import nyx.popups

+import stem.control +import stem.manual + from nyx.util import panel, tor_config, tor_controller, ui_tools

-from stem.control import State -from stem.util import conf, enum, str_tools +from stem.util import conf, enum, log, str_tools + +try: + # added in python 3.2 + from functools import lru_cache +except ImportError: + from stem.util.lru_cache import lru_cache

SortAttr = enum.Enum('OPTION', 'VALUE', 'VALUE_TYPE', 'CATEGORY', 'USAGE', 'SUMMARY', 'DESCRIPTION', 'MAN_PAGE_ENTRY', 'IS_SET')

@@ -34,6 +42,15 @@ CONFIG = conf.config_dict('nyx', { }, conf_handler)

+@lru_cache() +def tor_manual(): + try: + return stem.manual.Manual.from_man() + except IOError as exc: + log.debug("Unable to use 'man tor' to get information about config options (%s), using bundled information instead" % exc) + return stem.manual.Manual.from_cache() + + class ConfigEntry(): """ Configuration option in the panel. @@ -42,7 +59,7 @@ class ConfigEntry(): def __init__(self, option, entry_type): self._option = option self._value_type = entry_type - self._man_entry = tor_config.get_config_description(option) + self._man_entry = tor_manual().config_options.get(option)

def category(self): """ @@ -51,7 +68,7 @@ class ConfigEntry(): :returns: **Category** this option belongs to """

- return self._man_entry.category if self._man_entry else tor_config.Category.UNKNOWN + return self._man_entry.category if self._man_entry else stem.manual.Category.UNKNOWN

def option(self): """ @@ -100,19 +117,13 @@ class ConfigEntry(): :returns: short **str** description of the option """

- summary = tor_config.get_config_summary(self.option()) - - if summary: - return summary - else: - # uses the full man page description if a summary is unavailable - return self._man_entry.description if self._man_entry else '' + return self._man_entry.summary if self._man_entry else ''

def manual_entry(self): """ Provides the entry's man page entry.

- :returns: **ManPageEntry** if it was loaded, and **None** otherwise + :returns: :class:`~stem.manual.ConfigOption` if it was loaded, and **None** otherwise """

return self._man_entry @@ -144,13 +155,13 @@ class ConfigEntry(): elif attr == SortAttr.VALUE_TYPE: return self.value_type() elif attr == SortAttr.USAGE: - return self._man_entry.arg_usage if self._man_entry else '' + return self._man_entry.usage if self._man_entry else '' elif attr == SortAttr.SUMMARY: return self.summary() elif attr == SortAttr.DESCRIPTION: return self._man_entry.description if self._man_entry else '' elif attr == SortAttr.MAN_PAGE_ENTRY: - return self._man_entry.index if self._man_entry else 99999 # sorts non-man entries last + return tor_manual().config_options.keys().index(self.option()) if self.option() in tor_manual().config_options else 99999 # sorts non-man entries last elif attr == SortAttr.IS_SET: return not self.is_set()

@@ -177,7 +188,7 @@ class ConfigPanel(panel.Panel): # fetches configuration options if a new instance, otherewise keeps our # current contents

- if event_type == State.INIT: + if event_type == stem.control.State.INIT: self._load_config_options()

def _load_config_options(self): @@ -211,7 +222,7 @@ class ConfigPanel(panel.Panel):

# mirror listing with only the important configuration options

- self._conf_important_contents = filter(lambda entry: tor_config.is_important(entry.option()), self._conf_contents) + self._conf_important_contents = filter(lambda entry: stem.manual.is_important(entry.option()), self._conf_contents)

# if there aren't any important options then show everything

@@ -526,7 +537,7 @@ class ConfigPanel(panel.Panel): value_attr_label = ', '.join([ 'custom' if selection.is_set() else 'default', selection.value_type(), - 'usage: %s' % (selection.manual_entry().arg_usage if selection.manual_entry() else '') + 'usage: %s' % (selection.manual_entry().usage if selection.manual_entry() else '') ])

value_label_width = width - 12 - len(value_attr_label) diff --git a/nyx/starter.py b/nyx/starter.py index d541730..597a1c2 100644 --- a/nyx/starter.py +++ b/nyx/starter.py @@ -16,7 +16,6 @@ import nyx import nyx.arguments import nyx.controller import nyx.util.panel -import nyx.util.tor_config import nyx.util.tracker import nyx.util.ui_tools

@@ -24,7 +23,7 @@ import stem import stem.util.log import stem.util.system

-from nyx.util import log, BASE_DIR, init_controller, msg, uses_settings +from nyx.util import log, init_controller, msg, uses_settings

@uses_settings @@ -82,7 +81,6 @@ def main(config): _setup_freebsd_chroot(controller) _notify_of_unknown_events() _clear_password() - _load_tor_config_descriptions() _use_english_subcommands() _use_unicode() _set_process_name() @@ -227,14 +225,6 @@ def _clear_password(config): config.set('tor.password', '')

-def _load_tor_config_descriptions(): - """ - Attempt to determine descriptions for tor's configuration options. - """ - - nyx.util.tor_config.load_configuration_descriptions(BASE_DIR) - - def _use_english_subcommands(): """ Make subcommands we run (ps, netstat, etc) provide us with English results. diff --git a/nyx/util/tor_config.py b/nyx/util/tor_config.py index 6bb68b3..dc39395 100644 --- a/nyx/util/tor_config.py +++ b/nyx/util/tor_config.py @@ -2,7 +2,6 @@ Helper functions for working with tor's configuration file. """

-import codecs import os import time import socket @@ -14,31 +13,8 @@ from nyx.util import tor_controller, ui_tools

from stem.util import conf, enum, log, str_tools, system

-# filename used for cached tor config descriptions - -CONFIG_DESC_FILENAME = 'torConfigDesc.txt' - -# messages related to loading the tor configuration descriptions - -DESC_LOAD_SUCCESS_MSG = "Loaded configuration descriptions from '%s' (runtime: %0.3f)" -DESC_LOAD_FAILED_MSG = 'Unable to load configuration descriptions (%s)' -DESC_INTERNAL_LOAD_SUCCESS_MSG = 'Falling back to descriptions for Tor %s' -DESC_INTERNAL_LOAD_FAILED_MSG = "Unable to load fallback descriptions. Categories and help for Tor's configuration options won't be available. (%s)" -DESC_READ_MAN_SUCCESS_MSG = "Read descriptions for tor's configuration options from its man page (runtime %0.3f)" -DESC_READ_MAN_FAILED_MSG = "Unable to get the descriptions of Tor's configuration options from its man page (%s)" -DESC_SAVE_SUCCESS_MSG = "Saved configuration descriptions to '%s' (runtime: %0.3f)" -DESC_SAVE_FAILED_MSG = 'Unable to save configuration descriptions (%s)' - - -def conf_handler(key, value): - if key == 'torrc.important': - # stores lowercase entries to drop case sensitivity - return [entry.lower() for entry in value] - - CONFIG = conf.config_dict('nyx', { 'features.torrc.validate': True, - 'torrc.important': [], 'torrc.alias': {}, 'torrc.units.size.b': [], 'torrc.units.size.kb': [], @@ -54,16 +30,13 @@ CONFIG = conf.config_dict('nyx', { 'features.config.descriptions.enabled': True, 'features.config.descriptions.persist': True, 'tor.chroot': '', -}, conf_handler) +})

def general_conf_handler(config, key): value = config.get(key)

- if key.startswith('torrc.summary.'): - # we'll look for summary keys with a lowercase config name - CONFIG[key.lower()] = value - elif key.startswith('torrc.units.') and value: + if key.startswith('torrc.units.') and value: # all the torrc.units.* values are comma separated lists return [entry.strip() for entry in value[0].split(',')]

@@ -84,38 +57,9 @@ TIME_MULT = {'sec': 1, 'min': 60, 'hour': 3600, 'day': 86400, 'week': 604800}

ValidationError = enum.Enum('DUPLICATE', 'MISMATCH', 'MISSING', 'IS_DEFAULT')

-# descriptions of tor's configuration options fetched from its man page - -CONFIG_DESCRIPTIONS_LOCK = threading.RLock() -CONFIG_DESCRIPTIONS = {} - -# categories for tor configuration options - -Category = enum.Enum('GENERAL', 'CLIENT', 'RELAY', 'DIRECTORY', 'AUTHORITY', 'HIDDEN_SERVICE', 'TESTING', 'UNKNOWN') - TORRC = None # singleton torrc instance -MAN_OPT_INDENT = 7 # indentation before options in the man page -MAN_EX_INDENT = 15 # indentation used for man page examples -PERSIST_ENTRY_DIVIDER = '-' * 80 + '\n' # splits config entries when saving to a file MULTILINE_PARAM = None # cached multiline parameters (lazily loaded)

-# torrc options that bind to ports - -PORT_OPT = ('SocksPort', 'ORPort', 'DirPort', 'ControlPort', 'TransPort') - - -class ManPageEntry: - """ - Information provided about a tor configuration option in its man page entry. - """ - - def __init__(self, option, index, category, arg_usage, description): - self.option = option - self.index = index - self.category = category - self.arg_usage = arg_usage - self.description = description -

def get_torrc(): """ @@ -131,280 +75,6 @@ def get_torrc(): return TORRC

-def load_option_descriptions(load_path = None, check_version = True): - """ - Fetches and parses descriptions for tor's configuration options from its man - page. This can be a somewhat lengthy call, and raises an IOError if issues - occure. When successful loading from a file this returns the version for the - contents loaded. - - If available, this can load the configuration descriptions from a file where - they were previously persisted to cut down on the load time (latency for this - is around 200ms). - - Arguments: - load_path - if set, this attempts to fetch the configuration - descriptions from the given path instead of the man page - check_version - discards the results if true and tor's version doens't - match the cached descriptors, otherwise accepts anyway - """ - - with CONFIG_DESCRIPTIONS_LOCK: - CONFIG_DESCRIPTIONS.clear() - - raised_exc = None - loaded_version = '' - - try: - if load_path: - # Input file is expected to be of the form: - # <option> - # <arg description> - # <description, possibly multiple lines> - # <PERSIST_ENTRY_DIVIDER> - input_file = open(load_path, 'r') - input_file_contents = input_file.readlines() - input_file.close() - - try: - version_line = input_file_contents.pop(0).rstrip() - - if version_line.startswith('Tor Version '): - file_version = version_line[12:] - loaded_version = file_version - tor_version = tor_controller().get_info('version', '') - - if check_version and file_version != tor_version: - msg = "wrong version, tor is %s but the file's from %s" % (tor_version, file_version) - raise IOError(msg) - else: - raise IOError('unable to parse version') - - while input_file_contents: - # gets category enum, failing if it doesn't exist - category = input_file_contents.pop(0).rstrip() - - if category not in Category: - base_msg = "invalid category in input file: '%s'" - raise IOError(base_msg % category) - - # gets the position in the man page - index_arg, index_str = -1, input_file_contents.pop(0).rstrip() - - if index_str.startswith('index: '): - index_str = index_str[7:] - - if index_str.isdigit(): - index_arg = int(index_str) - else: - raise IOError('non-numeric index value: %s' % index_str) - else: - raise IOError('malformed index argument: %s' % index_str) - - option = input_file_contents.pop(0).rstrip() - argument = input_file_contents.pop(0).rstrip() - - description, loaded_line = '', input_file_contents.pop(0) - - while loaded_line != PERSIST_ENTRY_DIVIDER: - description += loaded_line - - if input_file_contents: - loaded_line = input_file_contents.pop(0) - else: - break - - CONFIG_DESCRIPTIONS[option.lower()] = ManPageEntry(option, index_arg, category, argument, description.rstrip()) - except IndexError: - CONFIG_DESCRIPTIONS.clear() - raise IOError('input file format is invalid') - else: - man_call_results = system.call('man tor', None) - - if not man_call_results: - raise IOError('man page not found') - - # Fetches all options available with this tor instance. This isn't - # vital, and the valid_options are left empty if the call fails. - - controller, valid_options = tor_controller(), [] - config_option_query = controller.get_info('config/names', None) - - if config_option_query: - for line in config_option_query.strip().split('\n'): - valid_options.append(line[:line.find(' ')].lower()) - - option_count, last_option, last_arg = 0, None, None - last_category, last_description = Category.GENERAL, '' - - for line in man_call_results: - line = codecs.latin_1_encode(line, 'replace')[0] - line = ui_tools.get_printable(line) - stripped_line = line.strip() - - # we have content, but an indent less than an option (ignore line) - # if stripped_line and not line.startswith(' ' * MAN_OPT_INDENT): continue - - # line starts with an indent equivilant to a new config option - - is_opt_indent = line.startswith(' ' * MAN_OPT_INDENT) and line[MAN_OPT_INDENT] != ' ' - - is_category_line = not line.startswith(' ') and 'OPTIONS' in line - - # if this is a category header or a new option, add an entry using the - # buffered results - - if is_opt_indent or is_category_line: - # Filters the line based on if the option is recognized by tor or - # not. This isn't necessary for nyx, so if unable to make the check - # then we skip filtering (no loss, the map will just have some extra - # noise). - - stripped_description = last_description.strip() - - if last_option and (not valid_options or last_option.lower() in valid_options): - CONFIG_DESCRIPTIONS[last_option.lower()] = ManPageEntry(last_option, option_count, last_category, last_arg, stripped_description) - option_count += 1 - - last_description = '' - - # parses the option and argument - - line = line.strip() - div_index = line.find(' ') - - if div_index != -1: - last_option, last_arg = line[:div_index], line[div_index + 1:] - - # if this is a category header then switch it - - if is_category_line: - if line.startswith('OPTIONS'): - last_category = Category.GENERAL - elif line.startswith('CLIENT'): - last_category = Category.CLIENT - elif line.startswith('SERVER'): - last_category = Category.RELAY - elif line.startswith('DIRECTORY SERVER'): - last_category = Category.DIRECTORY - elif line.startswith('DIRECTORY AUTHORITY SERVER'): - last_category = Category.AUTHORITY - elif line.startswith('HIDDEN SERVICE'): - last_category = Category.HIDDEN_SERVICE - elif line.startswith('TESTING NETWORK'): - last_category = Category.TESTING - else: - log.notice('Unrecognized category in the man page: %s' % line.strip()) - else: - # Appends the text to the running description. Empty lines and lines - # starting with a specific indentation are used for formatting, for - # instance the ExitPolicy and TestingTorNetwork entries. - - if last_description and last_description[-1] != '\n': - last_description += ' ' - - if not stripped_line: - last_description += '\n\n' - elif line.startswith(' ' * MAN_EX_INDENT): - last_description += ' %s\n' % stripped_line - else: - last_description += stripped_line - except IOError as exc: - raised_exc = exc - - if raised_exc: - raise raised_exc - else: - return loaded_version - - -def save_option_descriptions(path): - """ - Preserves the current configuration descriptors to the given path. This - raises an IOError or OSError if unable to do so. - - Arguments: - path - location to persist configuration descriptors - """ - - # make dir if the path doesn't already exist - - base_dir = os.path.dirname(path) - - if not os.path.exists(base_dir): - os.makedirs(base_dir) - - output_file = open(path, 'w') - - with CONFIG_DESCRIPTIONS_LOCK: - sorted_options = CONFIG_DESCRIPTIONS.keys() - sorted_options.sort() - - tor_version = tor_controller().get_info('version', '') - output_file.write('Tor Version %s\n' % tor_version) - - for i in range(len(sorted_options)): - man_entry = get_config_description(sorted_options[i]) - output_file.write('%s\nindex: %i\n%s\n%s\n%s\n' % (man_entry.category, man_entry.index, man_entry.option, man_entry.arg_usage, man_entry.description)) - - if i != len(sorted_options) - 1: - output_file.write(PERSIST_ENTRY_DIVIDER) - - output_file.close() - - -def get_config_summary(option): - """ - Provides a short summary description of the configuration option. If none is - known then this proivdes None. - - Arguments: - option - tor config option - """ - - return CONFIG.get('torrc.summary.%s' % option.lower()) - - -def is_important(option): - """ - Provides True if the option has the 'important' flag in the configuration, - False otherwise. - - Arguments: - option - tor config option - """ - - return option.lower() in CONFIG['torrc.important'] - - -def get_config_description(option): - """ - Provides ManPageEntry instances populated with information fetched from the - tor man page. This provides None if no such option has been loaded. If the - man page is in the process of being loaded then this call blocks until it - finishes. - - Arguments: - option - tor config option - """ - - with CONFIG_DESCRIPTIONS_LOCK: - if option.lower() in CONFIG_DESCRIPTIONS: - return CONFIG_DESCRIPTIONS[option.lower()] - else: - return None - - -def get_config_options(): - """ - Provides the configuration options from the loaded man page. This is an empty - list if no man page has been loaded. - """ - - with CONFIG_DESCRIPTIONS_LOCK: - return [CONFIG_DESCRIPTIONS[opt].option for opt in CONFIG_DESCRIPTIONS] - - def get_config_location(): """ Provides the location of the torrc, raising an IOError with the reason if the @@ -1016,78 +686,3 @@ class Torrc(): msg += ', '.join(missing_options)

log.warn(msg) - - -def load_configuration_descriptions(path_prefix): - """ - Attempts to load descriptions for tor's configuration options, fetching them - from the man page and persisting them to a file to speed future startups. - """ - - # It is important that this is loaded before entering the curses context, - # otherwise the man call pegs the cpu for around a minute (I'm not sure - # why... curses must mess the terminal in a way that's important to man). - - if CONFIG['features.config.descriptions.enabled']: - is_config_descriptions_loaded = False - - # determines the path where cached descriptions should be persisted (left - # undefined if caching is disabled) - - descriptor_path = None - - if CONFIG['features.config.descriptions.persist']: - data_dir = CONFIG['startup.data_directory'] - - if not data_dir.endswith('/'): - data_dir += '/' - - descriptor_path = os.path.expanduser(data_dir + 'cache/') + CONFIG_DESC_FILENAME - - # attempts to load configuration descriptions cached in the data directory - - if descriptor_path: - try: - load_start_time = time.time() - load_option_descriptions(descriptor_path) - is_config_descriptions_loaded = True - - log.info(DESC_LOAD_SUCCESS_MSG % (descriptor_path, time.time() - load_start_time)) - except IOError as exc: - log.info(DESC_LOAD_FAILED_MSG % exc.strerror) - - # fetches configuration options from the man page - - if not is_config_descriptions_loaded: - try: - load_start_time = time.time() - load_option_descriptions() - is_config_descriptions_loaded = True - - log.info(DESC_READ_MAN_SUCCESS_MSG % (time.time() - load_start_time)) - except IOError as exc: - log.notice(DESC_READ_MAN_FAILED_MSG % exc.strerror) - - # persists configuration descriptions - - if is_config_descriptions_loaded and descriptor_path: - try: - load_start_time = time.time() - save_option_descriptions(descriptor_path) - log.info(DESC_SAVE_SUCCESS_MSG % (descriptor_path, time.time() - load_start_time)) - except IOError as exc: - log.notice(DESC_SAVE_FAILED_MSG % exc.strerror) - except OSError as exc: - log.notice(DESC_SAVE_FAILED_MSG % exc) - - # finally fall back to the cached descriptors provided with nyx (this is - # often the case for tbb and manual builds) - - if not is_config_descriptions_loaded: - try: - load_start_time = time.time() - loaded_version = load_option_descriptions('%sresources/%s' % (path_prefix, CONFIG_DESC_FILENAME), False) - is_config_descriptions_loaded = True - log.notice(DESC_INTERNAL_LOAD_SUCCESS_MSG % loaded_version) - except IOError as exc: - log.error(DESC_INTERNAL_LOAD_FAILED_MSG % exc.strerror)