[tor-commits] [stem/master] Function to download tor's latest man page

commit f581cf22a4760e0a8cf92f383f4cca32ccc0f28e Author: Damian Johnson <atagar@torproject.org> Date: Mon Nov 23 14:33:53 2015 -0800 Function to download tor's latest man page Adding a function to fetch tor's latest asciidoc manual via gitweb, then compile it into a man page. Cringing? Don't worry. It's well documented as being best-effort and handy not only for our users but our tests as well... * Nyx will want this so it can learn descriptions for new tor options without updating. * Our tests use this so we don't need to bundle a copy of the man page to exercise our parser. * Integ testing to give tor some assurance its asciidoc can still be compiled and looks right. --- stem/descriptor/reader.py | 1 + stem/descriptor/remote.py | 5 +- stem/manual.cfg | 352 ----------------------------------------- stem/manual.py | 116 +++++++++++++- stem/settings.cfg | 352 +++++++++++++++++++++++++++++++++++++++++ test/integ/manual.py | 213 +++++++++++++++++++++++++ test/integ/tor.1_with_unknown | 51 ++++++ test/settings.cfg | 1 + test/unit/manual.py | 263 ++++++++++++------------------ test/unit/tor.1_with_unknown | 51 ------ 10 files changed, 835 insertions(+), 570 deletions(-) diff --git a/stem/descriptor/reader.py b/stem/descriptor/reader.py index a96406d..2877975 100644 --- a/stem/descriptor/reader.py +++ b/stem/descriptor/reader.py @@ -218,6 +218,7 @@ def save_processed_files(path, processed_files): """ # makes the parent directory if it doesn't already exist + try: path_dir = os.path.dirname(path) diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index 095f125..b2b5b8e 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -80,9 +80,10 @@ import time import zlib try: - import urllib.request as urllib + # account for urllib's change between python 2.x and 3.x + import urllib.request as urllib except ImportError: - import urllib2 as urllib + import urllib2 as urllib import stem.descriptor diff --git a/stem/manual.cfg b/stem/manual.cfg deleted file mode 100644 index 512501e..0000000 --- a/stem/manual.cfg +++ /dev/null @@ -1,352 +0,0 @@ -################################################################################ -# -# Information related to tor configuration options... -# -# * manual.important Most commonly used configuration options. -# * manual.summary Short summary describing the option. -# -################################################################################ - -manual.important BandwidthRate -manual.important BandwidthBurst -manual.important RelayBandwidthRate -manual.important RelayBandwidthBurst -manual.important ControlPort -manual.important HashedControlPassword -manual.important CookieAuthentication -manual.important DataDirectory -manual.important Log -manual.important RunAsDaemon -manual.important User - -manual.important Bridge -manual.important ExcludeNodes -manual.important MaxCircuitDirtiness -manual.important SOCKSPort -manual.important UseBridges - -manual.important BridgeRelay -manual.important ContactInfo -manual.important ExitPolicy -manual.important MyFamily -manual.important Nickname -manual.important ORPort -manual.important PortForwarding -manual.important AccountingMax -manual.important AccountingStart - -manual.important DirPortFrontPage -manual.important DirPort - -manual.important HiddenServiceDir -manual.important HiddenServicePort - -# General Config Options - -manual.summary.BandwidthRate Average bandwidth usage limit -manual.summary.BandwidthBurst Maximum bandwidth usage limit -manual.summary.MaxAdvertisedBandwidth Limit for the bandwidth we advertise as being available for relaying -manual.summary.RelayBandwidthRate Average bandwidth usage limit for relaying -manual.summary.RelayBandwidthBurst Maximum bandwidth usage limit for relaying -manual.summary.PerConnBWRate Average relayed bandwidth limit per connection -manual.summary.PerConnBWBurst Maximum relayed bandwidth limit per connection -manual.summary.ClientTransportPlugin Proxy when establishing bridge connections -manual.summary.ServerTransportPlugin Proxy when servicing bridge connections -manual.summary.ServerTransportListenAddr Endpoint for bridge's pluggable transport proxy -manual.summary.ServerTransportOptions Additional arguments for bridge's proxy -manual.summary.ExtORPort Endpoint for extended ORPort connections -manual.summary.ExtORPortCookieAuthFile Location of the ExtORPort's authentication cookie -manual.summary.ExtORPortCookieAuthFileGroupReadable Group read permissions for the ExtORPort's authentication cookie -manual.summary.ConnLimit Minimum number of file descriptors for Tor to start -manual.summary.DisableNetwork Don't accept non-controller connections -manual.summary.ConstrainedSockets Shrinks sockets to ConstrainedSockSize -manual.summary.ConstrainedSockSize Limit for the received and transmit buffers of sockets -manual.summary.ControlPort Port providing access to tor controllers (nyx, vidalia, etc) -manual.summary.ControlListenAddress Address providing controller access -manual.summary.ControlSocket Socket providing controller access -manual.summary.ControlSocketsGroupWritable Group read permissions for the control socket -manual.summary.HashedControlPassword Hash of the password for authenticating to the control port -manual.summary.CookieAuthentication If set, authenticates controllers via a cookie -manual.summary.CookieAuthFile Location of the authentication cookie -manual.summary.CookieAuthFileGroupReadable Group read permissions for the authentication cookie -manual.summary.ControlPortWriteToFile Path for a file tor writes containing its control port -manual.summary.ControlPortFileGroupReadable Group read permissions for the control port file -manual.summary.DataDirectory Location for storing runtime data (state, keys, etc) -manual.summary.FallbackDir Fallback when unable to retrieve descriptor information -manual.summary.DirAuthority Alternative directory authorities -manual.summary.DirAuthorityFallbackRate Rate at which to use fallback directory -manual.summary.AlternateDirAuthority Alternative directory authorities (consensus only) -manual.summary.AlternateBridgeAuthority Alternative directory authorities (bridges only) -manual.summary.DisableAllSwap Locks all allocated memory so they can't be paged out -manual.summary.DisableDebuggerAttachment Limit information applications can retrieve about the process -manual.summary.FetchDirInfoEarly Keeps consensus information up to date, even if unnecessary -manual.summary.FetchDirInfoExtraEarly Updates consensus information when it's first available -manual.summary.FetchHidServDescriptors Toggles if hidden service descriptors are fetched automatically or not -manual.summary.FetchServerDescriptors Toggles if the consensus is fetched automatically or not -manual.summary.FetchUselessDescriptors Toggles if relay descriptors are fetched when they aren't strictly necessary -manual.summary.HTTPProxy HTTP proxy for connecting to tor -manual.summary.HTTPProxyAuthenticator Authentication credentials for HTTPProxy -manual.summary.HTTPSProxy SSL proxy for connecting to tor -manual.summary.HTTPSProxyAuthenticator Authentication credentials for HTTPSProxy -manual.summary.Sandbox Run within a syscall sandbox -manual.summary.Socks4Proxy SOCKS 4 proxy for connecting to tor -manual.summary.Socks5Proxy SOCKS 5 for connecting to tor -manual.summary.Socks5ProxyUsername Username for connecting to the Socks5Proxy -manual.summary.Socks5ProxyPassword Password for connecting to the Socks5Proxy -manual.summary.SocksSocketsGroupWritable Group write permissions for the socks socket -manual.summary.KeepalivePeriod Rate at which to send keepalive packets -manual.summary.Log Runlevels and location for tor logging -manual.summary.LogMessageDomains Includes a domain when logging messages -manual.summary.OutboundBindAddress Sets the IP used for connecting to tor -manual.summary.PidFile Path for a file tor writes containing its process id -manual.summary.ProtocolWarnings Toggles if protocol errors give warnings or not -manual.summary.PredictedPortsRelevanceTime Duration to ensure circuits for previously used ports remain available -manual.summary.RunAsDaemon Toggles if tor runs as a daemon process -manual.summary.LogTimeGranularity limits granularity of log message timestamps -manual.summary.TruncateLogFile Overwrites log file rather than appending when restarted -manual.summary.SyslogIdentityTag Tag logs appended to the syslog as being from tor -manual.summary.SafeLogging Toggles if logs are scrubbed of sensitive information -manual.summary.User UID for the process when started -manual.summary.HardwareAccel Toggles if tor attempts to use hardware acceleration -manual.summary.AccelName OpenSSL engine name for crypto acceleration -manual.summary.AccelDir Crypto acceleration library path -manual.summary.AvoidDiskWrites Toggles if tor avoids frequently writing to disk -manual.summary.CircuitPriorityHalflife Overwrite method for prioritizing traffic among relayed connections -manual.summary.DisableIOCP Disables use of the Windows IOCP networking API -manual.summary.UserspaceIOCPBuffers Disable kernel-space IOCP TCP buffers -manual.summary.UseFilteringSSLBufferevents Use SSL for a chain of bufferevents -manual.summary.CountPrivateBandwidth Applies rate limiting to private IP addresses - -# Client Config Options - -manual.summary.AllowInvalidNodes Permits use of relays flagged as invalid by authorities -manual.summary.ExcludeSingleHopRelays Permits use of relays that allow single hop connections -manual.summary.Bridge Available bridges -manual.summary.LearnCircuitBuildTimeout Toggles adaptive timeouts for circuit creation -manual.summary.CircuitBuildTimeout Initial timeout for circuit creation -manual.summary.CircuitIdleTimeout Timeout for closing circuits that have never been used -manual.summary.CircuitStreamTimeout Timeout for shifting streams among circuits -manual.summary.ClientOnly Ensures that we aren't used as a relay or directory mirror -manual.summary.ExcludeNodes Relays or locales never to be used in circuits -manual.summary.ExcludeExitNodes Relays or locales never to be used for exits -manual.summary.GeoIPExcludeUnknown Don't use relays with an unknown locale in circuits -manual.summary.ExitNodes Preferred final hop for circuits -manual.summary.EntryNodes Preferred first hops for circuits -manual.summary.StrictNodes Never uses notes outside of Entry/ExitNodes -manual.summary.FascistFirewall Only make outbound connections on FirewallPorts -manual.summary.FirewallPorts Ports used by FascistFirewall -manual.summary.HidServAuth Authentication credentials for connecting to a hidden service -manual.summary.CloseHSClientCircuitsImmediatelyOnTimeout Close hidden service circuits that timeout -manual.summary.CloseHSServiceRendCircuitsImmediatelyOnTimeout Close hidden service rendezvous circuits that timeout -manual.summary.ReachableAddresses Rules for bypassing the local firewall -manual.summary.ReachableDirAddresses Rules for bypassing the local firewall (directory fetches) -manual.summary.ReachableORAddresses Rules for bypassing the local firewall (OR connections) -manual.summary.LongLivedPorts Ports requiring highly reliable relays -manual.summary.MapAddress Alias mappings for address requests -manual.summary.NewCircuitPeriod Period for considering the creation of new circuits -manual.summary.MaxCircuitDirtiness Duration for reusing constructed circuits -manual.summary.MaxClientCircuitsPending Number of circuits that can be in construction at once -manual.summary.NodeFamily Define relays as belonging to a family -manual.summary.EnforceDistinctSubnets Prevent use of multiple relays from the same subnet on a circuit -manual.summary.SOCKSPort Port for using tor as a Socks proxy -manual.summary.SOCKSListenAddress Address from which Socks connections can be made -manual.summary.SocksPolicy Access policy for the pocks port -manual.summary.SocksTimeout Time until idle or unestablished socks connections are closed -manual.summary.TokenBucketRefillInterval Frequency at which exhausted connections are checked for new traffic -manual.summary.TrackHostExits Maintains use of the same exit whenever connecting to this destination -manual.summary.TrackHostExitsExpire Time until use of an exit for tracking expires -manual.summary.UpdateBridgesFromAuthority Toggles fetching bridge descriptors from the authorities -manual.summary.UseBridges Make use of configured bridges -manual.summary.UseEntryGuards Use guard relays for first hop -manual.summary.UseEntryGuardsAsDirGuards Retrieve descriptors via guards when able -manual.summary.GuardfractionFile File containing information with duration of our guards -manual.summary.UseGuardFraction Take guardfraction into account for path selection -manual.summary.NumEntryGuards Pool size of guard relays we'll select from -manual.summary.NumDirectoryGuards Pool size of directory guards we'll select from -manual.summary.GuardLifetime Minimum time to keep entry guards -manual.summary.SafeSocks Toggles rejecting unsafe variants of the socks protocol -manual.summary.TestSocks Provide notices for if socks connections are of the safe or unsafe variants -manual.summary.WarnUnsafeSocks Toggle warning of unsafe socks connection -manual.summary.VirtualAddrNetworkIPv4 IPv4 address range to use when needing a virtual address -manual.summary.VirtualAddrNetworkIPv6 IPv6 address range to use when needing a virtual address -manual.summary.AllowNonRFC953Hostnames Toggles blocking invalid characters in hostname resolution -manual.summary.AllowDotExit Toggles allowing exit notation in addresses -manual.summary.FastFirstHopPK Toggle public key usage for the first hop -manual.summary.TransPort Port for transparent proxying if the OS supports it -manual.summary.TransListenAddress Address from which transparent proxy connections can be made -manual.summary.TransProxyType Proxy type to be used -manual.summary.NATDPort Port for forwarding ipfw NATD connections -manual.summary.NATDListenAddress Address from which NATD forwarded connections can be made -manual.summary.AutomapHostsOnResolve Map addresses ending with special suffixes to virtual addresses -manual.summary.AutomapHostsSuffixes Address suffixes recognized by AutomapHostsOnResolve -manual.summary.DNSPort Port from which DNS responses are fetched instead of tor -manual.summary.DNSListenAddress Address for performing DNS resolution -manual.summary.ClientDNSRejectInternalAddresses Ignores DNS responses for internal addresses -manual.summary.ClientRejectInternalAddresses Disables use of Tor for internal connections -manual.summary.DownloadExtraInfo Toggles fetching of extra information about relays -manual.summary.WarnPlaintextPorts Toggles warnings for using risky ports -manual.summary.RejectPlaintextPorts Prevents connections on risky ports -manual.summary.AllowSingleHopCircuits Makes use of single hop exits if able -manual.summary.OptimisticData Use exits without confirmation that prior connections succeeded -manual.summary.Tor2webMode Establish non-anonymous hidden service connections -manual.summary.Tor2webRendezvousPoints Rendezvous points to use for hidden services when in Tor2webMode -manual.summary.UseMicrodescriptors Retrieve microdescriptors rather than server descriptors -manual.summary.UseNTorHandshake Use ntor for establishing circuits with relays -manual.summary.PathBiasCircThreshold Number of circuits through a guard before applying bias checks -manual.summary.PathBiasNoticeRate Fraction of circuits that must succeed before logging a notice -manual.summary.PathBiasWarnRate Fraction of circuits that must succeed before logging a warning -manual.summary.PathBiasExtremeRate Fraction of circuits that must succeed before logging an error -manual.summary.PathBiasDropGuards Drop guards failing to establish circuits -manual.summary.PathBiasScaleThreshold Circuits through a guard before scaling past observations down -manual.summary.PathBiasUseThreshold Number of streams through a circuit before applying bias checks -manual.summary.PathBiasNoticeUseRate Fraction of streams that must succeed before logging a notice -manual.summary.PathBiasExtremeUseRate Fraction of streams that must succeed before logging an error -manual.summary.PathBiasScaleUseThreshold Streams through a circuit before scaling past observations down -manual.summary.ClientUseIPv6 Allow IPv6 connections to guards -manual.summary.ClientPreferIPv6ORPort Prefer a guard's IPv6 rather than IPv4 endpoint -manual.summary.PathsNeededToBuildCircuits Portion of relays to require information for before making circuits - -# Server Config Options - -manual.summary.Address Overwrites address others will use to reach this relay -manual.summary.AllowSingleHopExits Toggles permitting use of this relay as a single hop proxy -manual.summary.AssumeReachable Skips reachability test at startup -manual.summary.BridgeRelay Act as a bridge -manual.summary.ContactInfo Contact information for this relay -manual.summary.ExitRelay Allow relaying of exit traffic -manual.summary.ExitPolicy Traffic destinations that can exit from this relay -manual.summary.ExitPolicyRejectPrivate Prevent exiting connection on the local network -manual.summary.IPv6Exit Allow clients to use us for IPv6 traffic -manual.summary.MaxOnionQueueDelay Duration to reject new onionskins if we have more than we can process -manual.summary.MyFamily Other relays this operator administers -manual.summary.Nickname Identifier for this relay -manual.summary.NumCPUs Number of processes spawned for decryption -manual.summary.ORPort Port used to accept relay traffic -manual.summary.ORListenAddress Address for relay connections -manual.summary.PortForwarding Use UPnP or NAT-PMP if needed to relay -manual.summary.PortForwardingHelper Executable for configuring port forwarding -manual.summary.PublishServerDescriptor Types of descriptors published -manual.summary.ShutdownWaitLength Delay before quitting after receiving a SIGINT signal -manual.summary.SSLKeyLifetime Lifetime for our link certificate -manual.summary.HeartbeatPeriod Rate at which an INFO level heartbeat message is sent -manual.summary.AccountingMax Amount of traffic before hibernating -manual.summary.AccountingRule Method to determine when the accounting limit is reached -manual.summary.AccountingStart Duration of an accounting period -manual.summary.RefuseUnknownExits Prevents relays not in the consensus from using us as an exit -manual.summary.ServerDNSResolvConfFile Overriding resolver config for DNS queries we provide -manual.summary.ServerDNSAllowBrokenConfig Toggles if we persist despite configuration parsing errors or not -manual.summary.ServerDNSSearchDomains Toggles if our DNS queries search for addresses in the local domain -manual.summary.ServerDNSDetectHijacking Toggles testing for DNS hijacking -manual.summary.ServerDNSTestAddresses Addresses to test to see if valid DNS queries are being hijacked -manual.summary.ServerDNSAllowNonRFC953Hostnames Toggles if we reject DNS queries with invalid characters -manual.summary.BridgeRecordUsageByCountry Tracks geoip information on bridge usage -manual.summary.ServerDNSRandomizeCase Toggles DNS query case randomization -manual.summary.GeoIPFile Path to file containing IPv4 geoip information -manual.summary.GeoIPv6File Path to file containing IPv6 geoip information -manual.summary.TLSECGroup EC group for incoming SSL connections -manual.summary.CellStatistics Toggles storing circuit queue duration to disk -manual.summary.DirReqStatistics Toggles storing network status counts and performance to disk -manual.summary.EntryStatistics Toggles storing client connection counts to disk -manual.summary.ExitPortStatistics Toggles storing traffic and port usage data to disk -manual.summary.ConnDirectionStatistics Toggles storing connection use to disk -manual.summary.HiddenServiceStatistics Toggles storing hidden service stats to disk -manual.summary.ExtraInfoStatistics Publishes statistic data in the extra-info documents -manual.summary.ExtendAllowPrivateAddresses Allow circuits to be extended to the local network -manual.summary.MaxMemInQueues Threshold at which tor will terminate circuits to avoid running out of memory -manual.summary.SigningKeyLifetime Duration the Ed25519 signing key is valid for -manual.summary.OfflineMasterKey Don't generate the master secret key - -# Directory Server Options - -manual.summary.DirPortFrontPage Publish this html file on the DirPort -manual.summary.HidServDirectoryV2 Toggles accepting version 2 hidden service descriptors -manual.summary.DirPort Port for directory connections -manual.summary.DirListenAddress Address the directory service is bound to -manual.summary.DirPolicy Access policy for the DirPort - -# Directory Authority Server Options - -manual.summary.AuthoritativeDirectory Act as a directory authority -manual.summary.V3AuthoritativeDirectory Generates a version 3 consensus -manual.summary.VersioningAuthoritativeDirectory Provides opinions on recommended versions of tor -manual.summary.RecommendedVersions Suggested versions of tor -manual.summary.RecommendedPackageVersions Suggested versions of applications other than tor -manual.summary.RecommendedClientVersions Tor versions believed to be safe for clients -manual.summary.BridgeAuthoritativeDir Acts as a bridge authority -manual.summary.MinUptimeHidServDirectoryV2 Required uptime before accepting hidden service directory -manual.summary.RecommendedServerVersions Tor versions believed to be safe for relays -manual.summary.ConsensusParams Params entry of the networkstatus vote -manual.summary.DirAllowPrivateAddresses Toggles allowing arbitrary input or non-public IPs in descriptors -manual.summary.AuthDirBadExit Relays to be flagged as bad exits -manual.summary.AuthDirInvalid Relays from which the valid flag is withheld -manual.summary.AuthDirReject Relays to be dropped from the consensus -manual.summary.AuthDirBadExitCCs Countries for which to flag all relays as bad exits -manual.summary.AuthDirInvalidCCs Countries for which the valid flag is withheld -manual.summary.AuthDirRejectCCs Countries for which relays aren't accepted into the consensus -manual.summary.AuthDirListBadExits Toggles if we provide an opinion on bad exits -manual.summary.AuthDirMaxServersPerAddr Limit on the number of relays accepted per ip -manual.summary.AuthDirMaxServersPerAuthAddr Limit on the number of relays accepted per an authority's ip -manual.summary.AuthDirFastGuarantee Advertised rate at which the Fast flag is granted -manual.summary.AuthDirGuardBWGuarantee Advertised rate necessary to be a guard -manual.summary.AuthDirPinKeys Don't accept descriptors with conflicting identity keypairs -manual.summary.BridgePassword Password for requesting bridge information -manual.summary.V3AuthVotingInterval Consensus voting interval -manual.summary.V3AuthVoteDelay Wait time to collect votes of other authorities -manual.summary.V3AuthDistDelay Wait time to collect the signatures of other authorities -manual.summary.V3AuthNIntervalsValid Number of voting intervals a consensus is valid for -manual.summary.V3BandwidthsFile Path to a file containing measured relay bandwidths -manual.summary.V3AuthUseLegacyKey Signs consensus with both the current and legacy keys -manual.summary.RephistTrackTime Discards old, unchanged reliability information -manual.summary.VoteOnHidServDirectoriesV2 Determines if the authority votes on hidden service directories -manual.summary.AuthDirHasIPv6Connectivity Descriptors can be retrieved over the authority's IPv6 ORPort -manual.summary.MinMeasuredBWsForAuthToIgnoreAdvertised Total measured value before advertised bandwidths are treated as unreliable - -# Hidden Service Options - -manual.summary.HiddenServiceDir Directory contents for the hidden service -manual.summary.HiddenServicePort Port the hidden service is provided on -manual.summary.PublishHidServDescriptors Toggles automated publishing of the hidden service to the rendezvous directory -manual.summary.HiddenServiceVersion Version for published hidden service descriptors -manual.summary.HiddenServiceAuthorizeClient Restricts access to the hidden service -manual.summary.HiddenServiceAllowUnknownPorts Allow rendezvous circuits on unrecognized ports -manual.summary.HiddenServiceMaxStreams Maximum streams per rendezvous circuit -manual.summary.HiddenServiceMaxStreamsCloseCircuit Closes rendezvous circuits that exceed the maximum number of streams -manual.summary.RendPostPeriod Period at which the rendezvous service descriptors are refreshed -manual.summary.HiddenServiceDirGroupReadable Group read permissions for the hidden service directory -manual.summary.HiddenServiceNumIntroductionPoints Number of introduction points the hidden service will have - -# Testing Network Options - -manual.summary.TestingTorNetwork Overrides other options to be a testing network -manual.summary.TestingV3AuthInitialVotingInterval Overrides V3AuthVotingInterval for the first consensus -manual.summary.TestingV3AuthInitialVoteDelay Overrides TestingV3AuthInitialVoteDelay for the first consensus -manual.summary.TestingV3AuthInitialDistDelay Overrides TestingV3AuthInitialDistDelay for the first consensus -manual.summary.TestingV3AuthVotingStartOffset Offset for the point at which the authority votes -manual.summary.TestingAuthDirTimeToLearnReachability Delay until opinions are given about which relays are running or not -manual.summary.TestingEstimatedDescriptorPropagationTime Delay before clients attempt to fetch descriptors from directory caches -manual.summary.TestingMinFastFlagThreshold Minimum value for the Fast flag -manual.summary.TestingServerDownloadSchedule Schedule for when we should download resources as a relay -manual.summary.TestingClientDownloadSchedule Schedule for when we should download resources as a client -manual.summary.TestingServerConsensusDownloadSchedule Schedule for when we should download the consensus as a relay -manual.summary.TestingClientConsensusDownloadSchedule Schedule for when we should download the consensus as a client -manual.summary.TestingBridgeDownloadSchedule Schedule for when we should download bridge descriptors -manual.summary.TestingClientMaxIntervalWithoutRequest Maximum time to wait to batch requests for missing descriptors -manual.summary.TestingDirConnectionMaxStall Duration to let directory connections stall before timing out -manual.summary.TestingConsensusMaxDownloadTries Retries for downloading the consensus -manual.summary.TestingDescriptorMaxDownloadTries Retries for downloading server descriptors -manual.summary.TestingMicrodescMaxDownloadTries Retries for downloading microdescriptors -manual.summary.TestingCertMaxDownloadTries Retries for downloading authority certificates -manual.summary.TestingDirAuthVoteExit Relays to give the Exit flag to -manual.summary.TestingDirAuthVoteExitIsStrict Only grant the Exit flag to relays listed by TestingDirAuthVoteExit -manual.summary.TestingDirAuthVoteGuard Relays to give the Guard flag to -manual.summary.TestingDirAuthVoteGuardIsStrict Only grant the Guard flag to relays listed by TestingDirAuthVoteGuard -manual.summary.TestingDirAuthVoteHSDir Relays to give the HSDir flag to -manual.summary.TestingDirAuthVoteHSDirIsStrict Only grant the HSDir flag to relays listed by TestingDirAuthVoteHSDir -manual.summary.TestingEnableConnBwEvent Allow controllers to request CONN_BW events -manual.summary.TestingEnableCellStatsEvent Allow controllers to request CELL_STATS events -manual.summary.TestingEnableTbEmptyEvent Allow controllers to request TB_EMPTY events -manual.summary.TestingMinExitFlagThreshold Lower bound for assigning the Exit flag -manual.summary.TestingLinkCertifetime Duration of our ed25519 certificate -manual.summary.TestingAuthKeyLifetime Duration for our ed25519 signing key -manual.summary.TestingLinkKeySlop Time before expiration that we replace our ed25519 key - diff --git a/stem/manual.py b/stem/manual.py index cf1be50..4f51d1f 100644 --- a/stem/manual.py +++ b/stem/manual.py @@ -10,15 +10,20 @@ Provides information available about Tor from `its manual :: is_important - Indicates if a configuration option is of particularly common importance. + download_man_page - Downloads tor's latest man page. Manual - Information about Tor available from its manual. - +- from_man - Retrieves manual information from its man page. + |- from_man - Retrieves manual information from its man page. + +- from_remote - Retrieves manual information remotely from tor's latest manual. .. versionadded:: 1.5.0 """ import collections import os +import shutil +import sys +import tempfile import stem.prereq import stem.util.conf @@ -38,9 +43,17 @@ try: except ImportError: from stem.util.lru_cache import lru_cache +try: + # account for urllib's change between python 2.x and 3.x + import urllib.request as urllib +except ImportError: + import urllib2 as urllib + Category = stem.util.enum.Enum('GENERAL', 'CLIENT', 'RELAY', 'DIRECTORY', 'AUTHORITY', 'HIDDEN_SERVICE', 'TESTING', 'UNKNOWN') ConfigOption = collections.namedtuple('ConfigOption', ['category', 'name', 'usage', 'summary', 'description']) +GITWEB_MANUAL_URL = 'https://gitweb.torproject.org/tor.git/plain/doc/tor.1.txt' + CATEGORY_SECTIONS = { 'GENERAL OPTIONS': Category.GENERAL, 'CLIENT OPTIONS': Category.CLIENT, @@ -55,7 +68,7 @@ CATEGORY_SECTIONS = { @lru_cache() def _config(lowercase = True): """ - Provides a dictionary for our manual.cfg. This has a couple categories... + Provides a dictionary for our settings.cfg. This has a couple categories... * manual.important (list) - configuration options considered to be important * manual.summary.* (str) - summary descriptions of config options @@ -65,7 +78,7 @@ def _config(lowercase = True): """ config = stem.util.conf.Config() - config_path = os.path.join(os.path.dirname(__file__), 'manual.cfg') + config_path = os.path.join(os.path.dirname(__file__), 'settings.cfg') try: config.load(config_path) @@ -90,6 +103,65 @@ def is_important(option): return option.lower() in _config()['manual.important'] +def download_man_page(path = None, file_handle = None, url = GITWEB_MANUAL_URL, timeout = 20): + """ + Downloads tor's latest man page from `gitweb.torproject.org + <https://gitweb.torproject.org/tor.git/plain/doc/tor.1.txt>`_. This method is + both slow and unreliable - please see the warnings on + :func:`~stem.manual.Manual.from_remote`. + + :param str path: path to save tor's man page to + :param file file_handle: file handler to save tor's man page to + :param str url: url to download tor's asciidoc manual from + :param int timeout: seconds to wait before timing out the request + + :raises: **IOError** if unable to retrieve the manual + """ + + if not path and not file_handle: + raise ValueError("Either the path or file_handle we're saving to must be provided") + elif not stem.util.system.is_available('a2x'): + raise IOError('We require a2x from asciidoc to provide a man page') + + dirpath = tempfile.mkdtemp() + asciidoc_path = os.path.join(dirpath, 'tor.1.txt') + manual_path = os.path.join(dirpath, 'tor.1') + + try: + try: + with open(asciidoc_path, 'wb') as asciidoc_file: + request = urllib.urlopen(url, timeout = timeout) + shutil.copyfileobj(request, asciidoc_file) + except: + exc = sys.exc_info()[1] + raise IOError("Unable to download tor's manual from %s to %s: %s" % (url, asciidoc_path, exc)) + + try: + stem.util.system.call('a2x -f manpage %s' % asciidoc_path) + + if not os.path.exists(manual_path): + raise OSError('no man page was generated') + except OSError as exc: + raise IOError("Unable to run 'a2x -f manpage %s': %s" % (asciidoc_path, exc)) + + if path: + try: + path_dir = os.path.dirname(path) + + if not os.path.exists(path_dir): + os.makedirs(path_dir) + + shutil.copyfile(manual_path, path) + except OSError as exc: + raise IOError(exc) + + if file_handle: + with open(manual_path) as manual_file: + shutil.copyfileobj(manual_file, file_handle) + finally: + shutil.rmtree(dirpath) + + class Manual(object): """ Parsed tor man page. Tor makes no guarantees about its man page format so @@ -129,6 +201,10 @@ class Manual(object): :param str man_path: path argument for 'man', for example you might want '/path/to/tor/doc/tor.1' to read from tor's git repository + + :returns: :class:`~stem.manual.Manual` for the system's man page + + :raises: **IOError** if unable to retrieve the manual """ try: @@ -155,6 +231,40 @@ class Manual(object): config_options, ) + @staticmethod + def from_remote(timeout = 20): + """ + Reads and parses the latest tor man page `from gitweb.torproject.org + <https://gitweb.torproject.org/tor.git/plain/doc/tor.1.txt>`_. Note that + while convenient, this reliance on GitWeb means you should alway call with + a fallback, such as... + + :: + + try: + manual = stem.manual.from_remote() + except IOError: + manual = stem.manual.from_cache() + + In addition to our GitWeb dependency this requires 'a2x' which is part of + `asciidoc <http://asciidoc.org/INSTALL.html>`_ and... isn't quick. + Personally this takes ~7.41s, breaking down for me as follows... + + * 1.67s to download tor.1.txt + * 5.57s to convert the asciidoc to a man page + * 0.17s for stem to read and parse the manual + + :param int timeout: seconds to wait before timing out the request + + :returns: latest :class:`~stem.manual.Manual` available for tor + + :raises: **IOError** if unable to retrieve the manual + """ + + with tempfile.NamedTemporaryFile() as tmp: + download_man_page(file_handle = tmp, timeout = timeout) + return Manual.from_man(tmp) + def _get_categories(content): """ diff --git a/stem/settings.cfg b/stem/settings.cfg new file mode 100644 index 0000000..512501e --- /dev/null +++ b/stem/settings.cfg @@ -0,0 +1,352 @@ +################################################################################ +# +# Information related to tor configuration options... +# +# * manual.important Most commonly used configuration options. +# * manual.summary Short summary describing the option. +# +################################################################################ + +manual.important BandwidthRate +manual.important BandwidthBurst +manual.important RelayBandwidthRate +manual.important RelayBandwidthBurst +manual.important ControlPort +manual.important HashedControlPassword +manual.important CookieAuthentication +manual.important DataDirectory +manual.important Log +manual.important RunAsDaemon +manual.important User + +manual.important Bridge +manual.important ExcludeNodes +manual.important MaxCircuitDirtiness +manual.important SOCKSPort +manual.important UseBridges + +manual.important BridgeRelay +manual.important ContactInfo +manual.important ExitPolicy +manual.important MyFamily +manual.important Nickname +manual.important ORPort +manual.important PortForwarding +manual.important AccountingMax +manual.important AccountingStart + +manual.important DirPortFrontPage +manual.important DirPort + +manual.important HiddenServiceDir +manual.important HiddenServicePort + +# General Config Options + +manual.summary.BandwidthRate Average bandwidth usage limit +manual.summary.BandwidthBurst Maximum bandwidth usage limit +manual.summary.MaxAdvertisedBandwidth Limit for the bandwidth we advertise as being available for relaying +manual.summary.RelayBandwidthRate Average bandwidth usage limit for relaying +manual.summary.RelayBandwidthBurst Maximum bandwidth usage limit for relaying +manual.summary.PerConnBWRate Average relayed bandwidth limit per connection +manual.summary.PerConnBWBurst Maximum relayed bandwidth limit per connection +manual.summary.ClientTransportPlugin Proxy when establishing bridge connections +manual.summary.ServerTransportPlugin Proxy when servicing bridge connections +manual.summary.ServerTransportListenAddr Endpoint for bridge's pluggable transport proxy +manual.summary.ServerTransportOptions Additional arguments for bridge's proxy +manual.summary.ExtORPort Endpoint for extended ORPort connections +manual.summary.ExtORPortCookieAuthFile Location of the ExtORPort's authentication cookie +manual.summary.ExtORPortCookieAuthFileGroupReadable Group read permissions for the ExtORPort's authentication cookie +manual.summary.ConnLimit Minimum number of file descriptors for Tor to start +manual.summary.DisableNetwork Don't accept non-controller connections +manual.summary.ConstrainedSockets Shrinks sockets to ConstrainedSockSize +manual.summary.ConstrainedSockSize Limit for the received and transmit buffers of sockets +manual.summary.ControlPort Port providing access to tor controllers (nyx, vidalia, etc) +manual.summary.ControlListenAddress Address providing controller access +manual.summary.ControlSocket Socket providing controller access +manual.summary.ControlSocketsGroupWritable Group read permissions for the control socket +manual.summary.HashedControlPassword Hash of the password for authenticating to the control port +manual.summary.CookieAuthentication If set, authenticates controllers via a cookie +manual.summary.CookieAuthFile Location of the authentication cookie +manual.summary.CookieAuthFileGroupReadable Group read permissions for the authentication cookie +manual.summary.ControlPortWriteToFile Path for a file tor writes containing its control port +manual.summary.ControlPortFileGroupReadable Group read permissions for the control port file +manual.summary.DataDirectory Location for storing runtime data (state, keys, etc) +manual.summary.FallbackDir Fallback when unable to retrieve descriptor information +manual.summary.DirAuthority Alternative directory authorities +manual.summary.DirAuthorityFallbackRate Rate at which to use fallback directory +manual.summary.AlternateDirAuthority Alternative directory authorities (consensus only) +manual.summary.AlternateBridgeAuthority Alternative directory authorities (bridges only) +manual.summary.DisableAllSwap Locks all allocated memory so they can't be paged out +manual.summary.DisableDebuggerAttachment Limit information applications can retrieve about the process +manual.summary.FetchDirInfoEarly Keeps consensus information up to date, even if unnecessary +manual.summary.FetchDirInfoExtraEarly Updates consensus information when it's first available +manual.summary.FetchHidServDescriptors Toggles if hidden service descriptors are fetched automatically or not +manual.summary.FetchServerDescriptors Toggles if the consensus is fetched automatically or not +manual.summary.FetchUselessDescriptors Toggles if relay descriptors are fetched when they aren't strictly necessary +manual.summary.HTTPProxy HTTP proxy for connecting to tor +manual.summary.HTTPProxyAuthenticator Authentication credentials for HTTPProxy +manual.summary.HTTPSProxy SSL proxy for connecting to tor +manual.summary.HTTPSProxyAuthenticator Authentication credentials for HTTPSProxy +manual.summary.Sandbox Run within a syscall sandbox +manual.summary.Socks4Proxy SOCKS 4 proxy for connecting to tor +manual.summary.Socks5Proxy SOCKS 5 for connecting to tor +manual.summary.Socks5ProxyUsername Username for connecting to the Socks5Proxy +manual.summary.Socks5ProxyPassword Password for connecting to the Socks5Proxy +manual.summary.SocksSocketsGroupWritable Group write permissions for the socks socket +manual.summary.KeepalivePeriod Rate at which to send keepalive packets +manual.summary.Log Runlevels and location for tor logging +manual.summary.LogMessageDomains Includes a domain when logging messages +manual.summary.OutboundBindAddress Sets the IP used for connecting to tor +manual.summary.PidFile Path for a file tor writes containing its process id +manual.summary.ProtocolWarnings Toggles if protocol errors give warnings or not +manual.summary.PredictedPortsRelevanceTime Duration to ensure circuits for previously used ports remain available +manual.summary.RunAsDaemon Toggles if tor runs as a daemon process +manual.summary.LogTimeGranularity limits granularity of log message timestamps +manual.summary.TruncateLogFile Overwrites log file rather than appending when restarted +manual.summary.SyslogIdentityTag Tag logs appended to the syslog as being from tor +manual.summary.SafeLogging Toggles if logs are scrubbed of sensitive information +manual.summary.User UID for the process when started +manual.summary.HardwareAccel Toggles if tor attempts to use hardware acceleration +manual.summary.AccelName OpenSSL engine name for crypto acceleration +manual.summary.AccelDir Crypto acceleration library path +manual.summary.AvoidDiskWrites Toggles if tor avoids frequently writing to disk +manual.summary.CircuitPriorityHalflife Overwrite method for prioritizing traffic among relayed connections +manual.summary.DisableIOCP Disables use of the Windows IOCP networking API +manual.summary.UserspaceIOCPBuffers Disable kernel-space IOCP TCP buffers +manual.summary.UseFilteringSSLBufferevents Use SSL for a chain of bufferevents +manual.summary.CountPrivateBandwidth Applies rate limiting to private IP addresses + +# Client Config Options + +manual.summary.AllowInvalidNodes Permits use of relays flagged as invalid by authorities +manual.summary.ExcludeSingleHopRelays Permits use of relays that allow single hop connections +manual.summary.Bridge Available bridges +manual.summary.LearnCircuitBuildTimeout Toggles adaptive timeouts for circuit creation +manual.summary.CircuitBuildTimeout Initial timeout for circuit creation +manual.summary.CircuitIdleTimeout Timeout for closing circuits that have never been used +manual.summary.CircuitStreamTimeout Timeout for shifting streams among circuits +manual.summary.ClientOnly Ensures that we aren't used as a relay or directory mirror +manual.summary.ExcludeNodes Relays or locales never to be used in circuits +manual.summary.ExcludeExitNodes Relays or locales never to be used for exits +manual.summary.GeoIPExcludeUnknown Don't use relays with an unknown locale in circuits +manual.summary.ExitNodes Preferred final hop for circuits +manual.summary.EntryNodes Preferred first hops for circuits +manual.summary.StrictNodes Never uses notes outside of Entry/ExitNodes +manual.summary.FascistFirewall Only make outbound connections on FirewallPorts +manual.summary.FirewallPorts Ports used by FascistFirewall +manual.summary.HidServAuth Authentication credentials for connecting to a hidden service +manual.summary.CloseHSClientCircuitsImmediatelyOnTimeout Close hidden service circuits that timeout +manual.summary.CloseHSServiceRendCircuitsImmediatelyOnTimeout Close hidden service rendezvous circuits that timeout +manual.summary.ReachableAddresses Rules for bypassing the local firewall +manual.summary.ReachableDirAddresses Rules for bypassing the local firewall (directory fetches) +manual.summary.ReachableORAddresses Rules for bypassing the local firewall (OR connections) +manual.summary.LongLivedPorts Ports requiring highly reliable relays +manual.summary.MapAddress Alias mappings for address requests +manual.summary.NewCircuitPeriod Period for considering the creation of new circuits +manual.summary.MaxCircuitDirtiness Duration for reusing constructed circuits +manual.summary.MaxClientCircuitsPending Number of circuits that can be in construction at once +manual.summary.NodeFamily Define relays as belonging to a family +manual.summary.EnforceDistinctSubnets Prevent use of multiple relays from the same subnet on a circuit +manual.summary.SOCKSPort Port for using tor as a Socks proxy +manual.summary.SOCKSListenAddress Address from which Socks connections can be made +manual.summary.SocksPolicy Access policy for the pocks port +manual.summary.SocksTimeout Time until idle or unestablished socks connections are closed +manual.summary.TokenBucketRefillInterval Frequency at which exhausted connections are checked for new traffic +manual.summary.TrackHostExits Maintains use of the same exit whenever connecting to this destination +manual.summary.TrackHostExitsExpire Time until use of an exit for tracking expires +manual.summary.UpdateBridgesFromAuthority Toggles fetching bridge descriptors from the authorities +manual.summary.UseBridges Make use of configured bridges +manual.summary.UseEntryGuards Use guard relays for first hop +manual.summary.UseEntryGuardsAsDirGuards Retrieve descriptors via guards when able +manual.summary.GuardfractionFile File containing information with duration of our guards +manual.summary.UseGuardFraction Take guardfraction into account for path selection +manual.summary.NumEntryGuards Pool size of guard relays we'll select from +manual.summary.NumDirectoryGuards Pool size of directory guards we'll select from +manual.summary.GuardLifetime Minimum time to keep entry guards +manual.summary.SafeSocks Toggles rejecting unsafe variants of the socks protocol +manual.summary.TestSocks Provide notices for if socks connections are of the safe or unsafe variants +manual.summary.WarnUnsafeSocks Toggle warning of unsafe socks connection +manual.summary.VirtualAddrNetworkIPv4 IPv4 address range to use when needing a virtual address +manual.summary.VirtualAddrNetworkIPv6 IPv6 address range to use when needing a virtual address +manual.summary.AllowNonRFC953Hostnames Toggles blocking invalid characters in hostname resolution +manual.summary.AllowDotExit Toggles allowing exit notation in addresses +manual.summary.FastFirstHopPK Toggle public key usage for the first hop +manual.summary.TransPort Port for transparent proxying if the OS supports it +manual.summary.TransListenAddress Address from which transparent proxy connections can be made +manual.summary.TransProxyType Proxy type to be used +manual.summary.NATDPort Port for forwarding ipfw NATD connections +manual.summary.NATDListenAddress Address from which NATD forwarded connections can be made +manual.summary.AutomapHostsOnResolve Map addresses ending with special suffixes to virtual addresses +manual.summary.AutomapHostsSuffixes Address suffixes recognized by AutomapHostsOnResolve +manual.summary.DNSPort Port from which DNS responses are fetched instead of tor +manual.summary.DNSListenAddress Address for performing DNS resolution +manual.summary.ClientDNSRejectInternalAddresses Ignores DNS responses for internal addresses +manual.summary.ClientRejectInternalAddresses Disables use of Tor for internal connections +manual.summary.DownloadExtraInfo Toggles fetching of extra information about relays +manual.summary.WarnPlaintextPorts Toggles warnings for using risky ports +manual.summary.RejectPlaintextPorts Prevents connections on risky ports +manual.summary.AllowSingleHopCircuits Makes use of single hop exits if able +manual.summary.OptimisticData Use exits without confirmation that prior connections succeeded +manual.summary.Tor2webMode Establish non-anonymous hidden service connections +manual.summary.Tor2webRendezvousPoints Rendezvous points to use for hidden services when in Tor2webMode +manual.summary.UseMicrodescriptors Retrieve microdescriptors rather than server descriptors +manual.summary.UseNTorHandshake Use ntor for establishing circuits with relays +manual.summary.PathBiasCircThreshold Number of circuits through a guard before applying bias checks +manual.summary.PathBiasNoticeRate Fraction of circuits that must succeed before logging a notice +manual.summary.PathBiasWarnRate Fraction of circuits that must succeed before logging a warning +manual.summary.PathBiasExtremeRate Fraction of circuits that must succeed before logging an error +manual.summary.PathBiasDropGuards Drop guards failing to establish circuits +manual.summary.PathBiasScaleThreshold Circuits through a guard before scaling past observations down +manual.summary.PathBiasUseThreshold Number of streams through a circuit before applying bias checks +manual.summary.PathBiasNoticeUseRate Fraction of streams that must succeed before logging a notice +manual.summary.PathBiasExtremeUseRate Fraction of streams that must succeed before logging an error +manual.summary.PathBiasScaleUseThreshold Streams through a circuit before scaling past observations down +manual.summary.ClientUseIPv6 Allow IPv6 connections to guards +manual.summary.ClientPreferIPv6ORPort Prefer a guard's IPv6 rather than IPv4 endpoint +manual.summary.PathsNeededToBuildCircuits Portion of relays to require information for before making circuits + +# Server Config Options + +manual.summary.Address Overwrites address others will use to reach this relay +manual.summary.AllowSingleHopExits Toggles permitting use of this relay as a single hop proxy +manual.summary.AssumeReachable Skips reachability test at startup +manual.summary.BridgeRelay Act as a bridge +manual.summary.ContactInfo Contact information for this relay +manual.summary.ExitRelay Allow relaying of exit traffic +manual.summary.ExitPolicy Traffic destinations that can exit from this relay +manual.summary.ExitPolicyRejectPrivate Prevent exiting connection on the local network +manual.summary.IPv6Exit Allow clients to use us for IPv6 traffic +manual.summary.MaxOnionQueueDelay Duration to reject new onionskins if we have more than we can process +manual.summary.MyFamily Other relays this operator administers +manual.summary.Nickname Identifier for this relay +manual.summary.NumCPUs Number of processes spawned for decryption +manual.summary.ORPort Port used to accept relay traffic +manual.summary.ORListenAddress Address for relay connections +manual.summary.PortForwarding Use UPnP or NAT-PMP if needed to relay +manual.summary.PortForwardingHelper Executable for configuring port forwarding +manual.summary.PublishServerDescriptor Types of descriptors published +manual.summary.ShutdownWaitLength Delay before quitting after receiving a SIGINT signal +manual.summary.SSLKeyLifetime Lifetime for our link certificate +manual.summary.HeartbeatPeriod Rate at which an INFO level heartbeat message is sent +manual.summary.AccountingMax Amount of traffic before hibernating +manual.summary.AccountingRule Method to determine when the accounting limit is reached +manual.summary.AccountingStart Duration of an accounting period +manual.summary.RefuseUnknownExits Prevents relays not in the consensus from using us as an exit +manual.summary.ServerDNSResolvConfFile Overriding resolver config for DNS queries we provide +manual.summary.ServerDNSAllowBrokenConfig Toggles if we persist despite configuration parsing errors or not +manual.summary.ServerDNSSearchDomains Toggles if our DNS queries search for addresses in the local domain +manual.summary.ServerDNSDetectHijacking Toggles testing for DNS hijacking +manual.summary.ServerDNSTestAddresses Addresses to test to see if valid DNS queries are being hijacked +manual.summary.ServerDNSAllowNonRFC953Hostnames Toggles if we reject DNS queries with invalid characters +manual.summary.BridgeRecordUsageByCountry Tracks geoip information on bridge usage +manual.summary.ServerDNSRandomizeCase Toggles DNS query case randomization +manual.summary.GeoIPFile Path to file containing IPv4 geoip information +manual.summary.GeoIPv6File Path to file containing IPv6 geoip information +manual.summary.TLSECGroup EC group for incoming SSL connections +manual.summary.CellStatistics Toggles storing circuit queue duration to disk +manual.summary.DirReqStatistics Toggles storing network status counts and performance to disk +manual.summary.EntryStatistics Toggles storing client connection counts to disk +manual.summary.ExitPortStatistics Toggles storing traffic and port usage data to disk +manual.summary.ConnDirectionStatistics Toggles storing connection use to disk +manual.summary.HiddenServiceStatistics Toggles storing hidden service stats to disk +manual.summary.ExtraInfoStatistics Publishes statistic data in the extra-info documents +manual.summary.ExtendAllowPrivateAddresses Allow circuits to be extended to the local network +manual.summary.MaxMemInQueues Threshold at which tor will terminate circuits to avoid running out of memory +manual.summary.SigningKeyLifetime Duration the Ed25519 signing key is valid for +manual.summary.OfflineMasterKey Don't generate the master secret key + +# Directory Server Options + +manual.summary.DirPortFrontPage Publish this html file on the DirPort +manual.summary.HidServDirectoryV2 Toggles accepting version 2 hidden service descriptors +manual.summary.DirPort Port for directory connections +manual.summary.DirListenAddress Address the directory service is bound to +manual.summary.DirPolicy Access policy for the DirPort + +# Directory Authority Server Options + +manual.summary.AuthoritativeDirectory Act as a directory authority +manual.summary.V3AuthoritativeDirectory Generates a version 3 consensus +manual.summary.VersioningAuthoritativeDirectory Provides opinions on recommended versions of tor +manual.summary.RecommendedVersions Suggested versions of tor +manual.summary.RecommendedPackageVersions Suggested versions of applications other than tor +manual.summary.RecommendedClientVersions Tor versions believed to be safe for clients +manual.summary.BridgeAuthoritativeDir Acts as a bridge authority +manual.summary.MinUptimeHidServDirectoryV2 Required uptime before accepting hidden service directory +manual.summary.RecommendedServerVersions Tor versions believed to be safe for relays +manual.summary.ConsensusParams Params entry of the networkstatus vote +manual.summary.DirAllowPrivateAddresses Toggles allowing arbitrary input or non-public IPs in descriptors +manual.summary.AuthDirBadExit Relays to be flagged as bad exits +manual.summary.AuthDirInvalid Relays from which the valid flag is withheld +manual.summary.AuthDirReject Relays to be dropped from the consensus +manual.summary.AuthDirBadExitCCs Countries for which to flag all relays as bad exits +manual.summary.AuthDirInvalidCCs Countries for which the valid flag is withheld +manual.summary.AuthDirRejectCCs Countries for which relays aren't accepted into the consensus +manual.summary.AuthDirListBadExits Toggles if we provide an opinion on bad exits +manual.summary.AuthDirMaxServersPerAddr Limit on the number of relays accepted per ip +manual.summary.AuthDirMaxServersPerAuthAddr Limit on the number of relays accepted per an authority's ip +manual.summary.AuthDirFastGuarantee Advertised rate at which the Fast flag is granted +manual.summary.AuthDirGuardBWGuarantee Advertised rate necessary to be a guard +manual.summary.AuthDirPinKeys Don't accept descriptors with conflicting identity keypairs +manual.summary.BridgePassword Password for requesting bridge information +manual.summary.V3AuthVotingInterval Consensus voting interval +manual.summary.V3AuthVoteDelay Wait time to collect votes of other authorities +manual.summary.V3AuthDistDelay Wait time to collect the signatures of other authorities +manual.summary.V3AuthNIntervalsValid Number of voting intervals a consensus is valid for +manual.summary.V3BandwidthsFile Path to a file containing measured relay bandwidths +manual.summary.V3AuthUseLegacyKey Signs consensus with both the current and legacy keys +manual.summary.RephistTrackTime Discards old, unchanged reliability information +manual.summary.VoteOnHidServDirectoriesV2 Determines if the authority votes on hidden service directories +manual.summary.AuthDirHasIPv6Connectivity Descriptors can be retrieved over the authority's IPv6 ORPort +manual.summary.MinMeasuredBWsForAuthToIgnoreAdvertised Total measured value before advertised bandwidths are treated as unreliable + +# Hidden Service Options + +manual.summary.HiddenServiceDir Directory contents for the hidden service +manual.summary.HiddenServicePort Port the hidden service is provided on +manual.summary.PublishHidServDescriptors Toggles automated publishing of the hidden service to the rendezvous directory +manual.summary.HiddenServiceVersion Version for published hidden service descriptors +manual.summary.HiddenServiceAuthorizeClient Restricts access to the hidden service +manual.summary.HiddenServiceAllowUnknownPorts Allow rendezvous circuits on unrecognized ports +manual.summary.HiddenServiceMaxStreams Maximum streams per rendezvous circuit +manual.summary.HiddenServiceMaxStreamsCloseCircuit Closes rendezvous circuits that exceed the maximum number of streams +manual.summary.RendPostPeriod Period at which the rendezvous service descriptors are refreshed +manual.summary.HiddenServiceDirGroupReadable Group read permissions for the hidden service directory +manual.summary.HiddenServiceNumIntroductionPoints Number of introduction points the hidden service will have + +# Testing Network Options + +manual.summary.TestingTorNetwork Overrides other options to be a testing network +manual.summary.TestingV3AuthInitialVotingInterval Overrides V3AuthVotingInterval for the first consensus +manual.summary.TestingV3AuthInitialVoteDelay Overrides TestingV3AuthInitialVoteDelay for the first consensus +manual.summary.TestingV3AuthInitialDistDelay Overrides TestingV3AuthInitialDistDelay for the first consensus +manual.summary.TestingV3AuthVotingStartOffset Offset for the point at which the authority votes +manual.summary.TestingAuthDirTimeToLearnReachability Delay until opinions are given about which relays are running or not +manual.summary.TestingEstimatedDescriptorPropagationTime Delay before clients attempt to fetch descriptors from directory caches +manual.summary.TestingMinFastFlagThreshold Minimum value for the Fast flag +manual.summary.TestingServerDownloadSchedule Schedule for when we should download resources as a relay +manual.summary.TestingClientDownloadSchedule Schedule for when we should download resources as a client +manual.summary.TestingServerConsensusDownloadSchedule Schedule for when we should download the consensus as a relay +manual.summary.TestingClientConsensusDownloadSchedule Schedule for when we should download the consensus as a client +manual.summary.TestingBridgeDownloadSchedule Schedule for when we should download bridge descriptors +manual.summary.TestingClientMaxIntervalWithoutRequest Maximum time to wait to batch requests for missing descriptors +manual.summary.TestingDirConnectionMaxStall Duration to let directory connections stall before timing out +manual.summary.TestingConsensusMaxDownloadTries Retries for downloading the consensus +manual.summary.TestingDescriptorMaxDownloadTries Retries for downloading server descriptors +manual.summary.TestingMicrodescMaxDownloadTries Retries for downloading microdescriptors +manual.summary.TestingCertMaxDownloadTries Retries for downloading authority certificates +manual.summary.TestingDirAuthVoteExit Relays to give the Exit flag to +manual.summary.TestingDirAuthVoteExitIsStrict Only grant the Exit flag to relays listed by TestingDirAuthVoteExit +manual.summary.TestingDirAuthVoteGuard Relays to give the Guard flag to +manual.summary.TestingDirAuthVoteGuardIsStrict Only grant the Guard flag to relays listed by TestingDirAuthVoteGuard +manual.summary.TestingDirAuthVoteHSDir Relays to give the HSDir flag to +manual.summary.TestingDirAuthVoteHSDirIsStrict Only grant the HSDir flag to relays listed by TestingDirAuthVoteHSDir +manual.summary.TestingEnableConnBwEvent Allow controllers to request CONN_BW events +manual.summary.TestingEnableCellStatsEvent Allow controllers to request CELL_STATS events +manual.summary.TestingEnableTbEmptyEvent Allow controllers to request TB_EMPTY events +manual.summary.TestingMinExitFlagThreshold Lower bound for assigning the Exit flag +manual.summary.TestingLinkCertifetime Duration of our ed25519 certificate +manual.summary.TestingAuthKeyLifetime Duration for our ed25519 signing key +manual.summary.TestingLinkKeySlop Time before expiration that we replace our ed25519 key + diff --git a/test/integ/manual.py b/test/integ/manual.py new file mode 100644 index 0000000..02de19e --- /dev/null +++ b/test/integ/manual.py @@ -0,0 +1,213 @@ +""" +Integ testing for the stem.manual module, fetching the latest man page from the +tor git repository and checking for new additions. +""" + +import codecs +import os +import tempfile +import unittest + +import stem.manual +import stem.util.system +import test.runner + +from stem.manual import Category + +EXPECTED_CATEGORIES = set([ + 'NAME', + 'SYNOPSIS', + 'DESCRIPTION', + 'COMMAND-LINE OPTIONS', + 'THE CONFIGURATION FILE FORMAT', + 'GENERAL OPTIONS', + 'CLIENT OPTIONS', + 'SERVER OPTIONS', + 'DIRECTORY SERVER OPTIONS', + 'DIRECTORY AUTHORITY SERVER OPTIONS', + 'HIDDEN SERVICE OPTIONS', + 'TESTING NETWORK OPTIONS', + 'SIGNALS', + 'FILES', + 'SEE ALSO', + 'BUGS', + 'AUTHORS', +]) + +EXPECTED_CLI_OPTIONS = set(['-h, -help', '-f FILE', '--allow-missing-torrc', '--defaults-torrc FILE', '--ignore-missing-torrc', '--hash-password PASSWORD', '--list-fingerprint', '--verify-config', '--service install [--options command-line options]', '--service remove|start|stop', '--nt-service', '--list-torrc-options', '--version', '--quiet|--hush']) +EXPECTED_SIGNALS = set(['SIGTERM', 'SIGINT', 'SIGHUP', 'SIGUSR1', 'SIGUSR2', 'SIGCHLD', 'SIGPIPE', 'SIGXFSZ']) + +EXPECTED_OPTION_COUNTS = { + Category.GENERAL: 74, + Category.CLIENT: 86, + Category.RELAY: 47, + Category.DIRECTORY: 5, + Category.AUTHORITY: 34, + Category.HIDDEN_SERVICE: 11, + Category.TESTING: 32, + Category.UNKNOWN: 0, +} + +EXPECTED_DESCRIPTION = """ +Tor is a connection-oriented anonymizing communication service. Users choose a source-routed path through a set of nodes, and negotiate a "virtual circuit" through the network, in which each node knows its predecessor and successor, but no others. Traffic flowing down the circuit is unwrapped by a symmetric key at each node, which reveals the downstream node. + +Basically, Tor provides a distributed network of servers or relays ("onion routers"). Users bounce their TCP streams - web traffic, ftp, ssh, etc. - around the network, and recipients, observers, and even the relays themselves have difficulty tracking the source of the stream. + +By default, tor will only act as a client only. To help the network by providing bandwidth as a relay, change the ORPort configuration option - see below. Please also consult the documentation on the Tor Project's website. +""".strip() + +EXPECTED_FILE_DESCRIPTION = 'Specify a new configuration file to contain further Tor configuration options OR pass - to make Tor read its configuration from standard input. (Default: @CONFDIR@/torrc, or $HOME/.torrc if that file is not found)' + +EXPECTED_BANDWIDTH_RATE_DESCRIPTION = 'A token bucket limits the average incoming bandwidth usage on this node to the specified number of bytes per second, and the average outgoing bandwidth usage to that same value. If you want to run a relay in the public network, this needs to be at the very least 75 KBytes for a relay (that is, 600 kbits) or 50 KBytes for a bridge (400 kbits) - but of course, more is better; we recommend at least 250 KBytes (2 mbits) if possible. (Default: 1 GByte)\n\nWith this option, and in other options that take arguments in bytes, KBytes, and so on, other formats are also supported. Notably, "KBytes" can also be written as "kilobytes" or "kb"; "MBytes" can be written as "megabytes" or "MB"; "kbits" can be written as "kilobits"; and so forth. Tor also accepts "byte" and "bit" in the singular. The prefixes "tera" and "T" are also recognized. If no units are given, we default to bytes. To avoid confusion, we recommend writing "bytes" or "bits" explicitly, sinc e it\'s easy to forget that "B" means bytes, not bits.' + + +class TestManual(unittest.TestCase): + @classmethod + def setUpClass(self): + self.man_path = None + self.man_content = None + self.skip_reason = None + self.download_error = None + + if stem.util.system.is_windows(): + self.skip_reason = '(unavailable on windows)' + elif test.runner.Target.ONLINE not in test.runner.get_runner().attribute_targets: + self.skip_reason = '(requires online target)' + elif not stem.util.system.is_available('a2x'): + self.skip_reason = '(requires asciidoc)' + else: + try: + with tempfile.NamedTemporaryFile(prefix = 'tor_man_page.', delete = False) as tmp: + stem.manual.download_man_page(file_handle = tmp) + self.man_path = tmp.name + + self.man_content = stem.util.system.call('man -P cat %s' % self.man_path) + except Exception as exc: + self.download_error = 'Unable to download the man page: %s' % exc + + @classmethod + def tearDownClass(self): + if self.man_path and os.path.exists(self.man_path): + os.remove(self.man_path) + + def requires_downloaded_manual(self): + if self.skip_reason: + test.runner.skip(self, self.skip_reason) + return True + elif self.download_error: + self.fail(self.download_error) + + return False + + def test_get_categories(self): + if self.requires_downloaded_manual(): + return + + categories = stem.manual._get_categories(self.man_content) + + present = set(categories.keys()) + missing_categories = present.difference(EXPECTED_CATEGORIES) + extra_categories = EXPECTED_CATEGORIES.difference(present) + + if missing_categories: + self.fail("Changed tor's man page? We expected the %s man page sections but they're no longer around, if expected then please update our test." % ', '.join(missing_categories)) + elif extra_categories: + self.fail("Changed tor's man page? We weren't expecting the %s man page sections, if expected then please update our test." % ', '.join(extra_categories)) + + self.assertEqual(['tor - The second-generation onion router'], categories['NAME']) + self.assertEqual(['tor [OPTION value]...'], categories['SYNOPSIS']) + self.assertEqual(8, len(categories['DESCRIPTION'])) # check parsing of multi-line entries + + def test_escapes_non_ascii(self): + if self.requires_downloaded_manual(): + return + + def check(content): + try: + codecs.ascii_encode(content, 'strict') + except UnicodeEncodeError as exc: + self.fail("Unable to read '%s' as ascii: %s" % (content, exc)) + + categories = stem.manual._get_categories(self.man_content) + + for category, lines in categories.items(): + check(category) + + for line in lines: + check(line) + + def test_has_all_summaries(self): + if self.requires_downloaded_manual(): + return + + manual = stem.manual.Manual.from_man(self.man_path) + present = set(manual.config_options.keys()) + expected = set([key[15:] for key in stem.manual._config(lowercase = False) if key.startswith('manual.summary.')]) + + # TODO: The 'Recognized' config name is due to our man page being slightly + # malformed. Sending a tor patch later to fix it. + + missing_options = present.difference(expected).difference(set(['Recognized'])) + extra_options = expected.difference(present) + + if missing_options: + self.fail("Changed tor's man page? Please update Stem's settings.cfg with summaries of the following config options: %s" % ', '.join(missing_options)) + elif extra_options: + self.fail("Changed tor's man page? Please remove the following summaries from Stem's settings.cfg: %s" % ', '.join(extra_options)) + + def test_attributes(self): + if self.requires_downloaded_manual(): + return + + def assert_equal(category, expected, actual): + if expected != actual: + self.fail("Changed tor's man page? The %s changed as follows...\n\nexpected: %s\n\nactual: %s" % (category, expected, actual)) + + manual = stem.manual.Manual.from_man(self.man_path) + + assert_equal('name', 'tor - The second-generation onion router', manual.name) + assert_equal('synopsis', 'tor [OPTION value]...', manual.synopsis) + assert_equal('description', EXPECTED_DESCRIPTION, manual.description) + + assert_equal('commandline options', EXPECTED_CLI_OPTIONS, set(manual.commandline_options.keys())) + assert_equal('help option', 'Display a short help message and exit.', manual.commandline_options['-h, -help']) + assert_equal('file option', EXPECTED_FILE_DESCRIPTION, manual.commandline_options['-f FILE']) + + assert_equal('signals', EXPECTED_SIGNALS, set(manual.signals.keys())) + assert_equal('sighup description', 'Tor will catch this, clean up and sync to disk if necessary, and exit.', manual.signals['SIGTERM']) + + assert_equal('number of files', 31, len(manual.files)) + assert_equal('lib path description', 'The tor process stores keys and other data here.', manual.files['@LOCALSTATEDIR@/lib/tor/']) + + for category, expected_count in EXPECTED_OPTION_COUNTS.items(): + assert_equal('number of %s category entries' % category, expected_count, len([entry for entry in manual.config_options.values() if entry.category == category])) + + option = manual.config_options['BandwidthRate'] + self.assertEqual(Category.GENERAL, option.category) + self.assertEqual('BandwidthRate', option.name) + self.assertEqual('N bytes|KBytes|MBytes|GBytes|KBits|MBits|GBits', option.usage) + self.assertEqual('Average bandwidth usage limit', option.summary) + self.assertEqual(EXPECTED_BANDWIDTH_RATE_DESCRIPTION, option.description) + + def test_with_unknown_options(self): + if stem.util.system.is_windows(): + test.runner.skip(self, '(unavailable on windows)') + return + + manual = stem.manual.Manual.from_man(os.path.join(os.path.dirname(__file__), 'tor.1_with_unknown')) + + self.assertEqual('tor - The second-generation onion router', manual.name) + self.assertEqual('', manual.synopsis) + self.assertEqual('', manual.description) + self.assertEqual({}, manual.commandline_options) + self.assertEqual({}, manual.signals) + self.assertEqual({}, manual.files) + + self.assertEqual(2, len(manual.config_options)) + + option = [entry for entry in manual.config_options.values() if entry.category == Category.UNKNOWN][0] + self.assertEqual(Category.UNKNOWN, option.category) + self.assertEqual('SpiffyNewOption', option.name) + self.assertEqual('transport exec path-to-binary [options]', option.usage) + self.assertEqual('', option.summary) + self.assertEqual('Description of this new option.', option.description) diff --git a/test/integ/tor.1_with_unknown b/test/integ/tor.1_with_unknown new file mode 100644 index 0000000..b5e0b82 --- /dev/null +++ b/test/integ/tor.1_with_unknown @@ -0,0 +1,51 @@ +'\" t +.\" Title: tor +.\" Author: [see the "AUTHORS" section] +.\" Generator: DocBook XSL Stylesheets v1.76.1 <http://docbook.sf.net/> +.\" Date: 10/03/2015 +.\" Manual: Tor Manual +.\" Source: Tor +.\" Language: English +.\" +.TH "TOR" "1" "10/03/2015" "Tor" "Tor Manual" +.\" ----------------------------------------------------------------- +.\" * Define some portability stuff +.\" ----------------------------------------------------------------- +.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.\" http://bugs.debian.org/507673 +.\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html +.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.ie \n(.g .ds Aq \(aq +.el .ds Aq ' +.\" ----------------------------------------------------------------- +.\" * set default formatting +.\" ----------------------------------------------------------------- +.\" disable hyphenation +.nh +.\" disable justification (adjust text to left margin only) +.ad l +.\" ----------------------------------------------------------------- +.\" * MAIN CONTENT STARTS HERE * +.\" ----------------------------------------------------------------- +.SH "NAME" +tor \- The second\-generation onion router +.SH "GENERAL OPTIONS" +.PP +\fBBandwidthRate\fR \fIN\fR \fBbytes\fR|\fBKBytes\fR|\fBMBytes\fR|\fBGBytes\fR|\fBKBits\fR|\fBMBits\fR|\fBGBits\fR +.RS 4 +A token bucket limits the average incoming bandwidth usage on this node to the specified number of bytes per second, and the average outgoing bandwidth usage to that same value\&. If you want to run a relay in the public network, this needs to be +\fIat the very least\fR +30 KBytes (that is, 30720 bytes)\&. (Default: 1 GByte) + + +With this option, and in other options that take arguments in bytes, KBytes, and so on, other formats are also supported\&. Notably, "KBytes" can also be written as "kilobytes" or "kb"; "MBytes" can be written as "megabytes" or "MB"; "kbits" can be written as "kilobits"; and so forth\&. Tor also accepts "byte" and "bit" in the singular\&. The prefixes "tera" and "T" are also recognized\&. If no units are given, we default to bytes\&. To avoid confusion, we recommend writing "bytes" or "bits" explicitly, since it\(cqs easy to forget that "B" means bytes, not bits\&. +.RE +.PP +.SH "NEW OPTIONS" +.PP +\fBSpiffyNewOption\fR \fItransport\fR exec \fIpath\-to\-binary\fR [options] +.RS 4 +Description of this new option. +.RE +.PP + diff --git a/test/settings.cfg b/test/settings.cfg index 13fb342..6812a90 100644 --- a/test/settings.cfg +++ b/test/settings.cfg @@ -211,6 +211,7 @@ test.integ_tests |test.integ.descriptor.microdescriptor.TestMicrodescriptor |test.integ.descriptor.networkstatus.TestNetworkStatus |test.integ.version.TestVersion +|test.integ.manual.TestManual |test.integ.response.protocolinfo.TestProtocolInfo |test.integ.process.TestProcess |test.integ.socket.control_socket.TestControlSocket diff --git a/test/unit/manual.py b/test/unit/manual.py index 60ae291..7ca69da 100644 --- a/test/unit/manual.py +++ b/test/unit/manual.py @@ -1,77 +1,32 @@ """ -Unit tessts for the stem.manual module. Test data comes from the following... - - * test/unit/tor.1 - Tor version 0.2.8.0-alpha-dev (git-3c6782395743a089) +Unit testing for the stem.manual module. """ -import codecs -import os +import io import unittest +import stem.prereq import stem.manual -import stem.util.system -import test.runner -from stem.manual import Category +try: + # account for urllib's change between python 2.x and 3.x + import urllib.request as urllib +except ImportError: + import urllib2 as urllib + +try: + # added in python 3.3 + from unittest.mock import Mock, patch +except ImportError: + from mock import Mock, patch try: - # added in python 3.2 - from functools import lru_cache + # added in python 2.7 + from collections import OrderedDict except ImportError: - from stem.util.lru_cache import lru_cache - -TEST_MAN_PAGE = os.path.join(os.path.dirname(__file__), 'tor.1') - -EXPECTED_CATEGORIES = set([ - 'NAME', - 'SYNOPSIS', - 'DESCRIPTION', - 'COMMAND-LINE OPTIONS', - 'THE CONFIGURATION FILE FORMAT', - 'GENERAL OPTIONS', - 'CLIENT OPTIONS', - 'SERVER OPTIONS', - 'DIRECTORY SERVER OPTIONS', - 'DIRECTORY AUTHORITY SERVER OPTIONS', - 'HIDDEN SERVICE OPTIONS', - 'TESTING NETWORK OPTIONS', - 'SIGNALS', - 'FILES', - 'SEE ALSO', - 'BUGS', - 'AUTHORS', -]) - -EXPECTED_CLI_OPTIONS = set(['-h, -help', '-f FILE', '--allow-missing-torrc', '--defaults-torrc FILE', '--ignore-missing-torrc', '--hash-password PASSWORD', '--list-fingerprint', '--verify-config', '--service install [--options command-line options]', '--service remove|start|stop', '--nt-service', '--list-torrc-options', '--version', '--quiet|--hush']) -EXPECTED_SIGNALS = set(['SIGTERM', 'SIGINT', 'SIGHUP', 'SIGUSR1', 'SIGUSR2', 'SIGCHLD', 'SIGPIPE', 'SIGXFSZ']) - -EXPECTED_OPTION_COUNTS = { - Category.GENERAL: 74, - Category.CLIENT: 86, - Category.RELAY: 47, - Category.DIRECTORY: 5, - Category.AUTHORITY: 34, - Category.HIDDEN_SERVICE: 11, - Category.TESTING: 32, - Category.UNKNOWN: 0, -} - -EXPECTED_DESCRIPTION = """ -Tor is a connection-oriented anonymizing communication service. Users choose a source-routed path through a set of nodes, and negotiate a "virtual circuit" through the network, in which each node knows its predecessor and successor, but no others. Traffic flowing down the circuit is unwrapped by a symmetric key at each node, which reveals the downstream node. - -Basically, Tor provides a distributed network of servers or relays ("onion routers"). Users bounce their TCP streams - web traffic, ftp, ssh, etc. - around the network, and recipients, observers, and even the relays themselves have difficulty tracking the source of the stream. - -By default, tor will only act as a client only. To help the network by providing bandwidth as a relay, change the ORPort configuration option - see below. Please also consult the documentation on the Tor Project's website. -""".strip() - -EXPECTED_FILE_DESCRIPTION = 'Specify a new configuration file to contain further Tor configuration options OR pass - to make Tor read its configuration from standard input. (Default: /usr/local/etc/tor/torrc, or $HOME/.torrc if that file is not found)' - -EXPECTED_BANDWIDTH_RATE_DESCRIPTION = 'A token bucket limits the average incoming bandwidth usage on this node to the specified number of bytes per second, and the average outgoing bandwidth usage to that same value. If you want to run a relay in the public network, this needs to be at the very least 30 KBytes (that is, 30720 bytes). (Default: 1 GByte)\n\nWith this option, and in other options that take arguments in bytes, KBytes, and so on, other formats are also supported. Notably, "KBytes" can also be written as "kilobytes" or "kb"; "MBytes" can be written as "megabytes" or "MB"; "kbits" can be written as "kilobits"; and so forth. Tor also accepts "byte" and "bit" in the singular. The prefixes "tera" and "T" are also recognized. If no units are given, we default to bytes. To avoid confusion, we recommend writing "bytes" or "bits" explicitly, since it\'s easy to forget that "B" means bytes, not bits.' - - -@lru_cache() -def man_content(): - return stem.util.system.call('man -P cat %s' % TEST_MAN_PAGE) + from stem.util.ordereddict import OrderedDict + +URL_OPEN = 'urllib.request.urlopen' if stem.prereq.is_python_3() else 'urllib2.urlopen' class TestManual(unittest.TestCase): @@ -82,105 +37,89 @@ class TestManual(unittest.TestCase): self.assertFalse(stem.manual.is_important('ConstrainedSockSize')) - def test_get_categories(self): - if stem.util.system.is_windows(): - test.runner.skip(self, '(unavailable on windows)') - return - - categories = stem.manual._get_categories(man_content()) - self.assertEqual(EXPECTED_CATEGORIES, set(categories.keys())) - self.assertEqual(['tor - The second-generation onion router'], categories['NAME']) - self.assertEqual(['tor [OPTION value]...'], categories['SYNOPSIS']) - self.assertEqual(8, len(categories['DESCRIPTION'])) # check parsing of multi-line entries - - def test_escapes_non_ascii(self): - if stem.util.system.is_windows(): - test.runner.skip(self, '(unavailable on windows)') - return - - def check(content): - try: - codecs.ascii_encode(content, 'strict') - except UnicodeEncodeError as exc: - self.fail("Unable to read '%s' as ascii: %s" % (content, exc)) - - categories = stem.manual._get_categories(man_content()) - - for category, lines in categories.items(): - check(category) - - for line in lines: - check(line) - - def test_has_all_summaries(self): - if stem.util.system.is_windows(): - test.runner.skip(self, '(unavailable on windows)') - return - - manual = stem.manual.Manual.from_man(TEST_MAN_PAGE) - present = set(manual.config_options.keys()) - expected = set([key[15:] for key in stem.manual._config(lowercase = False) if key.startswith('manual.summary.')]) - - # TODO: The 'Recognized' config name is due to our man page being slightly - # malformed. Sending a tor patch later to fix it. - - missing_options = present.difference(expected).difference(set(['Recognized'])) - extra_options = expected.difference(present) - - if missing_options: - self.fail("The following config options are missing summaries: %s" % ', '.join(missing_options)) - elif extra_options: - self.fail("The following config options no longer exist in tor, so don't need summaries: %s" % ', '.join(extra_options)) - - def test_attributes(self): - if stem.util.system.is_windows(): - test.runner.skip(self, '(unavailable on windows)') - return - - manual = stem.manual.Manual.from_man(TEST_MAN_PAGE) - - self.assertEqual('tor - The second-generation onion router', manual.name) - self.assertEqual('tor [OPTION value]...', manual.synopsis) - self.assertEqual(EXPECTED_DESCRIPTION, manual.description) - - self.assertEqual(EXPECTED_CLI_OPTIONS, set(manual.commandline_options.keys())) - self.assertEqual('Display a short help message and exit.', manual.commandline_options['-h, -help']) - self.assertEqual(EXPECTED_FILE_DESCRIPTION, manual.commandline_options['-f FILE']) - - self.assertEqual(EXPECTED_SIGNALS, set(manual.signals.keys())) - self.assertEqual('Tor will catch this, clean up and sync to disk if necessary, and exit.', manual.signals['SIGTERM']) - - self.assertEqual(31, len(manual.files)) - self.assertEqual('The tor process stores keys and other data here.', manual.files['/usr/local/var/lib/tor/']) - - for category, expected_count in EXPECTED_OPTION_COUNTS.items(): - self.assertEqual(expected_count, len([entry for entry in manual.config_options.values() if entry.category == category])) - - option = manual.config_options['BandwidthRate'] - self.assertEqual(Category.GENERAL, option.category) - self.assertEqual('BandwidthRate', option.name) - self.assertEqual('N bytes|KBytes|MBytes|GBytes|KBits|MBits|GBits', option.usage) - self.assertEqual('Average bandwidth usage limit', option.summary) - self.assertEqual(EXPECTED_BANDWIDTH_RATE_DESCRIPTION, option.description) - - def test_with_unknown_options(self): - if stem.util.system.is_windows(): - test.runner.skip(self, '(unavailable on windows)') - return - - manual = stem.manual.Manual.from_man(TEST_MAN_PAGE + '_with_unknown') - - self.assertEqual('tor - The second-generation onion router', manual.name) + def test_download_man_page_without_arguments(self): + try: + stem.manual.download_man_page() + self.fail('we should fail without a path or file handler') + except ValueError as exc: + self.assertEqual("Either the path or file_handle we're saving to must be provided", str(exc)) + + @patch('stem.util.system.is_available', Mock(return_value = False)) + def test_download_man_page_requires_a2x(self): + try: + stem.manual.download_man_page('/tmp/no_such_file') + self.fail('we should require a2x to be available') + except IOError as exc: + self.assertEqual('We require a2x from asciidoc to provide a man page', str(exc)) + + @patch('tempfile.mkdtemp', Mock(return_value = '/no/such/path')) + @patch('shutil.rmtree', Mock()) + @patch('stem.manual.open', Mock(side_effect = IOError('unable to write to file')), create = True) + def test_download_man_page_when_unable_to_write(self): + try: + stem.manual.download_man_page('/tmp/no_such_file') + self.fail("we shouldn't be able to write to /no/such/path") + except IOError as exc: + self.assertEqual("Unable to download tor's manual from https://gitweb.torproject.org/tor.git/plain/doc/tor.1.txt to /no/such/path/tor.1.txt: unable to write to file", str(exc)) + + @patch('tempfile.mkdtemp', Mock(return_value = '/no/such/path')) + @patch('shutil.rmtree', Mock()) + @patch('stem.manual.open', Mock(return_value = io.BytesIO()), create = True) + @patch(URL_OPEN, Mock(side_effect = urllib.URLError('<urlopen error [Errno -2] Name or service not known>'))) + def test_download_man_page_when_download_fails(self): + try: + stem.manual.download_man_page('/tmp/no_such_file', url = 'https://www.atagar.com/foo/bar') + self.fail("downloading from test_invalid_url.org shouldn't work") + except IOError as exc: + self.assertEqual("Unable to download tor's manual from https://www.atagar.com/foo/bar to /no/such/path/tor.1.txt: <urlopen error <urlopen error [Errno -2] Name or service not known>>", str(exc)) + + @patch('tempfile.mkdtemp', Mock(return_value = '/no/such/path')) + @patch('shutil.rmtree', Mock()) + @patch('stem.manual.open', Mock(return_value = io.BytesIO()), create = True) + @patch('stem.util.system.call', Mock(side_effect = OSError('call failed'))) + @patch(URL_OPEN, Mock(return_value = io.BytesIO(b'test content'))) + def test_download_man_page_when_a2x_fails(self): + try: + stem.manual.download_man_page('/tmp/no_such_file', url = 'https://www.atagar.com/foo/bar') + self.fail("downloading from test_invalid_url.org shouldn't work") + except IOError as exc: + self.assertEqual("Unable to run 'a2x -f manpage /no/such/path/tor.1.txt': call failed", str(exc)) + + @patch('tempfile.mkdtemp', Mock(return_value = '/no/such/path')) + @patch('shutil.rmtree', Mock()) + @patch('stem.manual.open', create = True) + @patch('stem.util.system.call') + @patch('os.path.exists', Mock(return_value = True)) + @patch(URL_OPEN, Mock(return_value = io.BytesIO(b'test content'))) + def test_download_man_page_when_successful(self, call_mock, open_mock): + open_mock.side_effect = lambda path, *args: { + '/no/such/path/tor.1.txt': io.BytesIO(), + '/no/such/path/tor.1': io.BytesIO(b'a2x output'), + }[path] + + call_mock.return_value = Mock() + + output = io.BytesIO() + stem.manual.download_man_page(file_handle = output) + self.assertEqual(b'a2x output', output.getvalue()) + call_mock.assert_called_once_with('a2x -f manpage /no/such/path/tor.1.txt') + + @patch('stem.util.system.call', Mock(side_effect = OSError('man -P cat tor returned exit status 16'))) + def test_from_man_when_manual_is_unavailable(self): + try: + stem.manual.Manual.from_man() + self.fail("fetching the manual should fail when it's unavailable") + except IOError as exc: + self.assertEqual("Unable to run 'man -P cat tor': man -P cat tor returned exit status 16", str(exc)) + + @patch('stem.util.system.call', Mock(return_value = [])) + def test_when_man_is_empty(self): + manual = stem.manual.Manual.from_man() + + self.assertEqual('', manual.name) self.assertEqual('', manual.synopsis) self.assertEqual('', manual.description) self.assertEqual({}, manual.commandline_options) self.assertEqual({}, manual.signals) - - self.assertEqual(2, len(manual.config_options)) - - option = [entry for entry in manual.config_options.values() if entry.category == Category.UNKNOWN][0] - self.assertEqual(Category.UNKNOWN, option.category) - self.assertEqual('SpiffyNewOption', option.name) - self.assertEqual('transport exec path-to-binary [options]', option.usage) - self.assertEqual('', option.summary) - self.assertEqual('Description of this new option.', option.description) + self.assertEqual({}, manual.files) + self.assertEqual(OrderedDict(), manual.config_options) diff --git a/test/unit/tor.1_with_unknown b/test/unit/tor.1_with_unknown deleted file mode 100644 index b5e0b82..0000000 --- a/test/unit/tor.1_with_unknown +++ /dev/null @@ -1,51 +0,0 @@ -'\" t -.\" Title: tor -.\" Author: [see the "AUTHORS" section] -.\" Generator: DocBook XSL Stylesheets v1.76.1 <http://docbook.sf.net/> -.\" Date: 10/03/2015 -.\" Manual: Tor Manual -.\" Source: Tor -.\" Language: English -.\" -.TH "TOR" "1" "10/03/2015" "Tor" "Tor Manual" -.\" ----------------------------------------------------------------- -.\" * Define some portability stuff -.\" ----------------------------------------------------------------- -.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.\" http://bugs.debian.org/507673 -.\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html -.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.ie \n(.g .ds Aq \(aq -.el .ds Aq ' -.\" ----------------------------------------------------------------- -.\" * set default formatting -.\" ----------------------------------------------------------------- -.\" disable hyphenation -.nh -.\" disable justification (adjust text to left margin only) -.ad l -.\" ----------------------------------------------------------------- -.\" * MAIN CONTENT STARTS HERE * -.\" ----------------------------------------------------------------- -.SH "NAME" -tor \- The second\-generation onion router -.SH "GENERAL OPTIONS" -.PP -\fBBandwidthRate\fR \fIN\fR \fBbytes\fR|\fBKBytes\fR|\fBMBytes\fR|\fBGBytes\fR|\fBKBits\fR|\fBMBits\fR|\fBGBits\fR -.RS 4 -A token bucket limits the average incoming bandwidth usage on this node to the specified number of bytes per second, and the average outgoing bandwidth usage to that same value\&. If you want to run a relay in the public network, this needs to be -\fIat the very least\fR -30 KBytes (that is, 30720 bytes)\&. (Default: 1 GByte) - - -With this option, and in other options that take arguments in bytes, KBytes, and so on, other formats are also supported\&. Notably, "KBytes" can also be written as "kilobytes" or "kb"; "MBytes" can be written as "megabytes" or "MB"; "kbits" can be written as "kilobits"; and so forth\&. Tor also accepts "byte" and "bit" in the singular\&. The prefixes "tera" and "T" are also recognized\&. If no units are given, we default to bytes\&. To avoid confusion, we recommend writing "bytes" or "bits" explicitly, since it\(cqs easy to forget that "B" means bytes, not bits\&. -.RE -.PP -.SH "NEW OPTIONS" -.PP -\fBSpiffyNewOption\fR \fItransport\fR exec \fIpath\-to\-binary\fR [options] -.RS 4 -Description of this new option. -.RE -.PP -