[or-cvs] r14542: Backport documentation to trunk. (in torflow/trunk: . TorCtl)

mikeperry at seul.org mikeperry at seul.org
Sat May 3 09:30:18 UTC 2008


Author: mikeperry
Date: 2008-05-03 05:30:18 -0400 (Sat, 03 May 2008)
New Revision: 14542

Modified:
   torflow/trunk/README
   torflow/trunk/TorCtl/PathSupport.py
   torflow/trunk/TorCtl/README
   torflow/trunk/TorCtl/TorCtl.py
   torflow/trunk/TorCtl/TorUtil.py
   torflow/trunk/metatroller.py
Log:

Backport documentation to trunk.



Modified: torflow/trunk/README
===================================================================
--- torflow/trunk/README	2008-05-03 09:28:43 UTC (rev 14541)
+++ torflow/trunk/README	2008-05-03 09:30:18 UTC (rev 14542)
@@ -26,7 +26,14 @@
 circuit construction and stream attachment subject to policies defined
 by NodeRestrictor and PathRestrictor implementations.
 
+The TorCtl package is now fully pydoced. From this directory, you can 
+do:
 
+# pydoc TorCtl.TorCtl
+and 
+# pydoc TorCtl.PathSupport
+
+
 2. metatroller.py 
 
 Metatroller observes the paths created by PathSupport and gathers
@@ -34,7 +41,11 @@
 failures and stream failures. It also provides a meta control port for
 use by Tor scanners.
 
+The metatroller is also pydoced. To view the documentation, you can do:
 
+# pydoc metatroller
+
+
 3. soat.pl
 
 SoaT scans exit nodes to verify that SSL, SSH, and Web connections

Modified: torflow/trunk/TorCtl/PathSupport.py
===================================================================
--- torflow/trunk/TorCtl/PathSupport.py	2008-05-03 09:28:43 UTC (rev 14541)
+++ torflow/trunk/TorCtl/PathSupport.py	2008-05-03 09:30:18 UTC (rev 14542)
@@ -1,5 +1,44 @@
 #!/usr/bin/python
+"""
 
+Support classes for path construction
+
+The PathSupport package builds on top of TorCtl.TorCtl. It provides a
+number of interfaces that make path construction easier.
+
+The inheritance diagram for event handling is as follows:
+TorCtl.EventHandler <- PathBuilder <- CircuitHandler <- StreamHandler.
+
+Basically, EventHandler is what gets all the control port events
+packaged in nice clean classes (see help(TorCtl) for information on
+those). 
+
+PathBuilder inherits from EventHandler and is what builds all circuits
+based on the requirements specified in the SelectionManager instance
+passed to its constructor. It also handles attaching streams to
+circuits. It only handles one building one circuit at a time.
+
+CircuitHandler optionally inherits from PathBuilder, and overrides its
+circuit event handling to manage building a pool of circuits as opposed
+to just one. It still uses the SelectionManager for path selection.
+
+StreamHandler inherits from CircuitHandler, and is what governs the
+attachment of an incoming stream on to one of the multiple circuits of
+the circuit handler. 
+
+The SelectionManager is essentially a configuration wrapper around the
+most elegant portions of TorFlow: NodeGenerators, NodeRestrictions, and
+PathRestrictions. In the SelectionManager, a NodeGenerator is used to
+choose the nodes probabilistically according to some distribution while
+obeying the NodeRestrictions. These generators (one per hop) are handed
+off to the PathSelector, which uses the generators to build a complete
+path that satisfies the PathRestriction requirements.
+
+Have a look at the class hierarchy directly below to get a feel for how
+the restrictions fit together, and what options are available.
+
+"""
+
 import TorCtl
 import re
 import struct
@@ -13,58 +52,77 @@
 __all__ = ["NodeRestrictionList", "PathRestrictionList",
 "PercentileRestriction", "OSRestriction", "ConserveExitsRestriction",
 "FlagsRestriction", "MinBWRestriction", "VersionIncludeRestriction",
-"VersionExcludeRestriction", "ExitPolicyRestriction", "OrNodeRestriction",
+"VersionExcludeRestriction", "ExitPolicyRestriction", "NodeRestriction",
+"PathRestriction", "OrNodeRestriction", "MetaNodeRestriction",
 "AtLeastNNodeRestriction", "NotNodeRestriction", "Subnet16Restriction",
-"UniqueRestriction", "UniformGenerator", "OrderedExitGenerator",
-"BwWeightedGenerator", "PathSelector", "Connection", "NickRestriction", 
-"IdHexRestriction", "PathBuilder", "CircuitHandler", "StreamHandler", 
-"SelectionManager", "CountryCodeRestriction", "CountryRestriction", 
-"UniqueCountryRestriction", "SingleCountryRestriction", 
-"ContinentRestriction", "ContinentJumperRestriction", 
+"UniqueRestriction", "NodeGenerator", "UniformGenerator",
+"OrderedExitGenerator", "BwWeightedGenerator", "PathSelector",
+"Connection", "NickRestriction", "IdHexRestriction", "PathBuilder",
+"CircuitHandler", "StreamHandler", "SelectionManager",
+"CountryCodeRestriction", "CountryRestriction",
+"UniqueCountryRestriction", "SingleCountryRestriction",
+"ContinentRestriction", "ContinentJumperRestriction",
 "UniqueContinentRestriction"]
 
 #################### Path Support Interfaces #####################
 
 class NodeRestriction:
   "Interface for node restriction policies"
-  def r_is_ok(self, r): return True  
+  def r_is_ok(self, r):
+    "Returns true if Router 'r' is acceptable for this restriction"
+    return True  
 
 class NodeRestrictionList:
+  "Class to manage a list of NodeRestrictions"
   def __init__(self, restrictions):
+    "Constructor. 'restrictions' is a list of NodeRestriction instances"
     self.restrictions = restrictions
 
   def r_is_ok(self, r):
+    "Returns true of Router 'r' passes all of the contained restrictions"
     for rs in self.restrictions:
       if not rs.r_is_ok(r): return False
     return True
 
   def add_restriction(self, restr):
+    "Add a NodeRestriction 'restr' to the list of restrictions"
     self.restrictions.append(restr)
 
   # TODO: This does not collapse meta restrictions..
   def del_restriction(self, RestrictionClass):
+    """Remove all restrictions of type RestrictionClass from the list.
+       Does NOT inspect or collapse MetaNode Restrictions (though 
+       MetaRestrictions can be removed if RestrictionClass is 
+       MetaNodeRestriction)"""
     self.restrictions = filter(
         lambda r: not isinstance(r, RestrictionClass),
           self.restrictions)
 
 class PathRestriction:
   "Interface for path restriction policies"
-  def path_is_ok(self, path): return True  
+  def path_is_ok(self, path):
+    "Return true if the list of Routers in path satisfies this restriction"
+    return True  
 
 class PathRestrictionList:
+  """Class to manage a list of PathRestrictions"""
   def __init__(self, restrictions):
+    "Constructor. 'restrictions' is a list of PathRestriction instances"
     self.restrictions = restrictions
   
   def path_is_ok(self, path):
+    "Given list if Routers in 'path', check it against each restriction."
     for rs in self.restrictions:
       if not rs.path_is_ok(path):
         return False
     return True
 
   def add_restriction(self, rstr):
+    "Add a PathRestriction 'rstr' to the list"
     self.restrictions.append(rstr)
 
   def del_restriction(self, RestrictionClass):
+    "Remove all PathRestrictions of type RestrictionClass from the list."
     self.restrictions = filter(
         lambda r: not isinstance(r, RestrictionClass),
           self.restrictions)
@@ -72,26 +130,37 @@
 class NodeGenerator:
   "Interface for node generation"
   def __init__(self, sorted_r, rstr_list):
+    """Constructor. Takes a bandwidth-sorted list of Routers 'sorted_r' 
+    and a NodeRestrictionList 'rstr_list'"""
     self.rstr_list = rstr_list # Check me before you yield!
     self.sorted_r = sorted_r
     self.rewind()
 
   def reset_restriction(self, rstr_list):
+    "Reset the restriction list to a new list"
     self.rstr_list = rstr_list
 
   def rewind(self):
+    "Rewind the generator to the 'beginning'"
     self.routers = copy.copy(self.sorted_r)
 
   def mark_chosen(self, r):
+    """Mark a router as chosen: remove it from the list of routers 
+     that can be returned in the future"""
     self.routers.remove(r)
 
   def all_chosen(self):
+    "Return true if all the routers have been marked as chosen"
     return not self.routers
 
-  def next_r(self): raise NotImplemented()
+  def next_r(self):
+    "Yield the next router according to the policy"
+    raise NotImplemented()
 
 class Connection(TorCtl.Connection):
+  """Extended Connection class that provides a method for building circuits"""
   def build_circuit(self, pathlen, path_sel):
+    "Tell Tor to build a circuit chosen by the PathSelector 'path_sel'"
     circ = Circuit()
     circ.path = path_sel.build_path(pathlen)
     circ.exit = circ.path[pathlen-1]
@@ -118,14 +187,21 @@
 #          Exit->destination hops
 
 class PercentileRestriction(NodeRestriction):
+  """Restriction to cut out a percentile slice of the network."""
   def __init__(self, pct_skip, pct_fast, r_list):
+    """Constructor. Sets up the restriction such that routers in the 
+     'pct_skip' to 'pct_fast' percentile of bandwidth rankings are 
+     returned from the sorted list 'r_list'"""
     self.pct_fast = pct_fast
     self.pct_skip = pct_skip
     self.sorted_r = r_list
 
   def r_is_ok(self, r):
+    "Returns true if r is in the percentile boundaries (by rank)"
     # Hrmm.. technically we shouldn't count non-running routers in this..
-    # but that is tricky to do efficiently
+    # but that is tricky to do efficiently.
+    # XXX: Is there any reason why sorted_r should have non-running
+    # routers in the first place?
     
     if r.list_rank < len(self.sorted_r)*self.pct_skip/100: return False
     elif r.list_rank > len(self.sorted_r)*self.pct_fast/100: return False
@@ -133,11 +209,15 @@
     return True
     
 class OSRestriction(NodeRestriction):
+  "Restriction based on operating system"
   def __init__(self, ok, bad=[]):
+    """Constructor. Accept router OSes that match regexes in 'ok', 
+       rejects those that match regexes in 'bad'."""
     self.ok = ok
     self.bad = bad
 
   def r_is_ok(self, r):
+    "Returns true if r is in 'ok', false if 'r' is in 'bad'. If 'ok'"
     for y in self.ok:
       if re.search(y, r.os):
         return True
@@ -148,11 +228,15 @@
     if self.bad: return True
 
 class ConserveExitsRestriction(NodeRestriction):
-  # FIXME: Make this adaptive
+  "Restriction to reject exits from selection"
+  # XXX: Make this adaptive by ip/port
   def r_is_ok(self, r): return not "Exit" in r.flags
 
 class FlagsRestriction(NodeRestriction):
+  "Restriction for mandatory and forbidden router flags"
   def __init__(self, mandatory, forbidden=[]):
+    """Constructor. 'mandatory' and 'forbidden' are both lists of router 
+     flags as strings."""
     self.mandatory = mandatory
     self.forbidden = forbidden
 
@@ -183,32 +267,42 @@
     return router.idhex == self.idhex
   
 class MinBWRestriction(NodeRestriction):
+  """Require a minimum bandwidth"""
   def __init__(self, minbw):
     self.min_bw = minbw
 
   def r_is_ok(self, router): return router.bw >= self.min_bw
    
 class VersionIncludeRestriction(NodeRestriction):
+  """Require that the version match one in the list"""
   def __init__(self, eq):
+    "Constructor. 'eq' is a list of versions as strings"
     self.eq = map(TorCtl.RouterVersion, eq)
   
   def r_is_ok(self, router):
+    """Returns true if the version of 'router' matches one of the 
+     specified versions."""
     for e in self.eq:
       if e == router.version:
         return True
     return False
 
 class VersionExcludeRestriction(NodeRestriction):
+  """Require that the version not match one in the list"""
   def __init__(self, exclude):
+    "Constructor. 'exclude' is a list of versions as strings"
     self.exclude = map(TorCtl.RouterVersion, exclude)
   
   def r_is_ok(self, router):
+    """Returns false if the version of 'router' matches one of the 
+     specified versions."""
     for e in self.exclude:
       if e == router.version:
         return False
     return True
 
 class VersionRangeRestriction(NodeRestriction):
+  """Require that the versions be inside a specified range""" 
   def __init__(self, gr_eq, less_eq=None):
     self.gr_eq = TorCtl.RouterVersion(gr_eq)
     if less_eq: self.less_eq = TorCtl.RouterVersion(less_eq)
@@ -219,6 +313,7 @@
         (not self.less_eq or router.version <= self.less_eq)
 
 class ExitPolicyRestriction(NodeRestriction):
+  """Require that a router exit to an ip+port"""
   def __init__(self, to_ip, to_port):
     self.to_ip = to_ip
     self.to_port = to_port
@@ -226,28 +321,37 @@
   def r_is_ok(self, r): return r.will_exit_to(self.to_ip, self.to_port)
 
 class MetaNodeRestriction(NodeRestriction):
+  """Interface for a NodeRestriction that is an expression consisting of 
+     multiple other NodeRestrictions"""
   # TODO: these should collapse the restriction and return a new
   # instance for re-insertion (or None)
   def next_rstr(self): raise NotImplemented()
   def del_restriction(self, RestrictionClass): raise NotImplemented()
 
 class OrNodeRestriction(MetaNodeRestriction):
+  """MetaNodeRestriction that is the boolean or of two or more
+     NodeRestrictions"""
   def __init__(self, rs):
+    "Constructor. 'rs' is a list of NodeRestrictions"
     self.rstrs = rs
 
   def r_is_ok(self, r):
+    "Returns true if one of 'rs' is true for this router"
     for rs in self.rstrs:
       if rs.r_is_ok(r):
         return True
     return False
 
 class NotNodeRestriction(MetaNodeRestriction):
+  """Negates a single restriction"""
   def __init__(self, a):
     self.a = a
 
   def r_is_ok(self, r): return not self.a.r_is_ok(r)
 
 class AtLeastNNodeRestriction(MetaNodeRestriction):
+  """MetaNodeRestriction that is true if at least n member 
+     restrictions are true."""
   def __init__(self, rstrs, n):
     self.rstrs = rstrs
     self.n = n
@@ -264,6 +368,8 @@
 #################### Path Restrictions #####################
 
 class Subnet16Restriction(PathRestriction):
+  """PathRestriction that mandates that no two nodes from the same 
+     /16 subnet be in the path"""
   def path_is_ok(self, path):
     mask16 = struct.unpack(">I", socket.inet_aton("255.255.0.0"))[0]
     ip16 = path[0].ip & mask16
@@ -273,6 +379,8 @@
     return True
 
 class UniqueRestriction(PathRestriction):
+  """Path restriction that mandates that the same router can't appear more
+     than once in a path"""
   def path_is_ok(self, path):
     for i in xrange(0,len(path)):
       if path[i] in path[:i]:
@@ -287,7 +395,7 @@
     return r.country_code != None
 
 class CountryRestriction(NodeRestriction):
-  """ Ensure a specific country_code for nodes """
+  """ Only accept nodes that are in 'country_code' """
   def __init__(self, country_code):
     self.country_code = country_code
 
@@ -382,12 +490,15 @@
 #################### Node Generators ######################
 
 class UniformGenerator(NodeGenerator):
+  """NodeGenerator that produces nodes in the uniform distribution"""
   def next_r(self):
     while not self.all_chosen():
       r = random.choice(self.routers)
       if self.rstr_list.r_is_ok(r): yield r
 
 class OrderedExitGenerator(NodeGenerator):
+  """NodeGenerator that produces exits in an ordered fashion for a 
+     specific port"""
   def __init__(self, to_port, sorted_r, rstr_list):
     self.to_port = to_port
     self.next_exit_by_port = {}
@@ -421,8 +532,8 @@
         break
 
 class BwWeightedGenerator(NodeGenerator):
-  """ Pass exit=True to create a generator for exit-nodes """
   def __init__(self, sorted_r, rstr_list, pathlen, exit=False):
+    """ Pass exit=True to create a generator for exit-nodes """
     # Out for an exit-node?
     self.exit = exit
     # Different sums of bandwidths
@@ -505,14 +616,21 @@
   pass
 
 class PathSelector:
-  "Implementation of path selection policies"
+  """Implementation of path selection policies. Builds a path according
+     to entry, middle, and exit generators that satisfies the path 
+     restrictions."""
   def __init__(self, entry_gen, mid_gen, exit_gen, path_restrict):
+    """Constructor. The first three arguments are NodeGenerators with 
+     their appropriate restrictions. The 'path_restrict' is a
+     PathRestrictionList"""
     self.entry_gen = entry_gen
     self.mid_gen = mid_gen
     self.exit_gen = exit_gen
     self.path_restrict = path_restrict
 
   def build_path(self, pathlen):
+    """Creates a path of 'pathlen' hops, and returns it as a list of
+       Router instances"""
     self.entry_gen.rewind()
     self.mid_gen.rewind()
     self.exit_gen.rewind()
@@ -550,11 +668,10 @@
   """Helper class to handle configuration updates
     
     The methods are NOT threadsafe. They may ONLY be called from
-    EventHandler's thread.
-
-    To update the selection manager, schedule a config update job
-    using PathBuilder.schedule_selmgr() with a worker function
-    to modify this object.
+    EventHandler's thread. This means that to update the selection 
+    manager, you must schedule a config update job using 
+    PathBuilder.schedule_selmgr() with a worker function to modify 
+    this object.
     """
   def __init__(self, pathlen, order_exits,
          percent_fast, percent_skip, min_bw, use_all_exits,
@@ -572,6 +689,8 @@
     self.geoip_config = geoip_config
 
   def reconfigure(self, sorted_r):
+    """This function is called after a configuration change, 
+     to rebuild the RestrictionLists."""
     if self.use_all_exits:
       self.path_rstr = PathRestrictionList([UniqueRestriction()])
     else:
@@ -679,6 +798,7 @@
       return
 
   def set_target(self, ip, port):
+    "Called to update the ExitPolicyRestrictions with a new ip and port"
     self.exit_rstr.del_restriction(ExitPolicyRestriction)
     self.exit_rstr.add_restriction(ExitPolicyRestriction(ip, port))
     if self.__ordered_exit_gen: self.__ordered_exit_gen.set_port(port)
@@ -699,6 +819,7 @@
           self.exit_rstr.add_restriction(CountryRestriction(self.geoip_config.exit_country))
 
 class Circuit:
+  "Class to describe a circuit"
   def __init__(self):
     self.circ_id = 0
     self.path = [] # routers
@@ -712,9 +833,12 @@
     self.setup_duration = None  # Sum of extend-times
     self.pending_streams = []   # Which stream IDs are pending us
   
-  def id_path(self): return map(lambda r: r.idhex, self.path)
+  def id_path(self):
+    "Returns a list of idhex keys for the path of Routers"
+    return map(lambda r: r.idhex, self.path)
 
 class Stream:
+  "Class to describe a stream"
   def __init__(self, sid, host, port, kind):
     self.strm_id = sid
     self.detached_from = [] # circ id #'s
@@ -729,7 +853,9 @@
     self.failed = False
     self.failed_reason = None # Cheating a little.. Only used by StatsHandler
 
-  def lifespan(self, now): return now-self.attached_at
+  def lifespan(self, now):
+    "Returns the age of the stream"
+    return now-self.attached_at
 
 # TODO: Make passive "PathWatcher" so people can get aggregate 
 # node reliability stats for normal usage without us attaching streams
@@ -744,6 +870,9 @@
   of the EventHandler.
   """
   def __init__(self, c, selmgr, RouterClass):
+    """Constructor. 'c' is a Connection, 'selmgr' is a SelectionManager,
+    and 'RouterClass' is a class that inherits from Router and is used
+    to create annotated Routers."""
     TorCtl.EventHandler.__init__(self)
     self.c = c
     nslist = c.get_network_status()
@@ -792,6 +921,9 @@
 
      
   def heartbeat_event(self, event):
+    """This function handles dispatching scheduled jobs. If you 
+       extend PathBuilder and want to implement this function for 
+       some reason, be sure to call the parent class"""
     while not self.imm_jobs.empty():
       imm_job = self.imm_jobs.get_nowait()
       imm_job(self)
@@ -848,6 +980,7 @@
     return self.selmgr.path_selector.build_path(self.selmgr.pathlen)
 
   def attach_stream_any(self, stream, badcircs):
+    "Attach a stream to a valid circuit, avoiding any in 'badcircs'"
     # Newnym, and warn if not built plus pending
     unattached_streams = [stream]
     if self.new_nym:
@@ -1034,8 +1167,13 @@
 ################### CircuitHandler #############################
 
 class CircuitHandler(PathBuilder):
-  """ CircuitHandler that extends from PathBuilder """
+  """ CircuitHandler that extends from PathBuilder to handle multiple
+      circuits as opposed to just one. """
   def __init__(self, c, selmgr, num_circuits, RouterClass):
+    """Constructor. 'c' is a Connection, 'selmgr' is a SelectionManager,
+    'num_circuits' is the number of circuits to keep in the pool,
+    and 'RouterClass' is a class that inherits from Router and is used
+    to create annotated Routers."""
     PathBuilder.__init__(self, c, selmgr, RouterClass)
     # Set handler to the connection here to 
     # not miss any circuit events on startup
@@ -1143,7 +1281,9 @@
 ################### StreamHandler ##############################
 
 class StreamHandler(CircuitHandler):
-  """ StreamHandler that extends from the CircuitHandler """
+  """ StreamHandler that extends from the CircuitHandler 
+      to handle attaching streams to an appropriate circuit 
+      in the pool. """
   def __init__(self, c, selmgr, num_circs, RouterClass):
     CircuitHandler.__init__(self, c, selmgr, num_circs, RouterClass)
     self.sorted_circs = None    # optional sorted list

Modified: torflow/trunk/TorCtl/README
===================================================================
--- torflow/trunk/TorCtl/README	2008-05-03 09:28:43 UTC (rev 14541)
+++ torflow/trunk/TorCtl/README	2008-05-03 09:30:18 UTC (rev 14542)
@@ -1,5 +1,10 @@
-TODO: Write me.
+See the pydoc:
 
-In the mean time, try
-svn co https://tor-svn.freehaven.net/svn/torctl/trunk/doc/howto.txt
+# python
 
+>>> import TorCtl
+>>> help(TorCtl)
+
+>>> import PathSupport
+>>> help(PathSupport)
+

Modified: torflow/trunk/TorCtl/TorCtl.py
===================================================================
--- torflow/trunk/TorCtl/TorCtl.py	2008-05-03 09:28:43 UTC (rev 14541)
+++ torflow/trunk/TorCtl/TorCtl.py	2008-05-03 09:30:18 UTC (rev 14542)
@@ -4,10 +4,32 @@
 # Copyright 2007 Mike Perry. See LICENSE file.
 
 """
-TorCtl -- Library to control Tor processes.
+Library to control Tor processes.
+
+This library handles sending commands, parsing responses, and delivering
+events to and from the control port. The basic usage is to create a
+socket, wrap that in a TorCtl.Connection, and then add an EventHandler
+to that connection. A simple example with a DebugEventHandler (that just
+echoes the events back to stdout) is present in run_example().
+
+Note that the TorCtl.Connection is fully compatible with the more
+advanced EventHandlers in TorCtl.PathSupport (and of course any other
+custom event handlers that you may extend off of those).
+
+This package also contains a helper class for representing Routers, and
+classes and constants for each event.
+
 """
 
-# XXX: Docstring all exported classes/interfaces. Also need __all__
+__all__ = ["EVENTTYPE", "CIRC", "STREAM", "ORCONN", "STREAM_BW", "BW",
+           "NS", "NEWDESC", "ADDRMAP", "DEBUG", "INFO", "NOTICE", "WARN",
+           "ERR", "TorCtlError", "TorCtlClosed", "ProtocolError",
+           "ErrorReply", "NetworkStatus", "ExitPolicyLine", "Router",
+           "RouterVersion", "Connection", "parse_ns_body",
+           "EventHandler", "DebugEventHandler", "NetworkStatusEvent",
+           "NewDescEvent", "CircuitEvent", "StreamEvent", "ORConnEvent",
+           "StreamBwEvent", "LogEvent", "AddrMapEvent", "BWEvent",
+           "UnknownEvent" ]
 
 import os
 import re
@@ -151,6 +173,8 @@
     self.event_string = event_string
 
 class ExitPolicyLine:
+  """ Class to represent a line in a Router's exit policy in a way 
+      that can be easily checked. """
   def __init__(self, match, ip_mask, port_low, port_high):
     self.match = match
     if ip_mask == "*":
@@ -177,6 +201,8 @@
       self.port_high = int(port_high)
   
   def check(self, ip, port):
+    """Check to see if an ip and port is matched by this line. 
+     Returns true if the line is an Accept, and False if it is a Reject. """
     ip = struct.unpack(">I", socket.inet_aton(ip))[0]
     if (ip & self.netmask) == self.ip:
       if self.port_low <= port and port <= self.port_high:
@@ -184,6 +210,8 @@
     return -1
 
 class RouterVersion:
+  """ Represents a Router's version. Overloads all comparison operators
+      to check for newer, older, or equivalent versions. """
   def __init__(self, version):
     if version:
       v = re.search("^(\d+).(\d+).(\d+).(\d+)", version).groups()
@@ -202,6 +230,11 @@
   def __str__(self): return self.ver_string
 
 class Router:
+  """ 
+  Class to represent a router from a descriptor. Can either be
+  created from the parsed fields, or can be built from a
+  descriptor+NetworkStatus 
+  """     
   def __init__(self, idhex, name, bw, down, exitpolicy, flags, ip, version, os, uptime):
     self.idhex = idhex
     self.nickname = name
@@ -216,6 +249,13 @@
     self.uptime = uptime
 
   def build_from_desc(desc, ns):
+    """
+    Static method of Router that parses a descriptor string into this class.
+    'desc' is a full descriptor as a string. 
+    'ns' is a TorCtl.NetworkStatus instance for this router (needed for
+    the flags, the nickname, and the idhex string). 
+    Returns a Router instance.
+    """
     # XXX: Compile these regular expressions? This is an expensive process
     # Use http://docs.python.org/lib/profile.html to verify this is 
     # the part of startup that is slow
@@ -264,6 +304,8 @@
   build_from_desc = Callable(build_from_desc)
 
   def update_to(self, new):
+    """ Somewhat hackish method to update this router to be a copy of
+    'new' """
     if self.idhex != new.idhex:
       plog("ERROR", "Update of router "+self.nickname+"changes idhex!")
     self.idhex = new.idhex
@@ -277,6 +319,8 @@
     self.uptime = new.uptime
 
   def will_exit_to(self, ip, port):
+    """ Check the entire exitpolicy to see if the router will allow
+        connections to 'ip':'port' """
     for line in self.exitpolicy:
       ret = line.check(ip, port)
       if ret != -1:
@@ -285,7 +329,8 @@
     return False
    
 class Connection:
-  """A Connection represents a connection to the Tor process."""
+  """A Connection represents a connection to the Tor process via the 
+     control port."""
   def __init__(self, sock):
     """Create a Connection to communicate with the Tor process over the
        socket 'sock'.
@@ -454,7 +499,7 @@
     """Cause future events from the Tor process to be sent to 'handler'.
     """
     self._handler = handler
-    self._handleFn = handler.handle1
+    self._handleFn = handler._handle1
 
   def _read_reply(self):
     lines = []
@@ -572,7 +617,8 @@
     self.sendAndRecv("RESETCONF %s\r\n"%(" ".join(keylist)))
 
   def get_network_status(self, who="all"):
-    """Get the entire network status list"""
+    """Get the entire network status list. Returns a list of
+       TorCtl.NetworkStatus instances."""
     return parse_ns_body(self.sendAndRecv("GETINFO ns/"+who+"\r\n")[0][2])
 
   def get_router(self, ns):
@@ -582,6 +628,9 @@
 
 
   def read_routers(self, nslist):
+    """ Given a list a NetworkStatuses in 'nslist', this function will 
+        return a list of new Router instances.
+    """
     bad_key = 0
     new = []
     for ns in nslist:
@@ -656,6 +705,7 @@
     self.sendAndRecv("RESOLVE %s\r\n"%host)
 
   def map_address(self, kvList):
+    """ Sends the MAPADDRESS command for each of the tuples in kvList """
     if not kvList:
       return
     m = " ".join([ "%s=%s" for k,v in kvList])
@@ -693,9 +743,9 @@
       self.sendAndRecv("REDIRECTSTREAM %d %s\r\n"%(streamid, newaddr))
 
   def attach_stream(self, streamid, circid, hop=None):
-    """Attach a stream to a circuit, specify both by IDs. 
-       If hop is given, try to use the specified hop in 
-       the circuit as the exit node for this stream.
+    """Attach a stream to a circuit, specify both by IDs. If hop is given, 
+       try to use the specified hop in the circuit as the exit node for 
+       this stream.
     """
     if hop:
       self.sendAndRecv("ATTACHSTREAM %d %d HOP=%d\r\n"%(streamid, circid, hop))
@@ -718,7 +768,8 @@
     self.sendAndRecv("+POSTDESCRIPTOR\r\n%s"%escape_dots(desc))
 
 def parse_ns_body(data):
-  "Parse the body of an NS event or command."
+  """Parse the body of an NS event or command into a list of
+     NetworkStatus instances"""
   nsgroups = re.compile(r"^r ", re.M).split(data)
   nsgroups.pop(0)
   nslist = []
@@ -731,7 +782,9 @@
   return nslist
 
 class EventHandler:
-  """An 'EventHandler' wraps callbacks for the events Tor can return."""
+  """An 'EventHandler' wraps callbacks for the events Tor can return. 
+     Each event argument is an instance of the corresponding event
+     class."""
   def __init__(self):
     """Create a new EventHandler."""
     self._map1 = {
@@ -750,15 +803,15 @@
       "NS" : self.ns_event
       }
 
-  def handle1(self, timestamp, lines):
+  def _handle1(self, timestamp, lines):
     """Dispatcher: called from Connection when an event is received."""
     for code, msg, data in lines:
-      event = self.decode1(msg, data)
+      event = self._decode1(msg, data)
       event.arrived_at = timestamp
       self.heartbeat_event(event)
       self._map1.get(event.event_name, self.unknown_event)(event)
 
-  def decode1(self, body, data):
+  def _decode1(self, body, data):
     """Unpack an event message into a type/arguments-tuple tuple."""
     if " " in body:
       evtype,body = body.split(" ",1)
@@ -977,6 +1030,9 @@
   return host, port
 
 def run_example(host,port):
+  """ Example of basic TorCtl usage. See PathSupport for more advanced
+      usage.
+  """
   print "host is %s:%d"%(host,port)
   s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
   s.connect((host,port))

Modified: torflow/trunk/TorCtl/TorUtil.py
===================================================================
--- torflow/trunk/TorCtl/TorUtil.py	2008-05-03 09:28:43 UTC (rev 14541)
+++ torflow/trunk/TorCtl/TorUtil.py	2008-05-03 09:30:18 UTC (rev 14542)
@@ -17,10 +17,10 @@
 
 __all__ = ["Enum", "Enum2", "Callable", "sort_list", "quote", "escape_dots", "unescape_dots",
       "BufSock", "secret_to_key", "urandom_rng", "s2k_gen", "s2k_check", "plog", 
-      "ListenSocket", "zprob"]
+     "ListenSocket", "zprob"]
 
 class Enum:
-  # Helper: define an ordered dense name-to-number 1-1 mapping.
+  """ Defines an ordered dense name-to-number 1-1 mapping """
   def __init__(self, start, names):
     self.nameOf = {}
     idx = start
@@ -30,7 +30,7 @@
       idx += 1
 
 class Enum2:
-  # Helper: define an ordered sparse name-to-number 1-1 mapping.
+  """ Defines an ordered sparse name-to-number 1-1 mapping """
   def __init__(self, **args):
     self.__dict__.update(args)
     self.nameOf = {}

Modified: torflow/trunk/metatroller.py
===================================================================
--- torflow/trunk/metatroller.py	2008-05-03 09:28:43 UTC (rev 14541)
+++ torflow/trunk/metatroller.py	2008-05-03 09:30:18 UTC (rev 14542)
@@ -2,7 +2,14 @@
 # Metatroller. 
 
 """
-Metatroller - Tor Meta controller
+Tor Meta controller
+
+The Metatroller uses TorCtl.PathSupport to build a meta-controller that
+listens for commands on a local TCP port. In addition, it gathers a
+large amount of statistics on circuit failure rates, streams failure
+rates, stream bandwidths, probabilities of bandwidth ratios, and much
+much more.
+
 """
 
 import atexit
@@ -44,7 +51,10 @@
       use_exit=None,
       use_guards=False)
 
+# FIXME: Much of this should be moved into TorCtl/StatsSupport.py
+
 class BandwidthStats:
+  "Class that manages observed bandwidth through a Router"
   def __init__(self):
     self.byte_list = []
     self.duration_list = []
@@ -54,6 +64,7 @@
     self.dev = 0
 
   def _exp(self): # Weighted avg
+    "Expectation - weighted average of the bandwidth through this node"
     tot_bw = reduce(lambda x, y: x+y, self.byte_list, 0.0)
     EX = 0.0
     for i in xrange(len(self.byte_list)):
@@ -63,6 +74,7 @@
     return EX
 
   def _exp2(self): # E[X^2]
+    "Second moment of the bandwidth"
     tot_bw = reduce(lambda x, y: x+y, self.byte_list, 0.0)
     EX = 0.0
     for i in xrange(len(self.byte_list)):
@@ -72,6 +84,7 @@
     return EX
     
   def _dev(self): # Weighted dev
+    "Standard deviation of bandwidth"
     EX = self.mean
     EX2 = self._exp2()
     arg = EX2 - (EX*EX)
@@ -80,6 +93,7 @@
     return math.sqrt(abs(arg))
 
   def add_bw(self, bytes, duration):
+    "Add an observed transfer of 'bytes' for 'duration' seconds"
     if not bytes: plog("WARN", "No bytes for bandwidth")
     bytes /= 1024.
     self.byte_list.append(bytes)
@@ -94,11 +108,15 @@
 # Technically we could just add member vars as we need them, but this
 # is a bit more clear
 class StatsRouter(TorCtl.Router):
+  "Extended Router to handle statistics markup"
   def __init__(self, router): # Promotion constructor :)
+    """'Promotion Constructor' that converts a Router directly into a 
+    StatsRouter without a copy."""
     self.__dict__ = router.__dict__
     self.reset()
   
   def reset(self):
+    "Reset all stats on this Router"
     self.circ_uncounted = 0
     self.circ_failed = 0
     self.circ_succeeded = 0 # disjoint from failed
@@ -129,11 +147,15 @@
     self.prob_zb = 0
 
   def avg_extend_time(self):
+    """Return the average amount of time it took for this router
+     to extend a circuit one hop"""
     if self.total_extended:
       return self.total_extend_time/self.total_extended
     else: return 0
 
   def bw_ratio(self):
+    """Return the ratio of the Router's advertised bandwidth to its 
+     observed average stream bandwidth"""
     bw = self.bwstats.mean
     if bw == 0.0: return 0
     else: return self.bw/(1024.*bw)
@@ -147,10 +169,14 @@
     else: return ret
         
   def failed_per_hour(self):
+    """Return the number of circuit extend failures per hour for this 
+     Router"""
     return (3600.*(self.circ_failed+self.strm_failed))/self.current_uptime()
 
   # XXX: Seperate suspected from failed in totals 
   def suspected_per_hour(self):
+    """Return the number of circuits that failed with this router as an
+     earlier hop"""
     return (3600.*(self.circ_suspected+self.strm_suspected
           +self.circ_failed+self.strm_failed))/self.current_uptime()
 
@@ -196,6 +222,7 @@
       +" U="+str(round(self.current_uptime()/3600, 1))+"\n")
 
   def sanity_check(self):
+    "Makes sure all stats are self-consistent"
     if (self.circ_failed + self.circ_succeeded + self.circ_suspected
       + self.circ_uncounted != self.circ_chosen):
       plog("ERROR", self.nickname+" does not add up for circs")
@@ -228,7 +255,7 @@
                     +str(per_hour_tot) +" vs "+str(chosen_tot))
 
 class ReasonRouterList:
-  "Helper class to track which reasons are in which routers."
+  "Helper class to track which Routers have failed for a given reason"
   def __init__(self, reason):
     self.reason = reason
     self.rlist = {}
@@ -236,6 +263,7 @@
   def sort_list(self): raise NotImplemented()
 
   def write_list(self, f):
+    "Write the list of failure counts for this reason 'f'"
     rlist = self.sort_list()
     for r in rlist:
       susp = 0
@@ -251,9 +279,11 @@
       f.write(str(susp)+"/"+str(tot_susp)+"\n")
     
   def add_r(self, r):
+    "Add a router to the list for this reason"
     self.rlist[r] = 1
 
   def total_suspected(self):
+    "Get a list of total suspected failures for this reason"
     # suspected is disjoint from failed. The failed table
     # may not have an entry
     def notlambda(x, y):
@@ -270,6 +300,7 @@
     return reduce(notlambda, self.rlist.iterkeys(), 0)
 
   def total_failed(self):
+    "Get a list of total failures for this reason"
     def notlambda(x, y):
       if self.reason in y.reason_failed:
         return (x + y.reason_failed[self.reason])
@@ -277,6 +308,9 @@
     return reduce(notlambda, self.rlist.iterkeys(), 0)
  
 class SuspectRouterList(ReasonRouterList):
+  """Helper class to track all routers suspected of failing for a given
+     reason. The main difference between this and the normal
+     ReasonRouterList is the sort order and the verification."""
   def __init__(self, reason): ReasonRouterList.__init__(self,reason)
   
   def sort_list(self):
@@ -290,6 +324,9 @@
             self.rlist.iterkeys(), 0)
 
 class FailedRouterList(ReasonRouterList):
+  """Helper class to track all routers that failed for a given
+     reason. The main difference between this and the normal
+     ReasonRouterList is the sort order and the verification."""
   def __init__(self, reason): ReasonRouterList.__init__(self,reason)
 
   def sort_list(self):
@@ -304,6 +341,8 @@
 
 
 class StatsHandler(PathSupport.PathBuilder):
+  """An extension of PathSupport.PathBuilder that keeps track of 
+     router statistics for every circuit and stream"""
   def __init__(self, c, slmgr):
     PathBuilder.__init__(self, c, slmgr, StatsRouter)
     self.circ_count = 0
@@ -314,6 +353,8 @@
     self.suspect_reasons = {}
 
   def run_zbtest(self): # Unweighted z-test
+    """Run unweighted z-test to calculate the probabilities of a node
+       having a given stream bandwidth based on the Normal distribution"""
     n = reduce(lambda x, y: x+(y.bwstats.mean > 0), self.sorted_r, 0)
     if n == 0: return (0, 0)
     avg = reduce(lambda x, y: x+y.bwstats.mean, self.sorted_r, 0)/float(n)
@@ -329,6 +370,9 @@
     return (avg, stddev)
 
   def run_zrtest(self): # Unweighted z-test
+    """Run unweighted z-test to calculate the probabilities of a node
+       having a given ratio of stream bandwidth to advertised bandwidth
+       based on the Normal distribution"""
     n = reduce(lambda x, y: x+(y.bw_ratio() > 0), self.sorted_r, 0)
     if n == 0: return (0, 0)
     avg = reduce(lambda x, y: x+y.bw_ratio(), self.sorted_r, 0)/float(n)
@@ -344,6 +388,7 @@
     return (avg, stddev)
 
   def write_reasons(self, f, reasons, name):
+    "Write out all the failure reasons and statistics for all Routers"
     f.write("\n\n\t----------------- "+name+" -----------------\n")
     for rsn in reasons:
       f.write("\n"+rsn.reason+". Failed: "+str(rsn.total_failed())
@@ -351,12 +396,14 @@
       rsn.write_list(f)
 
   def write_routers(self, f, rlist, name):
+    "Write out all the usage statistics for all Routers"
     f.write("\n\n\t----------------- "+name+" -----------------\n\n")
     for r in rlist:
       # only print it if we've used it.
       if r.circ_chosen+r.strm_chosen > 0: f.write(str(r))
 
   def write_stats(self, filename):
+    "Write out all the statistics the StatsHandler has gathered"
     # TODO: all this shit should be configurable. Some of it only makes
     # sense when scanning in certain modes.
     plog("DEBUG", "Writing stats")
@@ -520,6 +567,7 @@
     PathBuilder.circ_status_event(self, c)
 
   def count_stream_reason_failed(self, s, reason):
+    "Count the routers involved in a failure"
     # Update failed count,reason_failed for exit
     r = self.circuits[s.circ_id].exit
     if not reason in r.reason_failed: r.reason_failed[reason] = 1
@@ -530,6 +578,7 @@
     self.failed_reasons[reason].add_r(r)
 
   def count_stream_suspects(self, s, lreason, reason):
+    "Count the routers 'suspected' of being involved in a failure"
     if lreason in ("TIMEOUT", "INTERNAL", "TORPROTOCOL" "DESTROY"):
       for r in self.circuits[s.circ_id].path[:-1]:
         r.strm_suspected += 1
@@ -635,6 +684,7 @@
     plog("DEBUG", msg)
  
 def commandloop(s, c, h):
+  "The main metatroller listener loop"
   s.write("220 Welcome to the Tor Metatroller "+mt_version+"! Try HELP for Info\r\n\r\n")
   while 1:
     buf = s.readline()



More information about the tor-commits mailing list