[or-cvs] r15742: Add external property. Keep TorCtl.local just in case. (in torflow/branches/gsoc2008: . TorCtl.local)
mikeperry at seul.org
mikeperry at seul.org
Mon Jul 7 19:37:48 UTC 2008
Author: mikeperry
Date: 2008-07-07 15:37:47 -0400 (Mon, 07 Jul 2008)
New Revision: 15742
Added:
torflow/branches/gsoc2008/TorCtl.local/
torflow/branches/gsoc2008/TorCtl.local/GeoIPSupport.py
torflow/branches/gsoc2008/TorCtl.local/PathSupport.py
torflow/branches/gsoc2008/TorCtl.local/README
torflow/branches/gsoc2008/TorCtl.local/StatsSupport.py
torflow/branches/gsoc2008/TorCtl.local/TorCtl.py
torflow/branches/gsoc2008/TorCtl.local/TorUtil.py
torflow/branches/gsoc2008/TorCtl.local/__init__.py
Removed:
torflow/branches/gsoc2008/TorCtl.local/GeoIPSupport.py
torflow/branches/gsoc2008/TorCtl.local/PathSupport.py
torflow/branches/gsoc2008/TorCtl.local/README
torflow/branches/gsoc2008/TorCtl.local/StatsSupport.py
torflow/branches/gsoc2008/TorCtl.local/TorCtl.py
torflow/branches/gsoc2008/TorCtl.local/TorUtil.py
torflow/branches/gsoc2008/TorCtl.local/__init__.py
torflow/branches/gsoc2008/TorCtl/
Modified:
torflow/branches/gsoc2008/
Log:
Add external property. Keep TorCtl.local just in case.
Property changes on: torflow/branches/gsoc2008
___________________________________________________________________
Name: svn:externals
+ TorCtl https://tor-svn.freehaven.net/svn/torctl/trunk/python/TorCtl
Copied: torflow/branches/gsoc2008/TorCtl.local (from rev 15734, torflow/branches/gsoc2008/TorCtl)
Deleted: torflow/branches/gsoc2008/TorCtl.local/GeoIPSupport.py
===================================================================
--- torflow/branches/gsoc2008/TorCtl/GeoIPSupport.py 2008-07-07 16:40:57 UTC (rev 15734)
+++ torflow/branches/gsoc2008/TorCtl.local/GeoIPSupport.py 2008-07-07 19:37:47 UTC (rev 15742)
@@ -1,132 +0,0 @@
-#!/usr/bin/python
-
-import struct
-import socket
-import GeoIP
-import TorCtl
-
-from TorUtil import plog
-
-# GeoIP data object: choose database here
-geoip = GeoIP.new(GeoIP.GEOIP_STANDARD)
-#geoip = GeoIP.open("./GeoLiteCity.dat", GeoIP.GEOIP_STANDARD)
-
-class Continent:
- """ Continent class: The group attribute is to partition the continents
- in groups, to determine the number of ocean crossings """
- def __init__(self, continent_code):
- self.code = continent_code
- self.group = None
- self.countries = []
-
- def contains(self, country_code):
- return country_code in self.countries
-
-# Set countries to continents
-africa = Continent("AF")
-africa.group = 1
-africa.countries = ["AO","BF","BI","BJ","BV","BW","CD","CF","CG","CI","CM",
- "CV","DJ","DZ","EG","EH","ER","ET","GA","GH","GM","GN","GQ","GW","HM","KE",
- "KM","LR","LS","LY","MA","MG","ML","MR","MU","MW","MZ","NA","NE","NG","RE",
- "RW","SC","SD","SH","SL","SN","SO","ST","SZ","TD","TF","TG","TN","TZ","UG",
- "YT","ZA","ZM","ZR","ZW"]
-
-asia = Continent("AS")
-asia.group = 1
-asia.countries = ["AP","AE","AF","AM","AZ","BD","BH","BN","BT","CC","CN","CX",
- "CY","GE","HK","ID","IL","IN","IO","IQ","IR","JO","JP","KG","KH","KP","KR",
- "KW","KZ","LA","LB","LK","MM","MN","MO","MV","MY","NP","OM","PH","PK","PS",
- "QA","RU","SA","SG","SY","TH","TJ","TM","TP","TR","TW","UZ","VN","YE"]
-
-europe = Continent("EU")
-europe.group = 1
-europe.countries = ["EU","AD","AL","AT","BA","BE","BG","BY","CH","CZ","DE",
- "DK","EE","ES","FI","FO","FR","FX","GB","GI","GR","HR","HU","IE","IS","IT",
- "LI","LT","LU","LV","MC","MD","MK","MT","NL","NO","PL","PT","RO","SE","SI",
- "SJ","SK","SM","UA","VA","YU"]
-
-oceania = Continent("OC")
-oceania.group = 2
-oceania.countries = ["AS","AU","CK","FJ","FM","GU","KI","MH","MP","NC","NF",
- "NR","NU","NZ","PF","PG","PN","PW","SB","TK","TO","TV","UM","VU","WF","WS"]
-
-north_america = Continent("NA")
-north_america.group = 0
-north_america.countries = ["CA","MX","US"]
-
-south_america = Continent("SA")
-south_america.group = 0
-south_america.countries = ["AG","AI","AN","AR","AW","BB","BM","BO","BR","BS",
- "BZ","CL","CO","CR","CU","DM","DO","EC","FK","GD","GF","GL","GP","GS","GT",
- "GY","HN","HT","JM","KN","KY","LC","MQ","MS","NI","PA","PE","PM","PR","PY",
- "SA","SR","SV","TC","TT","UY","VC","VE","VG","VI"]
-
-# List of continents
-continents = [africa, asia, europe, north_america, oceania, south_america]
-
-def get_continent(country_code):
- """ Perform country -- continent mapping """
- for c in continents:
- if c.contains(country_code):
- return c
- plog("INFO", country_code + " is not on any continent")
- return None
-
-def get_country(ip):
- """ Get the country via the library """
- return geoip.country_code_by_addr(ip)
-
-def get_country_from_record(ip):
- """ Get the country code out of a GeoLiteCity record (not used) """
- record = geoip.record_by_addr(ip)
- if record != None:
- return record['country_code']
-
-class GeoIPRouter(TorCtl.Router):
- """ Router class extended to GeoIP """
- def __init__(self, router):
- self.__dict__ = router.__dict__
- self.country_code = get_country(self.get_ip_dotted())
- if self.country_code != None:
- c = get_continent(self.country_code)
- if c != None:
- self.continent = c.code
- self.cont_group = c.group
- else:
- plog("INFO", self.nickname + ": Country code not found")
- self.continent = None
-
- def get_ip_dotted(self):
- """ Convert long int back to dotted quad string """
- return socket.inet_ntoa(struct.pack('>I', self.ip))
-
-class GeoIPConfig:
- """ Class to configure GeoIP-based path building """
- def __init__(self, unique_countries, continent_crossings, ocean_crossings,
- entry_country, middle_country, exit_country, excludes):
- # TODO: Somehow ensure validity of a configuration:
- # - continent_crossings >= ocean_crossings
- # - unique_countries=False --> continent_crossings!=None
- # - echelon? set entry_country to source and exit_country to None
-
- # Do not use a country twice in a route
- # [True --> unique, False --> same or None --> pass]
- self.unique_countries = unique_countries
-
- # Configure max continent crossings in one path
- # [integer number 0-n or None --> ContinentJumper/UniqueContinent]
- self.continent_crossings = continent_crossings
- self.ocean_crossings = ocean_crossings
-
- # Try to find an exit node in the destination country
- # use exit_country as backup, if country cannot not be found
- self.echelon = False
-
- # Specify countries for positions [single country code or None]
- self.entry_country = entry_country
- self.middle_country = middle_country
- self.exit_country = exit_country
-
- # List of countries not to use in routes
- # [(empty) list of country codes or None]
- self.excludes = excludes
Copied: torflow/branches/gsoc2008/TorCtl.local/GeoIPSupport.py (from rev 15741, torflow/branches/gsoc2008/TorCtl/GeoIPSupport.py)
===================================================================
--- torflow/branches/gsoc2008/TorCtl.local/GeoIPSupport.py (rev 0)
+++ torflow/branches/gsoc2008/TorCtl.local/GeoIPSupport.py 2008-07-07 19:37:47 UTC (rev 15742)
@@ -0,0 +1,132 @@
+#!/usr/bin/python
+
+import struct
+import socket
+import GeoIP
+import TorCtl
+
+from TorUtil import plog
+
+# GeoIP data object: choose database here
+geoip = GeoIP.new(GeoIP.GEOIP_STANDARD)
+#geoip = GeoIP.open("./GeoLiteCity.dat", GeoIP.GEOIP_STANDARD)
+
+class Continent:
+ """ Continent class: The group attribute is to partition the continents
+ in groups, to determine the number of ocean crossings """
+ def __init__(self, continent_code):
+ self.code = continent_code
+ self.group = None
+ self.countries = []
+
+ def contains(self, country_code):
+ return country_code in self.countries
+
+# Set countries to continents
+africa = Continent("AF")
+africa.group = 1
+africa.countries = ["AO","BF","BI","BJ","BV","BW","CD","CF","CG","CI","CM",
+ "CV","DJ","DZ","EG","EH","ER","ET","GA","GH","GM","GN","GQ","GW","HM","KE",
+ "KM","LR","LS","LY","MA","MG","ML","MR","MU","MW","MZ","NA","NE","NG","RE",
+ "RW","SC","SD","SH","SL","SN","SO","ST","SZ","TD","TF","TG","TN","TZ","UG",
+ "YT","ZA","ZM","ZR","ZW"]
+
+asia = Continent("AS")
+asia.group = 1
+asia.countries = ["AP","AE","AF","AM","AZ","BD","BH","BN","BT","CC","CN","CX",
+ "CY","GE","HK","ID","IL","IN","IO","IQ","IR","JO","JP","KG","KH","KP","KR",
+ "KW","KZ","LA","LB","LK","MM","MN","MO","MV","MY","NP","OM","PH","PK","PS",
+ "QA","RU","SA","SG","SY","TH","TJ","TM","TP","TR","TW","UZ","VN","YE"]
+
+europe = Continent("EU")
+europe.group = 1
+europe.countries = ["EU","AD","AL","AT","BA","BE","BG","BY","CH","CZ","DE",
+ "DK","EE","ES","FI","FO","FR","FX","GB","GI","GR","HR","HU","IE","IS","IT",
+ "LI","LT","LU","LV","MC","MD","MK","MT","NL","NO","PL","PT","RO","SE","SI",
+ "SJ","SK","SM","UA","VA","YU"]
+
+oceania = Continent("OC")
+oceania.group = 2
+oceania.countries = ["AS","AU","CK","FJ","FM","GU","KI","MH","MP","NC","NF",
+ "NR","NU","NZ","PF","PG","PN","PW","SB","TK","TO","TV","UM","VU","WF","WS"]
+
+north_america = Continent("NA")
+north_america.group = 0
+north_america.countries = ["CA","MX","US"]
+
+south_america = Continent("SA")
+south_america.group = 0
+south_america.countries = ["AG","AI","AN","AR","AW","BB","BM","BO","BR","BS",
+ "BZ","CL","CO","CR","CU","DM","DO","EC","FK","GD","GF","GL","GP","GS","GT",
+ "GY","HN","HT","JM","KN","KY","LC","MQ","MS","NI","PA","PE","PM","PR","PY",
+ "SA","SR","SV","TC","TT","UY","VC","VE","VG","VI"]
+
+# List of continents
+continents = [africa, asia, europe, north_america, oceania, south_america]
+
+def get_continent(country_code):
+ """ Perform country -- continent mapping """
+ for c in continents:
+ if c.contains(country_code):
+ return c
+ plog("INFO", country_code + " is not on any continent")
+ return None
+
+def get_country(ip):
+ """ Get the country via the library """
+ return geoip.country_code_by_addr(ip)
+
+def get_country_from_record(ip):
+ """ Get the country code out of a GeoLiteCity record (not used) """
+ record = geoip.record_by_addr(ip)
+ if record != None:
+ return record['country_code']
+
+class GeoIPRouter(TorCtl.Router):
+ """ Router class extended to GeoIP """
+ def __init__(self, router):
+ self.__dict__ = router.__dict__
+ self.country_code = get_country(self.get_ip_dotted())
+ if self.country_code != None:
+ c = get_continent(self.country_code)
+ if c != None:
+ self.continent = c.code
+ self.cont_group = c.group
+ else:
+ plog("INFO", self.nickname + ": Country code not found")
+ self.continent = None
+
+ def get_ip_dotted(self):
+ """ Convert long int back to dotted quad string """
+ return socket.inet_ntoa(struct.pack('>I', self.ip))
+
+class GeoIPConfig:
+ """ Class to configure GeoIP-based path building """
+ def __init__(self, unique_countries, continent_crossings, ocean_crossings,
+ entry_country, middle_country, exit_country, excludes):
+ # TODO: Somehow ensure validity of a configuration:
+ # - continent_crossings >= ocean_crossings
+ # - unique_countries=False --> continent_crossings!=None
+ # - echelon? set entry_country to source and exit_country to None
+
+ # Do not use a country twice in a route
+ # [True --> unique, False --> same or None --> pass]
+ self.unique_countries = unique_countries
+
+ # Configure max continent crossings in one path
+ # [integer number 0-n or None --> ContinentJumper/UniqueContinent]
+ self.continent_crossings = continent_crossings
+ self.ocean_crossings = ocean_crossings
+
+ # Try to find an exit node in the destination country
+ # use exit_country as backup, if country cannot not be found
+ self.echelon = False
+
+ # Specify countries for positions [single country code or None]
+ self.entry_country = entry_country
+ self.middle_country = middle_country
+ self.exit_country = exit_country
+
+ # List of countries not to use in routes
+ # [(empty) list of country codes or None]
+ self.excludes = excludes
Deleted: torflow/branches/gsoc2008/TorCtl.local/PathSupport.py
===================================================================
--- torflow/branches/gsoc2008/TorCtl/PathSupport.py 2008-07-07 16:40:57 UTC (rev 15734)
+++ torflow/branches/gsoc2008/TorCtl.local/PathSupport.py 2008-07-07 19:37:47 UTC (rev 15742)
@@ -1,1712 +0,0 @@
-#!/usr/bin/python
-"""
-
-Support classes for path construction
-
-The PathSupport package builds on top of TorCtl.TorCtl. It provides a
-number of interfaces that make path construction easier.
-
-The inheritance diagram for event handling is as follows:
-TorCtl.EventHandler <- PathBuilder <- CircuitHandler <- StreamHandler.
-
-Basically, EventHandler is what gets all the control port events
-packaged in nice clean classes (see help(TorCtl) for information on
-those).
-
-PathBuilder inherits from EventHandler and is what builds all circuits
-based on the requirements specified in the SelectionManager instance
-passed to its constructor. It also handles attaching streams to
-circuits. It only handles one building one circuit at a time.
-
-CircuitHandler optionally inherits from PathBuilder, and overrides its
-circuit event handling to manage building a pool of circuits as opposed
-to just one. It still uses the SelectionManager for path selection.
-
-StreamHandler inherits from CircuitHandler, and is what governs the
-attachment of an incoming stream on to one of the multiple circuits of
-the circuit handler.
-
-The SelectionManager is essentially a configuration wrapper around the
-most elegant portions of TorFlow: NodeGenerators, NodeRestrictions, and
-PathRestrictions. In the SelectionManager, a NodeGenerator is used to
-choose the nodes probabilistically according to some distribution while
-obeying the NodeRestrictions. These generators (one per hop) are handed
-off to the PathSelector, which uses the generators to build a complete
-path that satisfies the PathRestriction requirements.
-
-Have a look at the class hierarchy directly below to get a feel for how
-the restrictions fit together, and what options are available.
-
-"""
-
-import TorCtl
-import re
-import struct
-import random
-import socket
-import copy
-import Queue
-import time
-import TorUtil
-from TorUtil import *
-
-__all__ = ["NodeRestrictionList", "PathRestrictionList",
-"PercentileRestriction", "OSRestriction", "ConserveExitsRestriction",
-"FlagsRestriction", "MinBWRestriction", "VersionIncludeRestriction",
-"VersionExcludeRestriction", "ExitPolicyRestriction", "NodeRestriction",
-"PathRestriction", "OrNodeRestriction", "MetaNodeRestriction",
-"AtLeastNNodeRestriction", "NotNodeRestriction", "Subnet16Restriction",
-"UniqueRestriction", "NodeGenerator", "UniformGenerator",
-"OrderedExitGenerator", "BwWeightedGenerator", "PathSelector",
-"Connection", "NickRestriction", "IdHexRestriction", "PathBuilder",
-"CircuitHandler", "StreamHandler", "SelectionManager",
-"CountryCodeRestriction", "CountryRestriction",
-"UniqueCountryRestriction", "SingleCountryRestriction",
-"ContinentRestriction", "ContinentJumperRestriction",
-"UniqueContinentRestriction"]
-
-#################### Path Support Interfaces #####################
-
-class NodeRestriction:
- "Interface for node restriction policies"
- def r_is_ok(self, r):
- "Returns true if Router 'r' is acceptable for this restriction"
- return True
-
-class NodeRestrictionList:
- "Class to manage a list of NodeRestrictions"
- def __init__(self, restrictions):
- "Constructor. 'restrictions' is a list of NodeRestriction instances"
- self.restrictions = restrictions
-
- def r_is_ok(self, r):
- "Returns true of Router 'r' passes all of the contained restrictions"
- for rs in self.restrictions:
- if not rs.r_is_ok(r): return False
- return True
-
- def add_restriction(self, restr):
- "Add a NodeRestriction 'restr' to the list of restrictions"
- self.restrictions.append(restr)
-
- # TODO: This does not collapse meta restrictions..
- def del_restriction(self, RestrictionClass):
- """Remove all restrictions of type RestrictionClass from the list.
- Does NOT inspect or collapse MetaNode Restrictions (though
- MetaRestrictions can be removed if RestrictionClass is
- MetaNodeRestriction)"""
- self.restrictions = filter(
- lambda r: not isinstance(r, RestrictionClass),
- self.restrictions)
-
-class PathRestriction:
- "Interface for path restriction policies"
- def path_is_ok(self, path):
- "Return true if the list of Routers in path satisfies this restriction"
- return True
-
-class PathRestrictionList:
- """Class to manage a list of PathRestrictions"""
- def __init__(self, restrictions):
- "Constructor. 'restrictions' is a list of PathRestriction instances"
- self.restrictions = restrictions
-
- def path_is_ok(self, path):
- "Given list if Routers in 'path', check it against each restriction."
- for rs in self.restrictions:
- if not rs.path_is_ok(path):
- return False
- return True
-
- def add_restriction(self, rstr):
- "Add a PathRestriction 'rstr' to the list"
- self.restrictions.append(rstr)
-
- def del_restriction(self, RestrictionClass):
- "Remove all PathRestrictions of type RestrictionClass from the list."
- self.restrictions = filter(
- lambda r: not isinstance(r, RestrictionClass),
- self.restrictions)
-
-class NodeGenerator:
- "Interface for node generation"
- def __init__(self, sorted_r, rstr_list):
- """Constructor. Takes a bandwidth-sorted list of Routers 'sorted_r'
- and a NodeRestrictionList 'rstr_list'"""
- self.rstr_list = rstr_list # Check me before you yield!
- self.sorted_r = sorted_r
- self.rewind()
-
- def reset_restriction(self, rstr_list):
- "Reset the restriction list to a new list"
- self.rstr_list = rstr_list
-
- def rewind(self):
- "Rewind the generator to the 'beginning'"
- # FIXME: If we apply the restrictions now, we can save cycles
- # during selection, and also some memory overhead (at the cost
- # of a much slower rewind() though..)
- self.routers = filter(lambda r: self.rstr_list.r_is_ok(r), self.sorted_r)
- #self.routers = copy.copy(self.sorted_r)
-
- def mark_chosen(self, r):
- """Mark a router as chosen: remove it from the list of routers
- that can be returned in the future"""
- self.routers.remove(r)
-
- def all_chosen(self):
- "Return true if all the routers have been marked as chosen"
- return not self.routers
-
- def generate(self):
- "Return a python generator that yields routers according to the policy"
- raise NotImplemented()
-
-class Connection(TorCtl.Connection):
- """Extended Connection class that provides a method for building circuits"""
- def __init__(self, sock):
- TorCtl.Connection.__init__(self,sock)
- def build_circuit(self, pathlen, path_sel):
- "Tell Tor to build a circuit chosen by the PathSelector 'path_sel'"
- circ = Circuit()
- circ.path = path_sel.build_path(pathlen)
- circ.exit = circ.path[pathlen-1]
- circ.circ_id = self.extend_circuit(0, circ.id_path())
- return circ
-
-######################## Node Restrictions ########################
-
-# TODO: We still need more path support implementations
-# - NodeRestrictions:
-# - Uptime/LongLivedPorts (Does/should hibernation count?)
-# - Published/Updated
-# - PathRestrictions:
-# - Family
-# - GeoIP:
-# - OceanPhobicRestrictor (avoids Pacific Ocean or two atlantic crossings)
-# or ContinentRestrictor (avoids doing more than N continent crossings)
-# - Mathematical/empirical study of predecessor expectation
-# - If middle node on the same continent as exit, exit learns nothing
-# - else, exit has a bias on the continent of origin of user
-# - Language and browser accept string determine this anyway
-# - EchelonPhobicRestrictor
-# - Does not cross international boundaries for client->Entry or
-# Exit->destination hops
-
-class PercentileRestriction(NodeRestriction):
- """Restriction to cut out a percentile slice of the network."""
- def __init__(self, pct_skip, pct_fast, r_list):
- """Constructor. Sets up the restriction such that routers in the
- 'pct_skip' to 'pct_fast' percentile of bandwidth rankings are
- returned from the sorted list 'r_list'"""
- self.pct_fast = pct_fast
- self.pct_skip = pct_skip
- self.sorted_r = r_list
-
- def r_is_ok(self, r):
- "Returns true if r is in the percentile boundaries (by rank)"
- # Hrmm.. technically we shouldn't count non-running routers in this..
- # but that is tricky to do efficiently.
- # XXX: Is there any reason why sorted_r should have non-running
- # routers in the first place?
-
- if r.list_rank < len(self.sorted_r)*self.pct_skip/100: return False
- elif r.list_rank > len(self.sorted_r)*self.pct_fast/100: return False
-
- return True
-
-class OSRestriction(NodeRestriction):
- "Restriction based on operating system"
- def __init__(self, ok, bad=[]):
- """Constructor. Accept router OSes that match regexes in 'ok',
- rejects those that match regexes in 'bad'."""
- self.ok = ok
- self.bad = bad
-
- def r_is_ok(self, r):
- "Returns true if r is in 'ok', false if 'r' is in 'bad'. If 'ok'"
- for y in self.ok:
- if re.search(y, r.os):
- return True
- for b in self.bad:
- if re.search(b, r.os):
- return False
- if self.ok: return False
- if self.bad: return True
-
-class ConserveExitsRestriction(NodeRestriction):
- "Restriction to reject exits from selection"
- # XXX: Make this adaptive by ip/port
- def r_is_ok(self, r): return not "Exit" in r.flags
-
-class FlagsRestriction(NodeRestriction):
- "Restriction for mandatory and forbidden router flags"
- def __init__(self, mandatory, forbidden=[]):
- """Constructor. 'mandatory' and 'forbidden' are both lists of router
- flags as strings."""
- self.mandatory = mandatory
- self.forbidden = forbidden
-
- def r_is_ok(self, router):
- for m in self.mandatory:
- if not m in router.flags: return False
- for f in self.forbidden:
- if f in router.flags: return False
- return True
-
-class NickRestriction(NodeRestriction):
- """Require that the node nickname is as specified"""
- def __init__(self, nickname):
- self.nickname = nickname
-
- def r_is_ok(self, router):
- return router.nickname == self.nickname
-
-class IdHexRestriction(NodeRestriction):
- """Require that the node idhash is as specified"""
- def __init__(self, idhex):
- if idhex[0] == '$':
- self.idhex = idhex[1:].upper()
- else:
- self.idhex = idhex.upper()
-
- def r_is_ok(self, router):
- return router.idhex == self.idhex
-
-class MinBWRestriction(NodeRestriction):
- """Require a minimum bandwidth"""
- def __init__(self, minbw):
- self.min_bw = minbw
-
- def r_is_ok(self, router): return router.bw >= self.min_bw
-
-class VersionIncludeRestriction(NodeRestriction):
- """Require that the version match one in the list"""
- def __init__(self, eq):
- "Constructor. 'eq' is a list of versions as strings"
- self.eq = map(TorCtl.RouterVersion, eq)
-
- def r_is_ok(self, router):
- """Returns true if the version of 'router' matches one of the
- specified versions."""
- for e in self.eq:
- if e == router.version:
- return True
- return False
-
-class VersionExcludeRestriction(NodeRestriction):
- """Require that the version not match one in the list"""
- def __init__(self, exclude):
- "Constructor. 'exclude' is a list of versions as strings"
- self.exclude = map(TorCtl.RouterVersion, exclude)
-
- def r_is_ok(self, router):
- """Returns false if the version of 'router' matches one of the
- specified versions."""
- for e in self.exclude:
- if e == router.version:
- return False
- return True
-
-class VersionRangeRestriction(NodeRestriction):
- """Require that the versions be inside a specified range"""
- def __init__(self, gr_eq, less_eq=None):
- self.gr_eq = TorCtl.RouterVersion(gr_eq)
- if less_eq: self.less_eq = TorCtl.RouterVersion(less_eq)
- else: self.less_eq = None
-
- def r_is_ok(self, router):
- return (not self.gr_eq or router.version >= self.gr_eq) and \
- (not self.less_eq or router.version <= self.less_eq)
-
-class ExitPolicyRestriction(NodeRestriction):
- """Require that a router exit to an ip+port"""
- def __init__(self, to_ip, to_port):
- self.to_ip = to_ip
- self.to_port = to_port
-
- def r_is_ok(self, r): return r.will_exit_to(self.to_ip, self.to_port)
-
-class MetaNodeRestriction(NodeRestriction):
- """Interface for a NodeRestriction that is an expression consisting of
- multiple other NodeRestrictions"""
- # TODO: these should collapse the restriction and return a new
- # instance for re-insertion (or None)
- def next_rstr(self): raise NotImplemented()
- def del_restriction(self, RestrictionClass): raise NotImplemented()
-
-class OrNodeRestriction(MetaNodeRestriction):
- """MetaNodeRestriction that is the boolean or of two or more
- NodeRestrictions"""
- def __init__(self, rs):
- "Constructor. 'rs' is a list of NodeRestrictions"
- self.rstrs = rs
-
- def r_is_ok(self, r):
- "Returns true if one of 'rs' is true for this router"
- for rs in self.rstrs:
- if rs.r_is_ok(r):
- return True
- return False
-
-class NotNodeRestriction(MetaNodeRestriction):
- """Negates a single restriction"""
- def __init__(self, a):
- self.a = a
-
- def r_is_ok(self, r): return not self.a.r_is_ok(r)
-
-class AtLeastNNodeRestriction(MetaNodeRestriction):
- """MetaNodeRestriction that is true if at least n member
- restrictions are true."""
- def __init__(self, rstrs, n):
- self.rstrs = rstrs
- self.n = n
-
- def r_is_ok(self, r):
- cnt = 0
- for rs in self.rstrs:
- if rs.r_is_ok(r):
- cnt += 1
- if cnt < self.n: return False
- else: return True
-
-
-#################### Path Restrictions #####################
-
-class Subnet16Restriction(PathRestriction):
- """PathRestriction that mandates that no two nodes from the same
- /16 subnet be in the path"""
- def path_is_ok(self, path):
- mask16 = struct.unpack(">I", socket.inet_aton("255.255.0.0"))[0]
- ip16 = path[0].ip & mask16
- for r in path[1:]:
- if ip16 == (r.ip & mask16):
- return False
- return True
-
-class UniqueRestriction(PathRestriction):
- """Path restriction that mandates that the same router can't appear more
- than once in a path"""
- def path_is_ok(self, path):
- for i in xrange(0,len(path)):
- if path[i] in path[:i]:
- return False
- return True
-
-#################### GeoIP Restrictions ###################
-
-class CountryCodeRestriction(NodeRestriction):
- """ Ensure that the country_code is set """
- def r_is_ok(self, r):
- return r.country_code != None
-
-class CountryRestriction(NodeRestriction):
- """ Only accept nodes that are in 'country_code' """
- def __init__(self, country_code):
- self.country_code = country_code
-
- def r_is_ok(self, r):
- return r.country_code == self.country_code
-
-class ExcludeCountriesRestriction(NodeRestriction):
- """ Exclude a list of countries """
- def __init__(self, countries):
- self.countries = countries
-
- def r_is_ok(self, r):
- return not (r.country_code in self.countries)
-
-class UniqueCountryRestriction(PathRestriction):
- """ Ensure every router to have a distinct country_code """
- def path_is_ok(self, path):
- for i in xrange(0, len(path)-1):
- for j in xrange(i+1, len(path)):
- if path[i].country_code == path[j].country_code:
- return False;
- return True;
-
-class SingleCountryRestriction(PathRestriction):
- """ Ensure every router to have the same country_code """
- def path_is_ok(self, path):
- country_code = path[0].country_code
- for r in path:
- if country_code != r.country_code:
- return False
- return True
-
-class ContinentRestriction(PathRestriction):
- """ Do not more than n continent crossings """
- # TODO: Add src and dest
- def __init__(self, n, src=None, dest=None):
- self.n = n
-
- def path_is_ok(self, path):
- crossings = 0
- prev = None
- # Compute crossings until now
- for r in path:
- # Jump over the first router
- if prev:
- if r.continent != prev.continent:
- crossings += 1
- prev = r
- if crossings > self.n: return False
- else: return True
-
-class ContinentJumperRestriction(PathRestriction):
- """ Ensure continent crossings between all hops """
- def path_is_ok(self, path):
- prev = None
- for r in path:
- # Jump over the first router
- if prev:
- if r.continent == prev.continent:
- return False
- prev = r
- return True
-
-class UniqueContinentRestriction(PathRestriction):
- """ Ensure every hop to be on a different continent """
- def path_is_ok(self, path):
- for i in xrange(0, len(path)-1):
- for j in xrange(i+1, len(path)):
- if path[i].continent == path[j].continent:
- return False;
- return True;
-
-class OceanPhobicRestriction(PathRestriction):
- """ Not more than n ocean crossings """
- # TODO: Add src and dest
- def __init__(self, n, src=None, dest=None):
- self.n = n
-
- def path_is_ok(self, path):
- crossings = 0
- prev = None
- # Compute ocean crossings until now
- for r in path:
- # Jump over the first router
- if prev:
- if r.cont_group != prev.cont_group:
- crossings += 1
- prev = r
- if crossings > self.n: return False
- else: return True
-
-#################### Node Generators ######################
-
-class UniformGenerator(NodeGenerator):
- """NodeGenerator that produces nodes in the uniform distribution"""
- def generate(self):
- while not self.all_chosen():
- r = random.choice(self.routers)
- if self.rstr_list.r_is_ok(r): yield r
-
-class OrderedExitGenerator(NodeGenerator):
- """NodeGenerator that produces exits in an ordered fashion for a
- specific port"""
- def __init__(self, to_port, sorted_r, rstr_list):
- self.to_port = to_port
- self.next_exit_by_port = {}
- NodeGenerator.__init__(self, sorted_r, rstr_list)
-
- def rewind(self):
- if self.to_port not in self.next_exit_by_port or not self.next_exit_by_port[self.to_port]:
- self.next_exit_by_port[self.to_port] = 0
- self.last_idx = len(self.sorted_r)
- else:
- self.last_idx = self.next_exit_by_port[self.to_port]
-
- def set_port(self, port):
- self.to_port = port
- self.rewind()
-
- def mark_chosen(self, r):
- self.next_exit_by_port[self.to_port] += 1
-
- def all_chosen(self):
- return self.last_idx == self.next_exit_by_port[self.to_port]
-
- def generate(self):
- while True: # A do..while would be real nice here..
- if self.next_exit_by_port[self.to_port] >= len(self.sorted_r):
- self.next_exit_by_port[self.to_port] = 0
- r = self.sorted_r[self.next_exit_by_port[self.to_port]]
- if self.rstr_list.r_is_ok(r): yield r
- else: self.next_exit_by_port[self.to_port] += 1
- if self.last_idx == self.next_exit_by_port[self.to_port]:
- break
-
-class BwWeightedGenerator(NodeGenerator):
- """
-
- This is a generator designed to match the Tor Path Selection
- algorithm. It will generate nodes weighted by their bandwidth,
- but take the appropriate weighting into account against guard
- nodes and exit nodes when they are chosen for positions other
- than guard/exit. For background see:
- routerlist.c::smartlist_choose_by_bandwidth(),
- http://archives.seul.org/or/dev/Jul-2007/msg00021.html,
- http://archives.seul.org/or/dev/Jul-2007/msg00056.html, and
- https://tor-svn.freehaven.net/svn/tor/trunk/doc/spec/path-spec.txt
- The formulas used are from the first or-dev link, but are proven
- optimal and equivalent to the ones now used in routerlist.c in the
- second or-dev link.
-
- """
- def __init__(self, sorted_r, rstr_list, pathlen, exit=False, guard=False):
- """ Pass exit=True to create a generator for exit-nodes """
- self.max_bandwidth = 10000000
- # Out for an exit-node?
- self.exit = exit
- # Is this a guard node?
- self.guard = guard
- # Different sums of bandwidths
- self.total_bw = 0
- self.total_exit_bw = 0
- self.total_guard_bw = 0
- self.total_weighted_bw = 0
- self.pathlen = pathlen
- NodeGenerator.__init__(self, sorted_r, rstr_list)
-
- def rewind(self):
- NodeGenerator.rewind(self)
- # Set the exit_weight
- # We are choosing a non-exit
- self.total_exit_bw = 0
- self.total_guard_bw = 0
- self.total_bw = 0
- for r in self.sorted_r:
- # Should this be outside the restriction checks?
- # TODO: Check max_bandwidth and cap...
- if self.rstr_list.r_is_ok(r):
- self.total_bw += r.bw
- if "Exit" in r.flags:
- self.total_exit_bw += r.bw
- if "Guard" in r.flags:
- self.total_guard_bw += r.bw
-
- bw_per_hop = (1.0*self.total_bw)/self.pathlen
-
- # Print some debugging info about bandwidth ratios
- if self.total_bw > 0:
- e_ratio = self.total_exit_bw/float(self.total_bw)
- g_ratio = self.total_guard_bw/float(self.total_bw)
- else:
- g_ratio = 0
- e_ratio = 0
- plog("DEBUG",
- "E = " + str(self.total_exit_bw) +
- ", G = " + str(self.total_guard_bw) +
- ", T = " + str(self.total_bw) +
- ", g_ratio = " + str(g_ratio) + ", e_ratio = " +str(e_ratio) +
- ", bw_per_hop = " + str(bw_per_hop))
-
- if self.exit:
- self.exit_weight = 1.0
- else:
- if self.total_exit_bw < bw_per_hop:
- # Don't use exit nodes at all
- self.exit_weight = 0
- else:
- if self.total_exit_bw > 0:
- self.exit_weight = ((self.total_exit_bw-bw_per_hop)/self.total_exit_bw)
- else: self.exit_weight = 0
-
- if self.guard:
- self.guard_weight = 1.0
- else:
- if self.total_guard_bw < bw_per_hop:
- # Don't use exit nodes at all
- self.guard_weight = 0
- else:
- if self.total_guard_bw > 0:
- self.guard_weight = ((self.total_guard_bw-bw_per_hop)/self.total_guard_bw)
- else: self.guard_weight = 0
-
- for r in self.sorted_r:
- bw = r.bw
- if "Exit" in r.flags:
- bw *= self.exit_weight
- if "Guard" in r.flags:
- bw *= self.guard_weight
- self.total_weighted_bw += bw
-
- self.total_weighted_bw = int(self.total_weighted_bw)
- plog("DEBUG", "Bw: "+str(self.total_weighted_bw)+"/"+str(self.total_bw)
- +". The exit-weight is: "+str(self.exit_weight)
- + ", guard weight is: "+str(self.guard_weight))
-
- def generate(self):
- while True:
- # Choose a suitable random int
- i = random.randint(0, self.total_weighted_bw)
-
- # Go through the routers
- for r in self.routers:
- # Below zero here means next() -> choose a new random int+router
- if i < 0: break
- if self.rstr_list.r_is_ok(r):
- bw = r.bw
- if "Exit" in r.flags:
- bw *= self.exit_weight
- if "Guard" in r.flags:
- bw *= self.guard_weight
-
- i -= bw
- if i < 0:
- plog("DEBUG", "Chosen router with a bandwidth of: " + str(r.bw))
- yield r
-
-####################### Secret Sauce ###########################
-
-class PathError(Exception):
- pass
-
-class NoRouters(PathError):
- pass
-
-class PathSelector:
- """Implementation of path selection policies. Builds a path according
- to entry, middle, and exit generators that satisfies the path
- restrictions."""
- def __init__(self, entry_gen, mid_gen, exit_gen, path_restrict):
- """Constructor. The first three arguments are NodeGenerators with
- their appropriate restrictions. The 'path_restrict' is a
- PathRestrictionList"""
- self.entry_gen = entry_gen
- self.mid_gen = mid_gen
- self.exit_gen = exit_gen
- self.path_restrict = path_restrict
-
- def build_path(self, pathlen):
- """Creates a path of 'pathlen' hops, and returns it as a list of
- Router instances"""
- self.entry_gen.rewind()
- self.mid_gen.rewind()
- self.exit_gen.rewind()
- entry = self.entry_gen.generate()
- mid = self.mid_gen.generate()
- ext = self.exit_gen.generate()
-
- while True:
- path = []
- try:
- if pathlen == 1:
- path = [ext.next()]
- else:
- path.append(entry.next())
- for i in xrange(1, pathlen-1):
- path.append(mid.next())
- path.append(ext.next())
- if self.path_restrict.path_is_ok(path):
- self.entry_gen.mark_chosen(path[0])
- for i in xrange(1, pathlen-1):
- self.mid_gen.mark_chosen(path[i])
- self.exit_gen.mark_chosen(path[pathlen-1])
- break
- except StopIteration:
- plog("NOTICE", "Ran out of routers during buildpath..");
- self.entry_gen.rewind()
- self.mid_gen.rewind()
- self.exit_gen.rewind()
- entry = self.entry_gen.generate()
- mid = self.entry_gen.generate()
- ext = self.entry_gen.generate()
- return path
-
-class SelectionManager:
- """Helper class to handle configuration updates
-
- The methods are NOT threadsafe. They may ONLY be called from
- EventHandler's thread. This means that to update the selection
- manager, you must schedule a config update job using
- PathBuilder.schedule_selmgr() with a worker function to modify
- this object.
- """
- def __init__(self, pathlen, order_exits,
- percent_fast, percent_skip, min_bw, use_all_exits,
- uniform, use_exit, use_guards,geoip_config=None,restrict_guards=False):
- self.__ordered_exit_gen = None
- self.pathlen = pathlen
- self.order_exits = order_exits
- self.percent_fast = percent_fast
- self.percent_skip = percent_skip
- self.min_bw = min_bw
- self.use_all_exits = use_all_exits
- self.uniform = uniform
- self.exit_name = use_exit
- self.use_guards = use_guards
- self.geoip_config = geoip_config
-
- self.restrict_guards_only = restrict_guards
-
- def reconfigure(self, sorted_r):
- """This function is called after a configuration change,
- to rebuild the RestrictionLists."""
- if self.use_all_exits:
- self.path_rstr = PathRestrictionList([UniqueRestriction()])
- else:
- self.path_rstr = PathRestrictionList(
- [Subnet16Restriction(), UniqueRestriction()])
-
- if self.use_guards: entry_flags = ["Guard", "Valid", "Running"]
- else: entry_flags = ["Valid", "Running"]
-
-
- if self.restrict_guards_only:
- entry_flags = ["Guard","Valid","Running"]
- nonentry_skip = 0
- nonentry_fast = 100
- else:
- entry_flags = ["Valid","Running"]
- nonentry_skip = self.percent_skip
- nonentry_fast = self.percent_fast
-
- entry_rstr = NodeRestrictionList(
- [PercentileRestriction(self.percent_skip, self.percent_fast, sorted_r),
- ConserveExitsRestriction(),
- FlagsRestriction(entry_flags, [])]
- )
- mid_rstr = NodeRestrictionList(
- [PercentileRestriction(nonentry_skip, nonentry_fast, sorted_r),
- ConserveExitsRestriction(),
- FlagsRestriction(["Running","Fast"], [])]
-
- )
- if self.use_all_exits:
- self.exit_rstr = NodeRestrictionList(
- [FlagsRestriction(["Valid", "Running","Fast"], ["BadExit"])])
- else:
- self.exit_rstr = NodeRestrictionList(
- [PercentileRestriction(nonentry_skip, nonentry_fast, sorted_r),
- FlagsRestriction(["Valid", "Running","Fast"], ["BadExit"])])
-
- if self.exit_name:
- self.exit_rstr.del_restriction(IdHexRestriction)
- self.exit_rstr.del_restriction(NickRestriction)
- if self.exit_name[0] == '$':
- self.exit_rstr.add_restriction(IdHexRestriction(self.exit_name))
- else:
- self.exit_rstr.add_restriction(NickRestriction(self.exit_name))
-
- # GeoIP configuration
- if self.geoip_config:
- # Every node needs country_code
- entry_rstr.add_restriction(CountryCodeRestriction())
- mid_rstr.add_restriction(CountryCodeRestriction())
- self.exit_rstr.add_restriction(CountryCodeRestriction())
-
- # Specified countries for different positions
- if self.geoip_config.entry_country:
- entry_rstr.add_restriction(CountryRestriction(self.geoip_config.entry_country))
- if self.geoip_config.middle_country:
- mid_rstr.add_restriction(CountryRestriction(self.geoip_config.middle_country))
- if self.geoip_config.exit_country:
- self.exit_rstr.add_restriction(CountryRestriction(self.geoip_config.exit_country))
-
- # Excluded countries
- if self.geoip_config.excludes:
- plog("INFO", "Excluded countries: " + str(self.geoip_config.excludes))
- if len(self.geoip_config.excludes) > 0:
- entry_rstr.add_restriction(ExcludeCountriesRestriction(self.geoip_config.excludes))
- mid_rstr.add_restriction(ExcludeCountriesRestriction(self.geoip_config.excludes))
- self.exit_rstr.add_restriction(ExcludeCountriesRestriction(self.geoip_config.excludes))
-
- # Unique countries set? None --> pass
- if self.geoip_config.unique_countries != None:
- if self.geoip_config.unique_countries:
- # If True: unique countries
- self.path_rstr.add_restriction(UniqueCountryRestriction())
- else:
- # False: use the same country for all nodes in a path
- self.path_rstr.add_restriction(SingleCountryRestriction())
-
- # Specify max number of continent crossings, None means UniqueContinents
- if self.geoip_config.continent_crossings == None:
- self.path_rstr.add_restriction(UniqueContinentRestriction())
- else: self.path_rstr.add_restriction(ContinentRestriction(self.geoip_config.continent_crossings))
- # Should even work in combination with continent crossings
- if self.geoip_config.ocean_crossings != None:
- self.path_rstr.add_restriction(OceanPhobicRestriction(self.geoip_config.ocean_crossings))
-
- # This is kind of hokey..
- if self.order_exits:
- if self.__ordered_exit_gen:
- exitgen = self.__ordered_exit_gen
- exitgen.reset_restriction(self.exit_rstr)
- else:
- exitgen = self.__ordered_exit_gen = \
- OrderedExitGenerator(80, sorted_r, self.exit_rstr)
- elif self.uniform:
- # 'real' exits should also be chosen when not using 'order_exits'
- self.exit_rstr.add_restriction(ExitPolicyRestriction("255.255.255.255", 80))
- exitgen = UniformGenerator(sorted_r, self.exit_rstr)
- else:
- self.exit_rstr.add_restriction(ExitPolicyRestriction("255.255.255.255", 80))
- exitgen = BwWeightedGenerator(sorted_r, self.exit_rstr, self.pathlen, exit=True)
-
- if self.uniform:
- self.path_selector = PathSelector(
- UniformGenerator(sorted_r, entry_rstr),
- UniformGenerator(sorted_r, mid_rstr),
- exitgen, self.path_rstr)
- else:
- # Remove ConserveExitsRestrictions for entry and middle positions
- entry_rstr.del_restriction(ConserveExitsRestriction)
- mid_rstr.del_restriction(ConserveExitsRestriction)
- self.path_selector = PathSelector(
- BwWeightedGenerator(sorted_r, entry_rstr, self.pathlen,
- guard=self.use_guards),
- BwWeightedGenerator(sorted_r, mid_rstr, self.pathlen),
- exitgen, self.path_rstr)
- return
-
- def set_target(self, ip, port):
- "Called to update the ExitPolicyRestrictions with a new ip and port"
- self.exit_rstr.del_restriction(ExitPolicyRestriction)
- self.exit_rstr.add_restriction(ExitPolicyRestriction(ip, port))
- if self.__ordered_exit_gen: self.__ordered_exit_gen.set_port(port)
- # Try to choose an exit node in the destination country
- # needs an IP != 255.255.255.255
- if self.geoip_config and self.geoip_config.echelon:
- import GeoIPSupport
- c = GeoIPSupport.get_country(ip)
- if c:
- plog("INFO", "[Echelon] IP "+ip+" is in ["+c+"]")
- self.exit_rstr.del_restriction(CountryRestriction)
- self.exit_rstr.add_restriction(CountryRestriction(c))
- else:
- plog("INFO", "[Echelon] Could not determine destination country of IP "+ip)
- # Try to use a backup country
- if self.geoip_config.exit_country:
- self.exit_rstr.del_restriction(CountryRestriction)
- self.exit_rstr.add_restriction(CountryRestriction(self.geoip_config.exit_country))
-
-class Circuit:
- "Class to describe a circuit"
- def __init__(self):
- self.circ_id = 0
- self.path = [] # routers
- self.exit = None
- self.built = False
- self.failed = False
- self.dirty = False
- self.closed = False
- self.detached_cnt = 0
- self.last_extended_at = time.time()
- self.extend_times = [] # List of all extend-durations
- self.setup_duration = None # Sum of extend-times
- self.pending_streams = [] # Which stream IDs are pending us
-
- def id_path(self):
- "Returns a list of idhex keys for the path of Routers"
- return map(lambda r: r.idhex, self.path)
-
-class Stream:
- "Class to describe a stream"
- def __init__(self, sid, host, port, kind):
- self.strm_id = sid
- self.detached_from = [] # circ id #'s
- self.pending_circ = None
- self.circ = None
- self.host = host
- self.port = port
- self.kind = kind
- self.attached_at = 0
- self.bytes_read = 0
- self.bytes_written = 0
- self.failed = False
- self.ignored = False # Set if PURPOSE=DIR_*
- self.failed_reason = None # Cheating a little.. Only used by StatsHandler
-
- def lifespan(self, now):
- "Returns the age of the stream"
- return now-self.attached_at
-
-# TODO: Make passive "PathWatcher" so people can get aggregate
-# node reliability stats for normal usage without us attaching streams
-
-class PathBuilder(TorCtl.EventHandler):
- """
- PathBuilder implementation. Handles circuit construction, subject
- to the constraints of the SelectionManager selmgr.
-
- Do not access this object from other threads. Instead, use the
- schedule_* functions to schedule work to be done in the thread
- of the EventHandler.
- """
- def __init__(self, c, selmgr, RouterClass):
- """Constructor. 'c' is a Connection, 'selmgr' is a SelectionManager,
- and 'RouterClass' is a class that inherits from Router and is used
- to create annotated Routers."""
- TorCtl.EventHandler.__init__(self)
- self.c = c
- nslist = c.get_network_status()
- self.last_exit = None
- self.new_nym = False
- self.resolve_port = 0
- self.num_circuits = 1
- self.RouterClass = RouterClass
- self.sorted_r = []
- self.name_to_key = {}
- self.routers = {}
- self.circuits = {}
- self.streams = {}
- self.read_routers(nslist)
- self.selmgr = selmgr
- self.selmgr.reconfigure(self.sorted_r)
- self.imm_jobs = Queue.Queue()
- self.low_prio_jobs = Queue.Queue()
- self.run_all_jobs = False
- self.do_reconfigure = False
- plog("INFO", "Read "+str(len(self.sorted_r))+"/"+str(len(nslist))+" routers")
-
- def schedule_immediate(self, job):
- """
- Schedules an immediate job to be run before the next event is
- processed.
- """
- self.imm_jobs.put(job)
-
- def schedule_low_prio(self, job):
- """
- Schedules a job to be run when a non-time critical event arrives.
- """
- self.low_prio_jobs.put(job)
-
- def schedule_selmgr(self, job):
- """
- Schedules an immediate job to be run before the next event is
- processed. Also notifies the selection manager that it needs
- to update itself.
- """
- def notlambda(this):
- job(this.selmgr)
- this.do_reconfigure = True
- self.schedule_immediate(notlambda)
-
-
- def heartbeat_event(self, event):
- """This function handles dispatching scheduled jobs. If you
- extend PathBuilder and want to implement this function for
- some reason, be sure to call the parent class"""
- while not self.imm_jobs.empty():
- imm_job = self.imm_jobs.get_nowait()
- imm_job(self)
-
- if self.do_reconfigure:
- self.selmgr.reconfigure(self.sorted_r)
- self.do_reconfigure = False
-
- if self.run_all_jobs:
- self.run_all_jobs = False
- while not self.low_prio_jobs.empty():
- imm_job = self.low_prio_jobs.get_nowait()
- imm_job(self)
- return
-
- # If event is stream:NEW*/DETACHED or circ BUILT/FAILED,
- # don't run low prio jobs.. No need to delay streams for them.
- if isinstance(event, TorCtl.CircuitEvent):
- if event.status in ("BUILT", "FAILED"):
- return
- elif isinstance(event, TorCtl.StreamEvent):
- if event.status in ("NEW", "NEWRESOLVE", "DETACHED"):
- return
-
- # Do the low prio jobs one at a time in case a
- # higher priority event is queued
- if not self.low_prio_jobs.empty():
- delay_job = self.low_prio_jobs.get_nowait()
- delay_job(self)
-
- def read_routers(self, nslist):
- routers = self.c.read_routers(nslist)
- new_routers = []
- for r in routers:
- self.name_to_key[r.nickname] = "$"+r.idhex
- if r.idhex in self.routers:
- if self.routers[r.idhex].nickname != r.nickname:
- plog("NOTICE", "Router "+r.idhex+" changed names from "
- +self.routers[r.idhex].nickname+" to "+r.nickname)
- # Must do IN-PLACE update to keep all the refs to this router
- # valid and current (especially for stats)
- self.routers[r.idhex].update_to(r)
- else:
- rc = self.RouterClass(r)
- self.routers[rc.idhex] = rc
- new_routers.append(rc)
- self.sorted_r.extend(new_routers)
- self.sorted_r.sort(lambda x, y: cmp(y.bw, x.bw))
- for i in xrange(len(self.sorted_r)): self.sorted_r[i].list_rank = i
-
- def build_path(self):
- """ Get a path from the SelectionManager's PathSelector, can be used
- e.g. for generating paths without actually creating any circuits """
- return self.selmgr.path_selector.build_path(self.selmgr.pathlen)
-
- def attach_stream_any(self, stream, badcircs):
- "Attach a stream to a valid circuit, avoiding any in 'badcircs'"
- # Newnym, and warn if not built plus pending
- unattached_streams = [stream]
- if self.new_nym:
- self.new_nym = False
- plog("DEBUG", "Obeying new nym")
- for key in self.circuits.keys():
- if (not self.circuits[key].dirty
- and len(self.circuits[key].pending_streams)):
- plog("WARN", "New nym called, destroying circuit "+str(key)
- +" with "+str(len(self.circuits[key].pending_streams))
- +" pending streams")
- unattached_streams.extend(self.circuits[key].pending_streams)
- self.circuits[key].pending_streams.clear()
- # FIXME: Consider actually closing circ if no streams.
- self.circuits[key].dirty = True
-
- for circ in self.circuits.itervalues():
- if circ.built and not circ.dirty and circ.circ_id not in badcircs:
- if circ.exit.will_exit_to(stream.host, stream.port):
- try:
- self.c.attach_stream(stream.strm_id, circ.circ_id)
- stream.pending_circ = circ # Only one possible here
- circ.pending_streams.append(stream)
- except TorCtl.ErrorReply, e:
- # No need to retry here. We should get the failed
- # event for either the circ or stream next
- plog("WARN", "Error attaching stream: "+str(e.args))
- return
- break
- else:
- circ = None
- self.selmgr.set_target(stream.host, stream.port)
- while circ == None:
- try:
- circ = self.c.build_circuit(
- self.selmgr.pathlen,
- self.selmgr.path_selector)
- except TorCtl.ErrorReply, e:
- # FIXME: How come some routers are non-existant? Shouldn't
- # we have gotten an NS event to notify us they
- # disappeared?
- plog("NOTICE", "Error building circ: "+str(e.args))
- for u in unattached_streams:
- plog("DEBUG",
- "Attaching "+str(u.strm_id)+" pending build of "+str(circ.circ_id))
- u.pending_circ = circ
- circ.pending_streams.extend(unattached_streams)
- self.circuits[circ.circ_id] = circ
- self.last_exit = circ.exit
-
- def circ_status_event(self, c):
- output = [c.event_name, str(c.circ_id), c.status]
- if c.path: output.append(",".join(c.path))
- if c.reason: output.append("REASON=" + c.reason)
- if c.remote_reason: output.append("REMOTE_REASON=" + c.remote_reason)
- plog("DEBUG", " ".join(output))
- # Circuits we don't control get built by Tor
- if c.circ_id not in self.circuits:
- plog("DEBUG", "Ignoring circ " + str(c.circ_id))
- return
- if c.status == "EXTENDED":
- self.circuits[c.circ_id].last_extended_at = c.arrived_at
- elif c.status == "FAILED" or c.status == "CLOSED":
- # XXX: Can still get a STREAM FAILED for this circ after this
- circ = self.circuits[c.circ_id]
- del self.circuits[c.circ_id]
- for stream in circ.pending_streams:
- plog("DEBUG", "Finding new circ for " + str(stream.strm_id))
- self.attach_stream_any(stream, stream.detached_from)
- elif c.status == "BUILT":
- self.circuits[c.circ_id].built = True
- try:
- for stream in self.circuits[c.circ_id].pending_streams:
- self.c.attach_stream(stream.strm_id, c.circ_id)
- except TorCtl.ErrorReply, e:
- # No need to retry here. We should get the failed
- # event for either the circ or stream next
- plog("WARN", "Error attaching stream: "+str(e.args))
- return
-
- def stream_status_event(self, s):
- output = [s.event_name, str(s.strm_id), s.status, str(s.circ_id),
- s.target_host, str(s.target_port)]
- if s.reason: output.append("REASON=" + s.reason)
- if s.remote_reason: output.append("REMOTE_REASON=" + s.remote_reason)
- if s.purpose: output.append("PURPOSE=" + s.purpose)
- plog("DEBUG", " ".join(output))
- if not re.match(r"\d+.\d+.\d+.\d+", s.target_host):
- s.target_host = "255.255.255.255" # ignore DNS for exit policy check
-
- # Hack to ignore Tor-handled streams (Currently only directory streams)
- if s.strm_id in self.streams and self.streams[s.strm_id].ignored:
- plog("DEBUG", "Ignoring stream: " + str(s.strm_id))
- return
-
- if s.status == "NEW" or s.status == "NEWRESOLVE":
- if s.status == "NEWRESOLVE" and not s.target_port:
- s.target_port = self.resolve_port
- self.streams[s.strm_id] = Stream(s.strm_id, s.target_host, s.target_port, s.status)
- # Remember Tor-handled streams (Currently only directory streams)
- if s.purpose and s.purpose.find("DIR_") == 0:
- self.streams[s.strm_id].ignored = True
- plog("DEBUG", "Ignoring stream: " + str(s.strm_id))
- return
- else:
- self.attach_stream_any(self.streams[s.strm_id],
- self.streams[s.strm_id].detached_from)
- elif s.status == "DETACHED":
- if s.strm_id not in self.streams:
- plog("WARN", "Detached stream "+str(s.strm_id)+" not found")
- self.streams[s.strm_id] = Stream(s.strm_id, s.target_host,
- s.target_port, "NEW")
- # FIXME Stats (differentiate Resolved streams also..)
- if not s.circ_id:
- plog("WARN", "Stream "+str(s.strm_id)+" detached from no circuit!")
- else:
- self.streams[s.strm_id].detached_from.append(s.circ_id)
-
- if self.streams[s.strm_id] in self.streams[s.strm_id].pending_circ.pending_streams:
- self.streams[s.strm_id].pending_circ.pending_streams.remove(self.streams[s.strm_id])
- self.streams[s.strm_id].pending_circ = None
- self.attach_stream_any(self.streams[s.strm_id],
- self.streams[s.strm_id].detached_from)
- elif s.status == "SUCCEEDED":
- if s.strm_id not in self.streams:
- plog("NOTICE", "Succeeded stream "+str(s.strm_id)+" not found")
- return
- if s.circ_id and self.streams[s.strm_id].pending_circ.circ_id != s.circ_id:
- # Hrmm.. this can happen on a new-nym.. Very rare, putting warn
- # in because I'm still not sure this is correct
- plog("WARN", "Mismatch of pending: "
- +str(self.streams[s.strm_id].pending_circ.circ_id)+" vs "
- +str(s.circ_id))
- # This can happen if the circuit existed before we started up
- if s.circ_id in self.circuits:
- self.streams[s.strm_id].circ = self.circuits[s.circ_id]
- else:
- plog("NOTICE", "Stream "+str(s.strm_id)+" has unknown circuit: "+str(s.circ_id))
- else:
- self.streams[s.strm_id].circ = self.streams[s.strm_id].pending_circ
- self.streams[s.strm_id].pending_circ.pending_streams.remove(self.streams[s.strm_id])
- self.streams[s.strm_id].pending_circ = None
- self.streams[s.strm_id].attached_at = s.arrived_at
- elif s.status == "FAILED" or s.status == "CLOSED":
- # FIXME stats
- if s.strm_id not in self.streams:
- plog("NOTICE", "Failed stream "+str(s.strm_id)+" not found")
- return
-
- if not s.circ_id:
- plog("WARN", "Stream "+str(s.strm_id)+" failed from no circuit!")
-
- # We get failed and closed for each stream. OK to return
- # and let the closed do the cleanup
- if s.status == "FAILED":
- # Avoid busted circuits that will not resolve or carry
- # traffic.
- self.streams[s.strm_id].failed = True
- if s.circ_id in self.circuits: self.circuits[s.circ_id].dirty = True
- else: plog("WARN","Failed stream on unknown circ "+str(s.circ_id))
- return
-
- if self.streams[s.strm_id].pending_circ:
- self.streams[s.strm_id].pending_circ.pending_streams.remove(self.streams[s.strm_id])
- del self.streams[s.strm_id]
- elif s.status == "REMAP":
- if s.strm_id not in self.streams:
- plog("WARN", "Remap id "+str(s.strm_id)+" not found")
- else:
- if not re.match(r"\d+.\d+.\d+.\d+", s.target_host):
- s.target_host = "255.255.255.255"
- plog("NOTICE", "Non-IP remap for "+str(s.strm_id)+" to "
- + s.target_host)
- self.streams[s.strm_id].host = s.target_host
- self.streams[s.strm_id].port = s.target_port
-
- def stream_bw_event(self, s):
- output = [s.event_name, str(s.strm_id), str(s.bytes_read),
- str(s.bytes_written)]
- plog("DEBUG", " ".join(output))
- if not s.strm_id in self.streams:
- plog("WARN", "BW event for unknown stream id: "+str(s.strm_id))
- else:
- self.streams[s.strm_id].bytes_read += s.bytes_read
- self.streams[s.strm_id].bytes_written += s.bytes_written
-
- def ns_event(self, n):
- self.read_routers(n.nslist)
- plog("DEBUG", "Read " + str(len(n.nslist))+" NS => "
- + str(len(self.sorted_r)) + " routers")
-
- def new_desc_event(self, d):
- for i in d.idlist: # Is this too slow?
- self.read_routers(self.c.get_network_status("id/"+i))
- plog("DEBUG", "Read " + str(len(d.idlist))+" Desc => "
- + str(len(self.sorted_r)) + " routers")
-
- def bandwidth_event(self, b): pass # For heartbeat only..
-
-################### CircuitHandler #############################
-
-class CircuitHandler(PathBuilder):
- """ CircuitHandler that extends from PathBuilder to handle multiple
- circuits as opposed to just one. """
- def __init__(self, c, selmgr, num_circuits, RouterClass):
- """Constructor. 'c' is a Connection, 'selmgr' is a SelectionManager,
- 'num_circuits' is the number of circuits to keep in the pool,
- and 'RouterClass' is a class that inherits from Router and is used
- to create annotated Routers."""
- PathBuilder.__init__(self, c, selmgr, RouterClass)
- # Set handler to the connection here to
- # not miss any circuit events on startup
- c.set_event_handler(self)
- self.num_circuits = num_circuits # Size of the circuit pool
- self.check_circuit_pool() # Bring up the pool of circs
-
- def check_circuit_pool(self):
- """ Init or check the status of the circuit-pool """
- # Get current number of circuits
- n = len(self.circuits.values())
- i = self.num_circuits-n
- if i > 0:
- plog("INFO", "Checked pool of circuits: we need to build " +
- str(i) + " circuits")
- # Schedule (num_circs-n) circuit-buildups
- while (n < self.num_circuits):
- self.build_circuit("255.255.255.255", 80)
- plog("DEBUG", "Scheduled circuit No. " + str(n+1))
- n += 1
-
- def build_circuit(self, host, port):
- """ Build a circuit """
- circ = None
- while circ == None:
- try:
- self.selmgr.set_target(host, port)
- circ = self.c.build_circuit(self.selmgr.pathlen,
- self.selmgr.path_selector)
- self.circuits[circ.circ_id] = circ
- return circ
- except TorCtl.ErrorReply, e:
- # FIXME: How come some routers are non-existant? Shouldn't
- # we have gotten an NS event to notify us they disappeared?
- plog("NOTICE", "Error building circuit: " + str(e.args))
-
- def close_circuit(self, id):
- """ Close a circuit with given id """
- # TODO: Pass streams to another circ before closing?
- self.circuits[id].closed = True
- try: self.c.close_circuit(id)
- except TorCtl.ErrorReply, e:
- plog("ERROR", "Failed closing circuit " + str(id) + ": " + str(e))
-
- def circ_status_event(self, c):
- """ Handle circuit status events """
- output = [c.event_name, str(c.circ_id), c.status]
- if c.path: output.append(",".join(c.path))
- if c.reason: output.append("REASON=" + c.reason)
- if c.remote_reason: output.append("REMOTE_REASON=" + c.remote_reason)
- plog("DEBUG", " ".join(output))
-
- # Circuits we don't control get built by Tor
- if c.circ_id not in self.circuits:
- plog("DEBUG", "Ignoring circuit " + str(c.circ_id) +
- " (controlled by Tor)")
- return
-
- # EXTENDED
- if c.status == "EXTENDED":
- # Compute elapsed time
- extend_time = c.arrived_at-self.circuits[c.circ_id].last_extended_at
- self.circuits[c.circ_id].extend_times.append(extend_time)
- plog("INFO", "Circuit " + str(c.circ_id) + " extended in " +
- str(extend_time) + " sec")
- self.circuits[c.circ_id].last_extended_at = c.arrived_at
-
- # FAILED & CLOSED
- elif c.status == "FAILED" or c.status == "CLOSED":
- # XXX: Can still get a STREAM FAILED for this circ after this
- circ = self.circuits[c.circ_id]
- # Actual removal of the circ
- del self.circuits[c.circ_id]
- # Give away pending streams
- for stream in circ.pending_streams:
- plog("DEBUG", "Finding new circ for " + str(stream.strm_id))
- self.attach_stream_any(stream, stream.detached_from)
- # Check if there are enough circs
- self.check_circuit_pool()
- return
-
- # BUILT
- elif c.status == "BUILT":
- circ = self.circuits[c.circ_id]
- circ.built = True
- for stream in circ.pending_streams:
- try:
- self.c.attach_stream(stream.strm_id, c.circ_id)
- except TorCtl.ErrorReply, e:
- # No need to retry here. We should get the failed
- # event for either the circ or stream next
- plog("WARN", "Error attaching stream: " + str(e.args))
- # Compute duration by summing up extend_times
- duration = reduce(lambda x, y: x+y, circ.extend_times, 0.0)
- plog("INFO", "Circuit " + str(c.circ_id) + " needed " +
- str(duration) + " seconds to be built")
- # Save the duration to the circuit for later use
- circ.setup_duration = duration
-
- # OTHER?
- else:
- # If this was e.g. a LAUNCHED
- pass
-
-################### StreamHandler ##############################
-
-class StreamHandler(CircuitHandler):
- """ StreamHandler that extends from the CircuitHandler
- to handle attaching streams to an appropriate circuit
- in the pool. """
- def __init__(self, c, selmgr, num_circs, RouterClass):
- CircuitHandler.__init__(self, c, selmgr, num_circs, RouterClass)
- self.sorted_circs = None # optional sorted list
-
- def clear_dns_cache(self):
- """ Send signal CLEARDNSCACHE """
- lines = self.c.sendAndRecv("SIGNAL CLEARDNSCACHE\r\n")
- for _, msg, more in lines:
- plog("DEBUG", "CLEARDNSCACHE: " + msg)
-
- def close_stream(self, id, reason):
- """ Close a stream with given id and reason """
- self.c.close_stream(id, reason)
-
- def create_and_attach(self, stream, unattached_streams):
- """ Create a new circuit and attach (stream + unattached_streams) """
- circ = self.build_circuit(stream.host, stream.port)
- if circ:
- for u in unattached_streams:
- plog("DEBUG", "Attaching " + str(u.strm_id) +
- " pending build of circuit " + str(circ.circ_id))
- u.pending_circ = circ
- circ.pending_streams.extend(unattached_streams)
- self.circuits[circ.circ_id] = circ
- self.last_exit = circ.exit
-
- def attach_stream_any(self, stream, badcircs):
- """ Attach a regular user stream """
- unattached_streams = [stream]
- if self.new_nym:
- self.new_nym = False
- plog("DEBUG", "Obeying new nym")
- for key in self.circuits.keys():
- if (not self.circuits[key].dirty
- and len(self.circuits[key].pending_streams)):
- plog("WARN", "New nym called, destroying circuit "+str(key)
- +" with "+str(len(self.circuits[key].pending_streams))
- +" pending streams")
- unattached_streams.extend(self.circuits[key].pending_streams)
- del self.circuits[key].pending_streams[:]
- # FIXME: Consider actually closing circs if no streams
- self.circuits[key].dirty = True
-
- # Check if there is a sorted list of circs
- if self.sorted_circs: list = self.sorted_circs
- else: list = self.circuits.values()
- for circ in list:
- # Check each circuit
- if circ.built and not circ.closed and circ.circ_id not in badcircs and not circ.dirty:
- if circ.exit.will_exit_to(stream.host, stream.port):
- try:
- self.c.attach_stream(stream.strm_id, circ.circ_id)
- stream.pending_circ = circ # Only one possible here
- circ.pending_streams.append(stream)
- self.last_exit = circ.exit
- except TorCtl.ErrorReply, e:
- # No need to retry here. We should get the failed
- # event for either the circ or stream next
- plog("WARN", "Error attaching stream: " + str(e.args))
- return
- break
- else:
- plog("DEBUG", "Circuit " + str(circ.circ_id) + " won't exit")
- else:
- self.create_and_attach(stream, unattached_streams)
-
- def stream_status_event(self, s):
- """ Catch user stream events """
- # Construct debugging output
- output = [s.event_name, str(s.strm_id), s.status, str(s.circ_id), s.target_host+':'+str(s.target_port)]
- if s.reason: output.append("REASON=" + s.reason)
- if s.remote_reason: output.append("REMOTE_REASON=" + s.remote_reason)
- if s.purpose: output.append("PURPOSE=" + s.purpose)
- plog("DEBUG", " ".join(output))
-
- # If target_host is not an IP-address
- if not re.match(r"\d+.\d+.\d+.\d+", s.target_host):
- s.target_host = "255.255.255.255" # ignore DNS for exit policy check
-
- # Hack to ignore Tor-handled streams (Currently only directory streams)
- if s.strm_id in self.streams and self.streams[s.strm_id].ignored:
- plog("DEBUG", "Ignoring stream: " + str(s.strm_id))
- return
-
- # NEW or NEWRESOLVE
- if s.status == "NEW" or s.status == "NEWRESOLVE":
- if s.status == "NEWRESOLVE" and not s.target_port:
- s.target_port = self.resolve_port
- # Set up the new stream
- stream = Stream(s.strm_id, s.target_host, s.target_port, s.status)
-
- self.streams[s.strm_id] = stream
- if s.purpose and s.purpose.find("DIR_") == 0:
- stream.ignored = True
- plog("DEBUG", "Ignoring stream: " + str(s.strm_id))
- return
- else:
- self.attach_stream_any(self.streams[s.strm_id], self.streams[s.strm_id].detached_from)
-
- # DETACHED
- elif s.status == "DETACHED":
- # Stream not found
- if s.strm_id not in self.streams:
- plog("WARN", "Detached stream " + str(s.strm_id) + " not found")
- self.streams[s.strm_id] = Stream(s.strm_id, s.target_host, s.target_port, "NEW")
- # Circuit not found
- if not s.circ_id:
- plog("WARN", "Stream " + str(s.strm_id) + " detached from no circuit!")
- else:
- self.streams[s.strm_id].detached_from.append(s.circ_id)
- # Detect timeouts on user streams
- if s.reason == "TIMEOUT":
- # TODO: Count timeouts on streams?
- #self.streams[s.strm_id].timeout_counter += 1
- plog("DEBUG", "User stream timed out on circuit " + str(s.circ_id))
- # Stream was pending
- if self.streams[s.strm_id] in self.streams[s.strm_id].pending_circ.pending_streams:
- self.streams[s.strm_id].pending_circ.pending_streams.remove(self.streams[s.strm_id])
- # Attach to another circ
- self.streams[s.strm_id].pending_circ = None
- self.attach_stream_any(self.streams[s.strm_id], self.streams[s.strm_id].detached_from)
-
- # SUCCEEDED
- if s.status == "SUCCEEDED":
- if s.strm_id not in self.streams:
- plog("NOTICE", "Succeeded stream " + str(s.strm_id) + " not found")
- return
- if s.circ_id and self.streams[s.strm_id].pending_circ.circ_id != s.circ_id:
- # Hrmm.. this can happen on a new-nym.. Very rare, putting warn
- # in because I'm still not sure this is correct
- plog("WARN", "Mismatch of pending: "
- + str(self.streams[s.strm_id].pending_circ.circ_id) + " vs "
- + str(s.circ_id))
- self.streams[s.strm_id].circ = self.circuits[s.circ_id]
- else:
- self.streams[s.strm_id].circ = self.streams[s.strm_id].pending_circ
- self.streams[s.strm_id].pending_circ.pending_streams.remove(self.streams[s.strm_id])
- self.streams[s.strm_id].pending_circ = None
- self.streams[s.strm_id].attached_at = s.arrived_at
-
- # FAILED or CLOSED
- elif s.status == "FAILED" or s.status == "CLOSED":
- if s.strm_id not in self.streams:
- plog("NOTICE", "Failed stream " + str(s.strm_id) + " not found")
- return
- # if not s.circ_id:
- # plog("WARN", "Stream " + str(s.strm_id) + " closed/failed from no circuit")
- # We get failed and closed for each stream, let CLOSED do the cleanup
- if s.status == "FAILED":
- # Avoid busted circuits that will not resolve or carry traffic
- self.streams[s.strm_id].failed = True
- if s.circ_id in self.circuits: self.circuits[s.circ_id].dirty = True
- elif self.streams[s.strm_id].attached_at != 0:
- plog("WARN", "Failed stream on unknown circuit " + str(s.circ_id))
- return
- # CLOSED
- if self.streams[s.strm_id].pending_circ:
- self.streams[s.strm_id].pending_circ.pending_streams.remove(self.streams[s.strm_id])
- # Actual removal of the stream
- del self.streams[s.strm_id]
-
- # REMAP
- elif s.status == "REMAP":
- if s.strm_id not in self.streams:
- plog("WARN", "Remap id "+str(s.strm_id)+" not found")
- else:
- if not re.match(r"\d+.\d+.\d+.\d+", s.target_host):
- s.target_host = "255.255.255.255"
- plog("NOTICE", "Non-IP remap for "+str(s.strm_id) +
- " to " + s.target_host)
- self.streams[s.strm_id].host = s.target_host
- self.streams[s.strm_id].port = s.target_port
-
- def address_mapped_event(self, event):
- """ It is necessary to listen to ADDRMAP events to be able to
- perform DNS lookups using Tor """
- output = [event.event_name, event.from_addr, event.to_addr,
- time.asctime(event.when)]
- plog("DEBUG", " ".join(output))
-
- def unknown_event(self, event):
- plog("DEBUG", "UNKNOWN EVENT '" + event.event_name + "':" +
- event.event_string)
-
-########################## Unit tests ##########################
-
-def do_gen_unit(gen, r_list, weight_bw, num_print):
- trials = 0
- for r in r_list:
- if gen.rstr_list.r_is_ok(r):
- trials += weight_bw(gen, r)
- trials = int(trials/1024)
-
- print "Running "+str(trials)+" trials"
-
- # 0. Reset r.chosen = 0 for all routers
- for r in r_list:
- r.chosen = 0
-
- # 1. Generate 'trials' choices:
- # 1a. r.chosen++
-
- loglevel = TorUtil.loglevel
- TorUtil.loglevel = "INFO"
-
- #gen.rewind() - Just overhead if we create a fresh generator each time
- rtrs = gen.generate()
- for i in xrange(1, trials):
- r = rtrs.next()
- r.chosen += 1
-
- TorUtil.loglevel = loglevel
-
- # 2. Print top num_print routers choices+bandwidth stats+flags
- i = 0
- copy_rlist = copy.copy(r_list)
- copy_rlist.sort(lambda x, y: cmp(y.chosen, x.chosen))
- for r in copy_rlist:
- if not gen.rstr_list.r_is_ok(r): continue
- flag = ""
- bw = int(weight_bw(gen, r))
- if "Exit" in r.flags:
- flag += "E"
- if "Guard" in r.flags:
- flag += "G"
- print str(r.list_rank)+". "+r.nickname+" "+str(r.bw/1024)+"/"+str(bw/1024)+": "+str(r.chosen)+", "+flag
- i += 1
- if i > num_print: break
-
-def do_unit(rst, r_list, plamb):
- print "\n"
- print "-----------------------------------"
- print rst.r_is_ok.im_class
- above_i = 0
- above_bw = 0
- below_i = 0
- below_bw = 0
- for r in r_list:
- if rst.r_is_ok(r):
- print r.nickname+" "+plamb(r)+"="+str(rst.r_is_ok(r))+" "+str(r.bw)
- if r.bw > 400000:
- above_i = above_i + 1
- above_bw += r.bw
- else:
- below_i = below_i + 1
- below_bw += r.bw
-
- print "Routers above: " + str(above_i) + " bw: " + str(above_bw)
- print "Routers below: " + str(below_i) + " bw: " + str(below_bw)
-
-# TODO: Tests:
-# - Test each NodeRestriction and print in/out lines for it
-# - Test NodeGenerator and reapply NodeRestrictions
-# - Same for PathSelector and PathRestrictions
-# - Also Reapply each restriction by hand to path. Verify returns true
-
-if __name__ == '__main__':
- s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- s.connect(("127.0.0.1",9051))
- c = Connection(s)
- c.debug(file("control.log", "w"))
- c.authenticate()
- nslist = c.get_network_status()
- sorted_rlist = c.read_routers(c.get_network_status())
-
- sorted_rlist.sort(lambda x, y: cmp(y.bw, x.bw))
- for i in xrange(len(sorted_rlist)): sorted_rlist[i].list_rank = i
-
- def flag_weighting(bwgen, r):
- bw = r.bw
- if "Exit" in r.flags:
- bw *= bwgen.exit_weight
- if "Guard" in r.flags:
- bw *= bwgen.guard_weight
- return bw
-
- do_gen_unit(BwWeightedGenerator(sorted_rlist, FlagsRestriction(["Exit"]),
- 3, exit=True),
- sorted_rlist, flag_weighting, 500)
-
- do_gen_unit(BwWeightedGenerator(sorted_rlist, FlagsRestriction(["Guard"]),
- 3, guard=True),
- sorted_rlist, flag_weighting, 500)
-
- do_gen_unit(
- BwWeightedGenerator(sorted_rlist, FlagsRestriction(["Valid"]), 3),
- sorted_rlist, flag_weighting, 500)
-
- exit(0)
-
- for r in sorted_rlist:
- if r.will_exit_to("211.11.21.22", 465):
- print r.nickname+" "+str(r.bw)
-
- do_unit(FlagsRestriction(["Guard"], []), sorted_rlist, lambda r: " ".join(r.flags))
- do_unit(FlagsRestriction(["Fast"], []), sorted_rlist, lambda r: " ".join(r.flags))
-
- do_unit(ExitPolicyRestriction("2.11.2.2", 80), sorted_rlist,
- lambda r: "exits to 80")
- do_unit(PercentileRestriction(0, 100, sorted_rlist), sorted_rlist,
- lambda r: "")
- do_unit(PercentileRestriction(10, 20, sorted_rlist), sorted_rlist,
- lambda r: "")
- do_unit(OSRestriction([r"[lL]inux", r"BSD", "Darwin"], []), sorted_rlist,
- lambda r: r.os)
- do_unit(OSRestriction([], ["Windows", "Solaris"]), sorted_rlist,
- lambda r: r.os)
-
- do_unit(VersionRangeRestriction("0.1.2.0"), sorted_rlist,
- lambda r: str(r.version))
- do_unit(VersionRangeRestriction("0.1.2.0", "0.1.2.5"), sorted_rlist,
- lambda r: str(r.version))
- do_unit(VersionIncludeRestriction(["0.1.1.26-alpha", "0.1.2.7-ignored"]),
- sorted_rlist, lambda r: str(r.version))
- do_unit(VersionExcludeRestriction(["0.1.1.26"]), sorted_rlist,
- lambda r: str(r.version))
-
- do_unit(ConserveExitsRestriction(), sorted_rlist, lambda r: " ".join(r.flags))
- do_unit(FlagsRestriction([], ["Valid"]), sorted_rlist, lambda r: " ".join(r.flags))
-
- do_unit(IdHexRestriction("$FFCB46DB1339DA84674C70D7CB586434C4370441"),
- sorted_rlist, lambda r: r.idhex)
-
- rl = [AtLeastNNodeRestriction([ExitPolicyRestriction("255.255.255.255", 80), ExitPolicyRestriction("255.255.255.255", 443), ExitPolicyRestriction("255.255.255.255", 6667)], 2), FlagsRestriction([], ["BadExit"])]
-
- exit_rstr = NodeRestrictionList(rl)
-
- ug = UniformGenerator(sorted_rlist, exit_rstr)
-
- rlist = []
- for r in ug.generate():
- print "Checking: " + r.nickname
- for rs in rl:
- if not rs.r_is_ok(r):
- raise PathError()
- if not "Exit" in r.flags:
- print "No exit in flags of "+r.nickname
- rlist.append(r)
- for r in sorted_rlist:
- if "Exit" in r.flags and not r in rlist:
- print r.nickname+" is an exit not in rl!"
-
Copied: torflow/branches/gsoc2008/TorCtl.local/PathSupport.py (from rev 15741, torflow/branches/gsoc2008/TorCtl/PathSupport.py)
===================================================================
--- torflow/branches/gsoc2008/TorCtl.local/PathSupport.py (rev 0)
+++ torflow/branches/gsoc2008/TorCtl.local/PathSupport.py 2008-07-07 19:37:47 UTC (rev 15742)
@@ -0,0 +1,1712 @@
+#!/usr/bin/python
+"""
+
+Support classes for path construction
+
+The PathSupport package builds on top of TorCtl.TorCtl. It provides a
+number of interfaces that make path construction easier.
+
+The inheritance diagram for event handling is as follows:
+TorCtl.EventHandler <- PathBuilder <- CircuitHandler <- StreamHandler.
+
+Basically, EventHandler is what gets all the control port events
+packaged in nice clean classes (see help(TorCtl) for information on
+those).
+
+PathBuilder inherits from EventHandler and is what builds all circuits
+based on the requirements specified in the SelectionManager instance
+passed to its constructor. It also handles attaching streams to
+circuits. It only handles one building one circuit at a time.
+
+CircuitHandler optionally inherits from PathBuilder, and overrides its
+circuit event handling to manage building a pool of circuits as opposed
+to just one. It still uses the SelectionManager for path selection.
+
+StreamHandler inherits from CircuitHandler, and is what governs the
+attachment of an incoming stream on to one of the multiple circuits of
+the circuit handler.
+
+The SelectionManager is essentially a configuration wrapper around the
+most elegant portions of TorFlow: NodeGenerators, NodeRestrictions, and
+PathRestrictions. In the SelectionManager, a NodeGenerator is used to
+choose the nodes probabilistically according to some distribution while
+obeying the NodeRestrictions. These generators (one per hop) are handed
+off to the PathSelector, which uses the generators to build a complete
+path that satisfies the PathRestriction requirements.
+
+Have a look at the class hierarchy directly below to get a feel for how
+the restrictions fit together, and what options are available.
+
+"""
+
+import TorCtl
+import re
+import struct
+import random
+import socket
+import copy
+import Queue
+import time
+import TorUtil
+from TorUtil import *
+
+__all__ = ["NodeRestrictionList", "PathRestrictionList",
+"PercentileRestriction", "OSRestriction", "ConserveExitsRestriction",
+"FlagsRestriction", "MinBWRestriction", "VersionIncludeRestriction",
+"VersionExcludeRestriction", "ExitPolicyRestriction", "NodeRestriction",
+"PathRestriction", "OrNodeRestriction", "MetaNodeRestriction",
+"AtLeastNNodeRestriction", "NotNodeRestriction", "Subnet16Restriction",
+"UniqueRestriction", "NodeGenerator", "UniformGenerator",
+"OrderedExitGenerator", "BwWeightedGenerator", "PathSelector",
+"Connection", "NickRestriction", "IdHexRestriction", "PathBuilder",
+"CircuitHandler", "StreamHandler", "SelectionManager",
+"CountryCodeRestriction", "CountryRestriction",
+"UniqueCountryRestriction", "SingleCountryRestriction",
+"ContinentRestriction", "ContinentJumperRestriction",
+"UniqueContinentRestriction"]
+
+#################### Path Support Interfaces #####################
+
+class NodeRestriction:
+ "Interface for node restriction policies"
+ def r_is_ok(self, r):
+ "Returns true if Router 'r' is acceptable for this restriction"
+ return True
+
+class NodeRestrictionList:
+ "Class to manage a list of NodeRestrictions"
+ def __init__(self, restrictions):
+ "Constructor. 'restrictions' is a list of NodeRestriction instances"
+ self.restrictions = restrictions
+
+ def r_is_ok(self, r):
+ "Returns true of Router 'r' passes all of the contained restrictions"
+ for rs in self.restrictions:
+ if not rs.r_is_ok(r): return False
+ return True
+
+ def add_restriction(self, restr):
+ "Add a NodeRestriction 'restr' to the list of restrictions"
+ self.restrictions.append(restr)
+
+ # TODO: This does not collapse meta restrictions..
+ def del_restriction(self, RestrictionClass):
+ """Remove all restrictions of type RestrictionClass from the list.
+ Does NOT inspect or collapse MetaNode Restrictions (though
+ MetaRestrictions can be removed if RestrictionClass is
+ MetaNodeRestriction)"""
+ self.restrictions = filter(
+ lambda r: not isinstance(r, RestrictionClass),
+ self.restrictions)
+
+class PathRestriction:
+ "Interface for path restriction policies"
+ def path_is_ok(self, path):
+ "Return true if the list of Routers in path satisfies this restriction"
+ return True
+
+class PathRestrictionList:
+ """Class to manage a list of PathRestrictions"""
+ def __init__(self, restrictions):
+ "Constructor. 'restrictions' is a list of PathRestriction instances"
+ self.restrictions = restrictions
+
+ def path_is_ok(self, path):
+ "Given list if Routers in 'path', check it against each restriction."
+ for rs in self.restrictions:
+ if not rs.path_is_ok(path):
+ return False
+ return True
+
+ def add_restriction(self, rstr):
+ "Add a PathRestriction 'rstr' to the list"
+ self.restrictions.append(rstr)
+
+ def del_restriction(self, RestrictionClass):
+ "Remove all PathRestrictions of type RestrictionClass from the list."
+ self.restrictions = filter(
+ lambda r: not isinstance(r, RestrictionClass),
+ self.restrictions)
+
+class NodeGenerator:
+ "Interface for node generation"
+ def __init__(self, sorted_r, rstr_list):
+ """Constructor. Takes a bandwidth-sorted list of Routers 'sorted_r'
+ and a NodeRestrictionList 'rstr_list'"""
+ self.rstr_list = rstr_list # Check me before you yield!
+ self.sorted_r = sorted_r
+ self.rewind()
+
+ def reset_restriction(self, rstr_list):
+ "Reset the restriction list to a new list"
+ self.rstr_list = rstr_list
+
+ def rewind(self):
+ "Rewind the generator to the 'beginning'"
+ # FIXME: If we apply the restrictions now, we can save cycles
+ # during selection, and also some memory overhead (at the cost
+ # of a much slower rewind() though..)
+ self.routers = filter(lambda r: self.rstr_list.r_is_ok(r), self.sorted_r)
+ #self.routers = copy.copy(self.sorted_r)
+
+ def mark_chosen(self, r):
+ """Mark a router as chosen: remove it from the list of routers
+ that can be returned in the future"""
+ self.routers.remove(r)
+
+ def all_chosen(self):
+ "Return true if all the routers have been marked as chosen"
+ return not self.routers
+
+ def generate(self):
+ "Return a python generator that yields routers according to the policy"
+ raise NotImplemented()
+
+class Connection(TorCtl.Connection):
+ """Extended Connection class that provides a method for building circuits"""
+ def __init__(self, sock):
+ TorCtl.Connection.__init__(self,sock)
+ def build_circuit(self, pathlen, path_sel):
+ "Tell Tor to build a circuit chosen by the PathSelector 'path_sel'"
+ circ = Circuit()
+ circ.path = path_sel.build_path(pathlen)
+ circ.exit = circ.path[pathlen-1]
+ circ.circ_id = self.extend_circuit(0, circ.id_path())
+ return circ
+
+######################## Node Restrictions ########################
+
+# TODO: We still need more path support implementations
+# - NodeRestrictions:
+# - Uptime/LongLivedPorts (Does/should hibernation count?)
+# - Published/Updated
+# - PathRestrictions:
+# - Family
+# - GeoIP:
+# - OceanPhobicRestrictor (avoids Pacific Ocean or two atlantic crossings)
+# or ContinentRestrictor (avoids doing more than N continent crossings)
+# - Mathematical/empirical study of predecessor expectation
+# - If middle node on the same continent as exit, exit learns nothing
+# - else, exit has a bias on the continent of origin of user
+# - Language and browser accept string determine this anyway
+# - EchelonPhobicRestrictor
+# - Does not cross international boundaries for client->Entry or
+# Exit->destination hops
+
+class PercentileRestriction(NodeRestriction):
+ """Restriction to cut out a percentile slice of the network."""
+ def __init__(self, pct_skip, pct_fast, r_list):
+ """Constructor. Sets up the restriction such that routers in the
+ 'pct_skip' to 'pct_fast' percentile of bandwidth rankings are
+ returned from the sorted list 'r_list'"""
+ self.pct_fast = pct_fast
+ self.pct_skip = pct_skip
+ self.sorted_r = r_list
+
+ def r_is_ok(self, r):
+ "Returns true if r is in the percentile boundaries (by rank)"
+ # Hrmm.. technically we shouldn't count non-running routers in this..
+ # but that is tricky to do efficiently.
+ # XXX: Is there any reason why sorted_r should have non-running
+ # routers in the first place?
+
+ if r.list_rank < len(self.sorted_r)*self.pct_skip/100: return False
+ elif r.list_rank > len(self.sorted_r)*self.pct_fast/100: return False
+
+ return True
+
+class OSRestriction(NodeRestriction):
+ "Restriction based on operating system"
+ def __init__(self, ok, bad=[]):
+ """Constructor. Accept router OSes that match regexes in 'ok',
+ rejects those that match regexes in 'bad'."""
+ self.ok = ok
+ self.bad = bad
+
+ def r_is_ok(self, r):
+ "Returns true if r is in 'ok', false if 'r' is in 'bad'. If 'ok'"
+ for y in self.ok:
+ if re.search(y, r.os):
+ return True
+ for b in self.bad:
+ if re.search(b, r.os):
+ return False
+ if self.ok: return False
+ if self.bad: return True
+
+class ConserveExitsRestriction(NodeRestriction):
+ "Restriction to reject exits from selection"
+ # XXX: Make this adaptive by ip/port
+ def r_is_ok(self, r): return not "Exit" in r.flags
+
+class FlagsRestriction(NodeRestriction):
+ "Restriction for mandatory and forbidden router flags"
+ def __init__(self, mandatory, forbidden=[]):
+ """Constructor. 'mandatory' and 'forbidden' are both lists of router
+ flags as strings."""
+ self.mandatory = mandatory
+ self.forbidden = forbidden
+
+ def r_is_ok(self, router):
+ for m in self.mandatory:
+ if not m in router.flags: return False
+ for f in self.forbidden:
+ if f in router.flags: return False
+ return True
+
+class NickRestriction(NodeRestriction):
+ """Require that the node nickname is as specified"""
+ def __init__(self, nickname):
+ self.nickname = nickname
+
+ def r_is_ok(self, router):
+ return router.nickname == self.nickname
+
+class IdHexRestriction(NodeRestriction):
+ """Require that the node idhash is as specified"""
+ def __init__(self, idhex):
+ if idhex[0] == '$':
+ self.idhex = idhex[1:].upper()
+ else:
+ self.idhex = idhex.upper()
+
+ def r_is_ok(self, router):
+ return router.idhex == self.idhex
+
+class MinBWRestriction(NodeRestriction):
+ """Require a minimum bandwidth"""
+ def __init__(self, minbw):
+ self.min_bw = minbw
+
+ def r_is_ok(self, router): return router.bw >= self.min_bw
+
+class VersionIncludeRestriction(NodeRestriction):
+ """Require that the version match one in the list"""
+ def __init__(self, eq):
+ "Constructor. 'eq' is a list of versions as strings"
+ self.eq = map(TorCtl.RouterVersion, eq)
+
+ def r_is_ok(self, router):
+ """Returns true if the version of 'router' matches one of the
+ specified versions."""
+ for e in self.eq:
+ if e == router.version:
+ return True
+ return False
+
+class VersionExcludeRestriction(NodeRestriction):
+ """Require that the version not match one in the list"""
+ def __init__(self, exclude):
+ "Constructor. 'exclude' is a list of versions as strings"
+ self.exclude = map(TorCtl.RouterVersion, exclude)
+
+ def r_is_ok(self, router):
+ """Returns false if the version of 'router' matches one of the
+ specified versions."""
+ for e in self.exclude:
+ if e == router.version:
+ return False
+ return True
+
+class VersionRangeRestriction(NodeRestriction):
+ """Require that the versions be inside a specified range"""
+ def __init__(self, gr_eq, less_eq=None):
+ self.gr_eq = TorCtl.RouterVersion(gr_eq)
+ if less_eq: self.less_eq = TorCtl.RouterVersion(less_eq)
+ else: self.less_eq = None
+
+ def r_is_ok(self, router):
+ return (not self.gr_eq or router.version >= self.gr_eq) and \
+ (not self.less_eq or router.version <= self.less_eq)
+
+class ExitPolicyRestriction(NodeRestriction):
+ """Require that a router exit to an ip+port"""
+ def __init__(self, to_ip, to_port):
+ self.to_ip = to_ip
+ self.to_port = to_port
+
+ def r_is_ok(self, r): return r.will_exit_to(self.to_ip, self.to_port)
+
+class MetaNodeRestriction(NodeRestriction):
+ """Interface for a NodeRestriction that is an expression consisting of
+ multiple other NodeRestrictions"""
+ # TODO: these should collapse the restriction and return a new
+ # instance for re-insertion (or None)
+ def next_rstr(self): raise NotImplemented()
+ def del_restriction(self, RestrictionClass): raise NotImplemented()
+
+class OrNodeRestriction(MetaNodeRestriction):
+ """MetaNodeRestriction that is the boolean or of two or more
+ NodeRestrictions"""
+ def __init__(self, rs):
+ "Constructor. 'rs' is a list of NodeRestrictions"
+ self.rstrs = rs
+
+ def r_is_ok(self, r):
+ "Returns true if one of 'rs' is true for this router"
+ for rs in self.rstrs:
+ if rs.r_is_ok(r):
+ return True
+ return False
+
+class NotNodeRestriction(MetaNodeRestriction):
+ """Negates a single restriction"""
+ def __init__(self, a):
+ self.a = a
+
+ def r_is_ok(self, r): return not self.a.r_is_ok(r)
+
+class AtLeastNNodeRestriction(MetaNodeRestriction):
+ """MetaNodeRestriction that is true if at least n member
+ restrictions are true."""
+ def __init__(self, rstrs, n):
+ self.rstrs = rstrs
+ self.n = n
+
+ def r_is_ok(self, r):
+ cnt = 0
+ for rs in self.rstrs:
+ if rs.r_is_ok(r):
+ cnt += 1
+ if cnt < self.n: return False
+ else: return True
+
+
+#################### Path Restrictions #####################
+
+class Subnet16Restriction(PathRestriction):
+ """PathRestriction that mandates that no two nodes from the same
+ /16 subnet be in the path"""
+ def path_is_ok(self, path):
+ mask16 = struct.unpack(">I", socket.inet_aton("255.255.0.0"))[0]
+ ip16 = path[0].ip & mask16
+ for r in path[1:]:
+ if ip16 == (r.ip & mask16):
+ return False
+ return True
+
+class UniqueRestriction(PathRestriction):
+ """Path restriction that mandates that the same router can't appear more
+ than once in a path"""
+ def path_is_ok(self, path):
+ for i in xrange(0,len(path)):
+ if path[i] in path[:i]:
+ return False
+ return True
+
+#################### GeoIP Restrictions ###################
+
+class CountryCodeRestriction(NodeRestriction):
+ """ Ensure that the country_code is set """
+ def r_is_ok(self, r):
+ return r.country_code != None
+
+class CountryRestriction(NodeRestriction):
+ """ Only accept nodes that are in 'country_code' """
+ def __init__(self, country_code):
+ self.country_code = country_code
+
+ def r_is_ok(self, r):
+ return r.country_code == self.country_code
+
+class ExcludeCountriesRestriction(NodeRestriction):
+ """ Exclude a list of countries """
+ def __init__(self, countries):
+ self.countries = countries
+
+ def r_is_ok(self, r):
+ return not (r.country_code in self.countries)
+
+class UniqueCountryRestriction(PathRestriction):
+ """ Ensure every router to have a distinct country_code """
+ def path_is_ok(self, path):
+ for i in xrange(0, len(path)-1):
+ for j in xrange(i+1, len(path)):
+ if path[i].country_code == path[j].country_code:
+ return False;
+ return True;
+
+class SingleCountryRestriction(PathRestriction):
+ """ Ensure every router to have the same country_code """
+ def path_is_ok(self, path):
+ country_code = path[0].country_code
+ for r in path:
+ if country_code != r.country_code:
+ return False
+ return True
+
+class ContinentRestriction(PathRestriction):
+ """ Do not more than n continent crossings """
+ # TODO: Add src and dest
+ def __init__(self, n, src=None, dest=None):
+ self.n = n
+
+ def path_is_ok(self, path):
+ crossings = 0
+ prev = None
+ # Compute crossings until now
+ for r in path:
+ # Jump over the first router
+ if prev:
+ if r.continent != prev.continent:
+ crossings += 1
+ prev = r
+ if crossings > self.n: return False
+ else: return True
+
+class ContinentJumperRestriction(PathRestriction):
+ """ Ensure continent crossings between all hops """
+ def path_is_ok(self, path):
+ prev = None
+ for r in path:
+ # Jump over the first router
+ if prev:
+ if r.continent == prev.continent:
+ return False
+ prev = r
+ return True
+
+class UniqueContinentRestriction(PathRestriction):
+ """ Ensure every hop to be on a different continent """
+ def path_is_ok(self, path):
+ for i in xrange(0, len(path)-1):
+ for j in xrange(i+1, len(path)):
+ if path[i].continent == path[j].continent:
+ return False;
+ return True;
+
+class OceanPhobicRestriction(PathRestriction):
+ """ Not more than n ocean crossings """
+ # TODO: Add src and dest
+ def __init__(self, n, src=None, dest=None):
+ self.n = n
+
+ def path_is_ok(self, path):
+ crossings = 0
+ prev = None
+ # Compute ocean crossings until now
+ for r in path:
+ # Jump over the first router
+ if prev:
+ if r.cont_group != prev.cont_group:
+ crossings += 1
+ prev = r
+ if crossings > self.n: return False
+ else: return True
+
+#################### Node Generators ######################
+
+class UniformGenerator(NodeGenerator):
+ """NodeGenerator that produces nodes in the uniform distribution"""
+ def generate(self):
+ while not self.all_chosen():
+ r = random.choice(self.routers)
+ if self.rstr_list.r_is_ok(r): yield r
+
+class OrderedExitGenerator(NodeGenerator):
+ """NodeGenerator that produces exits in an ordered fashion for a
+ specific port"""
+ def __init__(self, to_port, sorted_r, rstr_list):
+ self.to_port = to_port
+ self.next_exit_by_port = {}
+ NodeGenerator.__init__(self, sorted_r, rstr_list)
+
+ def rewind(self):
+ if self.to_port not in self.next_exit_by_port or not self.next_exit_by_port[self.to_port]:
+ self.next_exit_by_port[self.to_port] = 0
+ self.last_idx = len(self.sorted_r)
+ else:
+ self.last_idx = self.next_exit_by_port[self.to_port]
+
+ def set_port(self, port):
+ self.to_port = port
+ self.rewind()
+
+ def mark_chosen(self, r):
+ self.next_exit_by_port[self.to_port] += 1
+
+ def all_chosen(self):
+ return self.last_idx == self.next_exit_by_port[self.to_port]
+
+ def generate(self):
+ while True: # A do..while would be real nice here..
+ if self.next_exit_by_port[self.to_port] >= len(self.sorted_r):
+ self.next_exit_by_port[self.to_port] = 0
+ r = self.sorted_r[self.next_exit_by_port[self.to_port]]
+ if self.rstr_list.r_is_ok(r): yield r
+ else: self.next_exit_by_port[self.to_port] += 1
+ if self.last_idx == self.next_exit_by_port[self.to_port]:
+ break
+
+class BwWeightedGenerator(NodeGenerator):
+ """
+
+ This is a generator designed to match the Tor Path Selection
+ algorithm. It will generate nodes weighted by their bandwidth,
+ but take the appropriate weighting into account against guard
+ nodes and exit nodes when they are chosen for positions other
+ than guard/exit. For background see:
+ routerlist.c::smartlist_choose_by_bandwidth(),
+ http://archives.seul.org/or/dev/Jul-2007/msg00021.html,
+ http://archives.seul.org/or/dev/Jul-2007/msg00056.html, and
+ https://tor-svn.freehaven.net/svn/tor/trunk/doc/spec/path-spec.txt
+ The formulas used are from the first or-dev link, but are proven
+ optimal and equivalent to the ones now used in routerlist.c in the
+ second or-dev link.
+
+ """
+ def __init__(self, sorted_r, rstr_list, pathlen, exit=False, guard=False):
+ """ Pass exit=True to create a generator for exit-nodes """
+ self.max_bandwidth = 10000000
+ # Out for an exit-node?
+ self.exit = exit
+ # Is this a guard node?
+ self.guard = guard
+ # Different sums of bandwidths
+ self.total_bw = 0
+ self.total_exit_bw = 0
+ self.total_guard_bw = 0
+ self.total_weighted_bw = 0
+ self.pathlen = pathlen
+ NodeGenerator.__init__(self, sorted_r, rstr_list)
+
+ def rewind(self):
+ NodeGenerator.rewind(self)
+ # Set the exit_weight
+ # We are choosing a non-exit
+ self.total_exit_bw = 0
+ self.total_guard_bw = 0
+ self.total_bw = 0
+ for r in self.sorted_r:
+ # Should this be outside the restriction checks?
+ # TODO: Check max_bandwidth and cap...
+ if self.rstr_list.r_is_ok(r):
+ self.total_bw += r.bw
+ if "Exit" in r.flags:
+ self.total_exit_bw += r.bw
+ if "Guard" in r.flags:
+ self.total_guard_bw += r.bw
+
+ bw_per_hop = (1.0*self.total_bw)/self.pathlen
+
+ # Print some debugging info about bandwidth ratios
+ if self.total_bw > 0:
+ e_ratio = self.total_exit_bw/float(self.total_bw)
+ g_ratio = self.total_guard_bw/float(self.total_bw)
+ else:
+ g_ratio = 0
+ e_ratio = 0
+ plog("DEBUG",
+ "E = " + str(self.total_exit_bw) +
+ ", G = " + str(self.total_guard_bw) +
+ ", T = " + str(self.total_bw) +
+ ", g_ratio = " + str(g_ratio) + ", e_ratio = " +str(e_ratio) +
+ ", bw_per_hop = " + str(bw_per_hop))
+
+ if self.exit:
+ self.exit_weight = 1.0
+ else:
+ if self.total_exit_bw < bw_per_hop:
+ # Don't use exit nodes at all
+ self.exit_weight = 0
+ else:
+ if self.total_exit_bw > 0:
+ self.exit_weight = ((self.total_exit_bw-bw_per_hop)/self.total_exit_bw)
+ else: self.exit_weight = 0
+
+ if self.guard:
+ self.guard_weight = 1.0
+ else:
+ if self.total_guard_bw < bw_per_hop:
+ # Don't use exit nodes at all
+ self.guard_weight = 0
+ else:
+ if self.total_guard_bw > 0:
+ self.guard_weight = ((self.total_guard_bw-bw_per_hop)/self.total_guard_bw)
+ else: self.guard_weight = 0
+
+ for r in self.sorted_r:
+ bw = r.bw
+ if "Exit" in r.flags:
+ bw *= self.exit_weight
+ if "Guard" in r.flags:
+ bw *= self.guard_weight
+ self.total_weighted_bw += bw
+
+ self.total_weighted_bw = int(self.total_weighted_bw)
+ plog("DEBUG", "Bw: "+str(self.total_weighted_bw)+"/"+str(self.total_bw)
+ +". The exit-weight is: "+str(self.exit_weight)
+ + ", guard weight is: "+str(self.guard_weight))
+
+ def generate(self):
+ while True:
+ # Choose a suitable random int
+ i = random.randint(0, self.total_weighted_bw)
+
+ # Go through the routers
+ for r in self.routers:
+ # Below zero here means next() -> choose a new random int+router
+ if i < 0: break
+ if self.rstr_list.r_is_ok(r):
+ bw = r.bw
+ if "Exit" in r.flags:
+ bw *= self.exit_weight
+ if "Guard" in r.flags:
+ bw *= self.guard_weight
+
+ i -= bw
+ if i < 0:
+ plog("DEBUG", "Chosen router with a bandwidth of: " + str(r.bw))
+ yield r
+
+####################### Secret Sauce ###########################
+
+class PathError(Exception):
+ pass
+
+class NoRouters(PathError):
+ pass
+
+class PathSelector:
+ """Implementation of path selection policies. Builds a path according
+ to entry, middle, and exit generators that satisfies the path
+ restrictions."""
+ def __init__(self, entry_gen, mid_gen, exit_gen, path_restrict):
+ """Constructor. The first three arguments are NodeGenerators with
+ their appropriate restrictions. The 'path_restrict' is a
+ PathRestrictionList"""
+ self.entry_gen = entry_gen
+ self.mid_gen = mid_gen
+ self.exit_gen = exit_gen
+ self.path_restrict = path_restrict
+
+ def build_path(self, pathlen):
+ """Creates a path of 'pathlen' hops, and returns it as a list of
+ Router instances"""
+ self.entry_gen.rewind()
+ self.mid_gen.rewind()
+ self.exit_gen.rewind()
+ entry = self.entry_gen.generate()
+ mid = self.mid_gen.generate()
+ ext = self.exit_gen.generate()
+
+ while True:
+ path = []
+ try:
+ if pathlen == 1:
+ path = [ext.next()]
+ else:
+ path.append(entry.next())
+ for i in xrange(1, pathlen-1):
+ path.append(mid.next())
+ path.append(ext.next())
+ if self.path_restrict.path_is_ok(path):
+ self.entry_gen.mark_chosen(path[0])
+ for i in xrange(1, pathlen-1):
+ self.mid_gen.mark_chosen(path[i])
+ self.exit_gen.mark_chosen(path[pathlen-1])
+ break
+ except StopIteration:
+ plog("NOTICE", "Ran out of routers during buildpath..");
+ self.entry_gen.rewind()
+ self.mid_gen.rewind()
+ self.exit_gen.rewind()
+ entry = self.entry_gen.generate()
+ mid = self.entry_gen.generate()
+ ext = self.entry_gen.generate()
+ return path
+
+class SelectionManager:
+ """Helper class to handle configuration updates
+
+ The methods are NOT threadsafe. They may ONLY be called from
+ EventHandler's thread. This means that to update the selection
+ manager, you must schedule a config update job using
+ PathBuilder.schedule_selmgr() with a worker function to modify
+ this object.
+ """
+ def __init__(self, pathlen, order_exits,
+ percent_fast, percent_skip, min_bw, use_all_exits,
+ uniform, use_exit, use_guards,geoip_config=None,restrict_guards=False):
+ self.__ordered_exit_gen = None
+ self.pathlen = pathlen
+ self.order_exits = order_exits
+ self.percent_fast = percent_fast
+ self.percent_skip = percent_skip
+ self.min_bw = min_bw
+ self.use_all_exits = use_all_exits
+ self.uniform = uniform
+ self.exit_name = use_exit
+ self.use_guards = use_guards
+ self.geoip_config = geoip_config
+
+ self.restrict_guards_only = restrict_guards
+
+ def reconfigure(self, sorted_r):
+ """This function is called after a configuration change,
+ to rebuild the RestrictionLists."""
+ if self.use_all_exits:
+ self.path_rstr = PathRestrictionList([UniqueRestriction()])
+ else:
+ self.path_rstr = PathRestrictionList(
+ [Subnet16Restriction(), UniqueRestriction()])
+
+ if self.use_guards: entry_flags = ["Guard", "Valid", "Running"]
+ else: entry_flags = ["Valid", "Running"]
+
+
+ if self.restrict_guards_only:
+ entry_flags = ["Guard","Valid","Running"]
+ nonentry_skip = 0
+ nonentry_fast = 100
+ else:
+ entry_flags = ["Valid","Running"]
+ nonentry_skip = self.percent_skip
+ nonentry_fast = self.percent_fast
+
+ entry_rstr = NodeRestrictionList(
+ [PercentileRestriction(self.percent_skip, self.percent_fast, sorted_r),
+ ConserveExitsRestriction(),
+ FlagsRestriction(entry_flags, [])]
+ )
+ mid_rstr = NodeRestrictionList(
+ [PercentileRestriction(nonentry_skip, nonentry_fast, sorted_r),
+ ConserveExitsRestriction(),
+ FlagsRestriction(["Running","Fast"], [])]
+
+ )
+ if self.use_all_exits:
+ self.exit_rstr = NodeRestrictionList(
+ [FlagsRestriction(["Valid", "Running","Fast"], ["BadExit"])])
+ else:
+ self.exit_rstr = NodeRestrictionList(
+ [PercentileRestriction(nonentry_skip, nonentry_fast, sorted_r),
+ FlagsRestriction(["Valid", "Running","Fast"], ["BadExit"])])
+
+ if self.exit_name:
+ self.exit_rstr.del_restriction(IdHexRestriction)
+ self.exit_rstr.del_restriction(NickRestriction)
+ if self.exit_name[0] == '$':
+ self.exit_rstr.add_restriction(IdHexRestriction(self.exit_name))
+ else:
+ self.exit_rstr.add_restriction(NickRestriction(self.exit_name))
+
+ # GeoIP configuration
+ if self.geoip_config:
+ # Every node needs country_code
+ entry_rstr.add_restriction(CountryCodeRestriction())
+ mid_rstr.add_restriction(CountryCodeRestriction())
+ self.exit_rstr.add_restriction(CountryCodeRestriction())
+
+ # Specified countries for different positions
+ if self.geoip_config.entry_country:
+ entry_rstr.add_restriction(CountryRestriction(self.geoip_config.entry_country))
+ if self.geoip_config.middle_country:
+ mid_rstr.add_restriction(CountryRestriction(self.geoip_config.middle_country))
+ if self.geoip_config.exit_country:
+ self.exit_rstr.add_restriction(CountryRestriction(self.geoip_config.exit_country))
+
+ # Excluded countries
+ if self.geoip_config.excludes:
+ plog("INFO", "Excluded countries: " + str(self.geoip_config.excludes))
+ if len(self.geoip_config.excludes) > 0:
+ entry_rstr.add_restriction(ExcludeCountriesRestriction(self.geoip_config.excludes))
+ mid_rstr.add_restriction(ExcludeCountriesRestriction(self.geoip_config.excludes))
+ self.exit_rstr.add_restriction(ExcludeCountriesRestriction(self.geoip_config.excludes))
+
+ # Unique countries set? None --> pass
+ if self.geoip_config.unique_countries != None:
+ if self.geoip_config.unique_countries:
+ # If True: unique countries
+ self.path_rstr.add_restriction(UniqueCountryRestriction())
+ else:
+ # False: use the same country for all nodes in a path
+ self.path_rstr.add_restriction(SingleCountryRestriction())
+
+ # Specify max number of continent crossings, None means UniqueContinents
+ if self.geoip_config.continent_crossings == None:
+ self.path_rstr.add_restriction(UniqueContinentRestriction())
+ else: self.path_rstr.add_restriction(ContinentRestriction(self.geoip_config.continent_crossings))
+ # Should even work in combination with continent crossings
+ if self.geoip_config.ocean_crossings != None:
+ self.path_rstr.add_restriction(OceanPhobicRestriction(self.geoip_config.ocean_crossings))
+
+ # This is kind of hokey..
+ if self.order_exits:
+ if self.__ordered_exit_gen:
+ exitgen = self.__ordered_exit_gen
+ exitgen.reset_restriction(self.exit_rstr)
+ else:
+ exitgen = self.__ordered_exit_gen = \
+ OrderedExitGenerator(80, sorted_r, self.exit_rstr)
+ elif self.uniform:
+ # 'real' exits should also be chosen when not using 'order_exits'
+ self.exit_rstr.add_restriction(ExitPolicyRestriction("255.255.255.255", 80))
+ exitgen = UniformGenerator(sorted_r, self.exit_rstr)
+ else:
+ self.exit_rstr.add_restriction(ExitPolicyRestriction("255.255.255.255", 80))
+ exitgen = BwWeightedGenerator(sorted_r, self.exit_rstr, self.pathlen, exit=True)
+
+ if self.uniform:
+ self.path_selector = PathSelector(
+ UniformGenerator(sorted_r, entry_rstr),
+ UniformGenerator(sorted_r, mid_rstr),
+ exitgen, self.path_rstr)
+ else:
+ # Remove ConserveExitsRestrictions for entry and middle positions
+ entry_rstr.del_restriction(ConserveExitsRestriction)
+ mid_rstr.del_restriction(ConserveExitsRestriction)
+ self.path_selector = PathSelector(
+ BwWeightedGenerator(sorted_r, entry_rstr, self.pathlen,
+ guard=self.use_guards),
+ BwWeightedGenerator(sorted_r, mid_rstr, self.pathlen),
+ exitgen, self.path_rstr)
+ return
+
+ def set_target(self, ip, port):
+ "Called to update the ExitPolicyRestrictions with a new ip and port"
+ self.exit_rstr.del_restriction(ExitPolicyRestriction)
+ self.exit_rstr.add_restriction(ExitPolicyRestriction(ip, port))
+ if self.__ordered_exit_gen: self.__ordered_exit_gen.set_port(port)
+ # Try to choose an exit node in the destination country
+ # needs an IP != 255.255.255.255
+ if self.geoip_config and self.geoip_config.echelon:
+ import GeoIPSupport
+ c = GeoIPSupport.get_country(ip)
+ if c:
+ plog("INFO", "[Echelon] IP "+ip+" is in ["+c+"]")
+ self.exit_rstr.del_restriction(CountryRestriction)
+ self.exit_rstr.add_restriction(CountryRestriction(c))
+ else:
+ plog("INFO", "[Echelon] Could not determine destination country of IP "+ip)
+ # Try to use a backup country
+ if self.geoip_config.exit_country:
+ self.exit_rstr.del_restriction(CountryRestriction)
+ self.exit_rstr.add_restriction(CountryRestriction(self.geoip_config.exit_country))
+
+class Circuit:
+ "Class to describe a circuit"
+ def __init__(self):
+ self.circ_id = 0
+ self.path = [] # routers
+ self.exit = None
+ self.built = False
+ self.failed = False
+ self.dirty = False
+ self.closed = False
+ self.detached_cnt = 0
+ self.last_extended_at = time.time()
+ self.extend_times = [] # List of all extend-durations
+ self.setup_duration = None # Sum of extend-times
+ self.pending_streams = [] # Which stream IDs are pending us
+
+ def id_path(self):
+ "Returns a list of idhex keys for the path of Routers"
+ return map(lambda r: r.idhex, self.path)
+
+class Stream:
+ "Class to describe a stream"
+ def __init__(self, sid, host, port, kind):
+ self.strm_id = sid
+ self.detached_from = [] # circ id #'s
+ self.pending_circ = None
+ self.circ = None
+ self.host = host
+ self.port = port
+ self.kind = kind
+ self.attached_at = 0
+ self.bytes_read = 0
+ self.bytes_written = 0
+ self.failed = False
+ self.ignored = False # Set if PURPOSE=DIR_*
+ self.failed_reason = None # Cheating a little.. Only used by StatsHandler
+
+ def lifespan(self, now):
+ "Returns the age of the stream"
+ return now-self.attached_at
+
+# TODO: Make passive "PathWatcher" so people can get aggregate
+# node reliability stats for normal usage without us attaching streams
+
+class PathBuilder(TorCtl.EventHandler):
+ """
+ PathBuilder implementation. Handles circuit construction, subject
+ to the constraints of the SelectionManager selmgr.
+
+ Do not access this object from other threads. Instead, use the
+ schedule_* functions to schedule work to be done in the thread
+ of the EventHandler.
+ """
+ def __init__(self, c, selmgr, RouterClass):
+ """Constructor. 'c' is a Connection, 'selmgr' is a SelectionManager,
+ and 'RouterClass' is a class that inherits from Router and is used
+ to create annotated Routers."""
+ TorCtl.EventHandler.__init__(self)
+ self.c = c
+ nslist = c.get_network_status()
+ self.last_exit = None
+ self.new_nym = False
+ self.resolve_port = 0
+ self.num_circuits = 1
+ self.RouterClass = RouterClass
+ self.sorted_r = []
+ self.name_to_key = {}
+ self.routers = {}
+ self.circuits = {}
+ self.streams = {}
+ self.read_routers(nslist)
+ self.selmgr = selmgr
+ self.selmgr.reconfigure(self.sorted_r)
+ self.imm_jobs = Queue.Queue()
+ self.low_prio_jobs = Queue.Queue()
+ self.run_all_jobs = False
+ self.do_reconfigure = False
+ plog("INFO", "Read "+str(len(self.sorted_r))+"/"+str(len(nslist))+" routers")
+
+ def schedule_immediate(self, job):
+ """
+ Schedules an immediate job to be run before the next event is
+ processed.
+ """
+ self.imm_jobs.put(job)
+
+ def schedule_low_prio(self, job):
+ """
+ Schedules a job to be run when a non-time critical event arrives.
+ """
+ self.low_prio_jobs.put(job)
+
+ def schedule_selmgr(self, job):
+ """
+ Schedules an immediate job to be run before the next event is
+ processed. Also notifies the selection manager that it needs
+ to update itself.
+ """
+ def notlambda(this):
+ job(this.selmgr)
+ this.do_reconfigure = True
+ self.schedule_immediate(notlambda)
+
+
+ def heartbeat_event(self, event):
+ """This function handles dispatching scheduled jobs. If you
+ extend PathBuilder and want to implement this function for
+ some reason, be sure to call the parent class"""
+ while not self.imm_jobs.empty():
+ imm_job = self.imm_jobs.get_nowait()
+ imm_job(self)
+
+ if self.do_reconfigure:
+ self.selmgr.reconfigure(self.sorted_r)
+ self.do_reconfigure = False
+
+ if self.run_all_jobs:
+ self.run_all_jobs = False
+ while not self.low_prio_jobs.empty():
+ imm_job = self.low_prio_jobs.get_nowait()
+ imm_job(self)
+ return
+
+ # If event is stream:NEW*/DETACHED or circ BUILT/FAILED,
+ # don't run low prio jobs.. No need to delay streams for them.
+ if isinstance(event, TorCtl.CircuitEvent):
+ if event.status in ("BUILT", "FAILED"):
+ return
+ elif isinstance(event, TorCtl.StreamEvent):
+ if event.status in ("NEW", "NEWRESOLVE", "DETACHED"):
+ return
+
+ # Do the low prio jobs one at a time in case a
+ # higher priority event is queued
+ if not self.low_prio_jobs.empty():
+ delay_job = self.low_prio_jobs.get_nowait()
+ delay_job(self)
+
+ def read_routers(self, nslist):
+ routers = self.c.read_routers(nslist)
+ new_routers = []
+ for r in routers:
+ self.name_to_key[r.nickname] = "$"+r.idhex
+ if r.idhex in self.routers:
+ if self.routers[r.idhex].nickname != r.nickname:
+ plog("NOTICE", "Router "+r.idhex+" changed names from "
+ +self.routers[r.idhex].nickname+" to "+r.nickname)
+ # Must do IN-PLACE update to keep all the refs to this router
+ # valid and current (especially for stats)
+ self.routers[r.idhex].update_to(r)
+ else:
+ rc = self.RouterClass(r)
+ self.routers[rc.idhex] = rc
+ new_routers.append(rc)
+ self.sorted_r.extend(new_routers)
+ self.sorted_r.sort(lambda x, y: cmp(y.bw, x.bw))
+ for i in xrange(len(self.sorted_r)): self.sorted_r[i].list_rank = i
+
+ def build_path(self):
+ """ Get a path from the SelectionManager's PathSelector, can be used
+ e.g. for generating paths without actually creating any circuits """
+ return self.selmgr.path_selector.build_path(self.selmgr.pathlen)
+
+ def attach_stream_any(self, stream, badcircs):
+ "Attach a stream to a valid circuit, avoiding any in 'badcircs'"
+ # Newnym, and warn if not built plus pending
+ unattached_streams = [stream]
+ if self.new_nym:
+ self.new_nym = False
+ plog("DEBUG", "Obeying new nym")
+ for key in self.circuits.keys():
+ if (not self.circuits[key].dirty
+ and len(self.circuits[key].pending_streams)):
+ plog("WARN", "New nym called, destroying circuit "+str(key)
+ +" with "+str(len(self.circuits[key].pending_streams))
+ +" pending streams")
+ unattached_streams.extend(self.circuits[key].pending_streams)
+ self.circuits[key].pending_streams.clear()
+ # FIXME: Consider actually closing circ if no streams.
+ self.circuits[key].dirty = True
+
+ for circ in self.circuits.itervalues():
+ if circ.built and not circ.dirty and circ.circ_id not in badcircs:
+ if circ.exit.will_exit_to(stream.host, stream.port):
+ try:
+ self.c.attach_stream(stream.strm_id, circ.circ_id)
+ stream.pending_circ = circ # Only one possible here
+ circ.pending_streams.append(stream)
+ except TorCtl.ErrorReply, e:
+ # No need to retry here. We should get the failed
+ # event for either the circ or stream next
+ plog("WARN", "Error attaching stream: "+str(e.args))
+ return
+ break
+ else:
+ circ = None
+ self.selmgr.set_target(stream.host, stream.port)
+ while circ == None:
+ try:
+ circ = self.c.build_circuit(
+ self.selmgr.pathlen,
+ self.selmgr.path_selector)
+ except TorCtl.ErrorReply, e:
+ # FIXME: How come some routers are non-existant? Shouldn't
+ # we have gotten an NS event to notify us they
+ # disappeared?
+ plog("NOTICE", "Error building circ: "+str(e.args))
+ for u in unattached_streams:
+ plog("DEBUG",
+ "Attaching "+str(u.strm_id)+" pending build of "+str(circ.circ_id))
+ u.pending_circ = circ
+ circ.pending_streams.extend(unattached_streams)
+ self.circuits[circ.circ_id] = circ
+ self.last_exit = circ.exit
+
+ def circ_status_event(self, c):
+ output = [c.event_name, str(c.circ_id), c.status]
+ if c.path: output.append(",".join(c.path))
+ if c.reason: output.append("REASON=" + c.reason)
+ if c.remote_reason: output.append("REMOTE_REASON=" + c.remote_reason)
+ plog("DEBUG", " ".join(output))
+ # Circuits we don't control get built by Tor
+ if c.circ_id not in self.circuits:
+ plog("DEBUG", "Ignoring circ " + str(c.circ_id))
+ return
+ if c.status == "EXTENDED":
+ self.circuits[c.circ_id].last_extended_at = c.arrived_at
+ elif c.status == "FAILED" or c.status == "CLOSED":
+ # XXX: Can still get a STREAM FAILED for this circ after this
+ circ = self.circuits[c.circ_id]
+ del self.circuits[c.circ_id]
+ for stream in circ.pending_streams:
+ plog("DEBUG", "Finding new circ for " + str(stream.strm_id))
+ self.attach_stream_any(stream, stream.detached_from)
+ elif c.status == "BUILT":
+ self.circuits[c.circ_id].built = True
+ try:
+ for stream in self.circuits[c.circ_id].pending_streams:
+ self.c.attach_stream(stream.strm_id, c.circ_id)
+ except TorCtl.ErrorReply, e:
+ # No need to retry here. We should get the failed
+ # event for either the circ or stream next
+ plog("WARN", "Error attaching stream: "+str(e.args))
+ return
+
+ def stream_status_event(self, s):
+ output = [s.event_name, str(s.strm_id), s.status, str(s.circ_id),
+ s.target_host, str(s.target_port)]
+ if s.reason: output.append("REASON=" + s.reason)
+ if s.remote_reason: output.append("REMOTE_REASON=" + s.remote_reason)
+ if s.purpose: output.append("PURPOSE=" + s.purpose)
+ plog("DEBUG", " ".join(output))
+ if not re.match(r"\d+.\d+.\d+.\d+", s.target_host):
+ s.target_host = "255.255.255.255" # ignore DNS for exit policy check
+
+ # Hack to ignore Tor-handled streams (Currently only directory streams)
+ if s.strm_id in self.streams and self.streams[s.strm_id].ignored:
+ plog("DEBUG", "Ignoring stream: " + str(s.strm_id))
+ return
+
+ if s.status == "NEW" or s.status == "NEWRESOLVE":
+ if s.status == "NEWRESOLVE" and not s.target_port:
+ s.target_port = self.resolve_port
+ self.streams[s.strm_id] = Stream(s.strm_id, s.target_host, s.target_port, s.status)
+ # Remember Tor-handled streams (Currently only directory streams)
+ if s.purpose and s.purpose.find("DIR_") == 0:
+ self.streams[s.strm_id].ignored = True
+ plog("DEBUG", "Ignoring stream: " + str(s.strm_id))
+ return
+ else:
+ self.attach_stream_any(self.streams[s.strm_id],
+ self.streams[s.strm_id].detached_from)
+ elif s.status == "DETACHED":
+ if s.strm_id not in self.streams:
+ plog("WARN", "Detached stream "+str(s.strm_id)+" not found")
+ self.streams[s.strm_id] = Stream(s.strm_id, s.target_host,
+ s.target_port, "NEW")
+ # FIXME Stats (differentiate Resolved streams also..)
+ if not s.circ_id:
+ plog("WARN", "Stream "+str(s.strm_id)+" detached from no circuit!")
+ else:
+ self.streams[s.strm_id].detached_from.append(s.circ_id)
+
+ if self.streams[s.strm_id] in self.streams[s.strm_id].pending_circ.pending_streams:
+ self.streams[s.strm_id].pending_circ.pending_streams.remove(self.streams[s.strm_id])
+ self.streams[s.strm_id].pending_circ = None
+ self.attach_stream_any(self.streams[s.strm_id],
+ self.streams[s.strm_id].detached_from)
+ elif s.status == "SUCCEEDED":
+ if s.strm_id not in self.streams:
+ plog("NOTICE", "Succeeded stream "+str(s.strm_id)+" not found")
+ return
+ if s.circ_id and self.streams[s.strm_id].pending_circ.circ_id != s.circ_id:
+ # Hrmm.. this can happen on a new-nym.. Very rare, putting warn
+ # in because I'm still not sure this is correct
+ plog("WARN", "Mismatch of pending: "
+ +str(self.streams[s.strm_id].pending_circ.circ_id)+" vs "
+ +str(s.circ_id))
+ # This can happen if the circuit existed before we started up
+ if s.circ_id in self.circuits:
+ self.streams[s.strm_id].circ = self.circuits[s.circ_id]
+ else:
+ plog("NOTICE", "Stream "+str(s.strm_id)+" has unknown circuit: "+str(s.circ_id))
+ else:
+ self.streams[s.strm_id].circ = self.streams[s.strm_id].pending_circ
+ self.streams[s.strm_id].pending_circ.pending_streams.remove(self.streams[s.strm_id])
+ self.streams[s.strm_id].pending_circ = None
+ self.streams[s.strm_id].attached_at = s.arrived_at
+ elif s.status == "FAILED" or s.status == "CLOSED":
+ # FIXME stats
+ if s.strm_id not in self.streams:
+ plog("NOTICE", "Failed stream "+str(s.strm_id)+" not found")
+ return
+
+ if not s.circ_id:
+ plog("WARN", "Stream "+str(s.strm_id)+" failed from no circuit!")
+
+ # We get failed and closed for each stream. OK to return
+ # and let the closed do the cleanup
+ if s.status == "FAILED":
+ # Avoid busted circuits that will not resolve or carry
+ # traffic.
+ self.streams[s.strm_id].failed = True
+ if s.circ_id in self.circuits: self.circuits[s.circ_id].dirty = True
+ else: plog("WARN","Failed stream on unknown circ "+str(s.circ_id))
+ return
+
+ if self.streams[s.strm_id].pending_circ:
+ self.streams[s.strm_id].pending_circ.pending_streams.remove(self.streams[s.strm_id])
+ del self.streams[s.strm_id]
+ elif s.status == "REMAP":
+ if s.strm_id not in self.streams:
+ plog("WARN", "Remap id "+str(s.strm_id)+" not found")
+ else:
+ if not re.match(r"\d+.\d+.\d+.\d+", s.target_host):
+ s.target_host = "255.255.255.255"
+ plog("NOTICE", "Non-IP remap for "+str(s.strm_id)+" to "
+ + s.target_host)
+ self.streams[s.strm_id].host = s.target_host
+ self.streams[s.strm_id].port = s.target_port
+
+ def stream_bw_event(self, s):
+ output = [s.event_name, str(s.strm_id), str(s.bytes_read),
+ str(s.bytes_written)]
+ plog("DEBUG", " ".join(output))
+ if not s.strm_id in self.streams:
+ plog("WARN", "BW event for unknown stream id: "+str(s.strm_id))
+ else:
+ self.streams[s.strm_id].bytes_read += s.bytes_read
+ self.streams[s.strm_id].bytes_written += s.bytes_written
+
+ def ns_event(self, n):
+ self.read_routers(n.nslist)
+ plog("DEBUG", "Read " + str(len(n.nslist))+" NS => "
+ + str(len(self.sorted_r)) + " routers")
+
+ def new_desc_event(self, d):
+ for i in d.idlist: # Is this too slow?
+ self.read_routers(self.c.get_network_status("id/"+i))
+ plog("DEBUG", "Read " + str(len(d.idlist))+" Desc => "
+ + str(len(self.sorted_r)) + " routers")
+
+ def bandwidth_event(self, b): pass # For heartbeat only..
+
+################### CircuitHandler #############################
+
+class CircuitHandler(PathBuilder):
+ """ CircuitHandler that extends from PathBuilder to handle multiple
+ circuits as opposed to just one. """
+ def __init__(self, c, selmgr, num_circuits, RouterClass):
+ """Constructor. 'c' is a Connection, 'selmgr' is a SelectionManager,
+ 'num_circuits' is the number of circuits to keep in the pool,
+ and 'RouterClass' is a class that inherits from Router and is used
+ to create annotated Routers."""
+ PathBuilder.__init__(self, c, selmgr, RouterClass)
+ # Set handler to the connection here to
+ # not miss any circuit events on startup
+ c.set_event_handler(self)
+ self.num_circuits = num_circuits # Size of the circuit pool
+ self.check_circuit_pool() # Bring up the pool of circs
+
+ def check_circuit_pool(self):
+ """ Init or check the status of the circuit-pool """
+ # Get current number of circuits
+ n = len(self.circuits.values())
+ i = self.num_circuits-n
+ if i > 0:
+ plog("INFO", "Checked pool of circuits: we need to build " +
+ str(i) + " circuits")
+ # Schedule (num_circs-n) circuit-buildups
+ while (n < self.num_circuits):
+ self.build_circuit("255.255.255.255", 80)
+ plog("DEBUG", "Scheduled circuit No. " + str(n+1))
+ n += 1
+
+ def build_circuit(self, host, port):
+ """ Build a circuit """
+ circ = None
+ while circ == None:
+ try:
+ self.selmgr.set_target(host, port)
+ circ = self.c.build_circuit(self.selmgr.pathlen,
+ self.selmgr.path_selector)
+ self.circuits[circ.circ_id] = circ
+ return circ
+ except TorCtl.ErrorReply, e:
+ # FIXME: How come some routers are non-existant? Shouldn't
+ # we have gotten an NS event to notify us they disappeared?
+ plog("NOTICE", "Error building circuit: " + str(e.args))
+
+ def close_circuit(self, id):
+ """ Close a circuit with given id """
+ # TODO: Pass streams to another circ before closing?
+ self.circuits[id].closed = True
+ try: self.c.close_circuit(id)
+ except TorCtl.ErrorReply, e:
+ plog("ERROR", "Failed closing circuit " + str(id) + ": " + str(e))
+
+ def circ_status_event(self, c):
+ """ Handle circuit status events """
+ output = [c.event_name, str(c.circ_id), c.status]
+ if c.path: output.append(",".join(c.path))
+ if c.reason: output.append("REASON=" + c.reason)
+ if c.remote_reason: output.append("REMOTE_REASON=" + c.remote_reason)
+ plog("DEBUG", " ".join(output))
+
+ # Circuits we don't control get built by Tor
+ if c.circ_id not in self.circuits:
+ plog("DEBUG", "Ignoring circuit " + str(c.circ_id) +
+ " (controlled by Tor)")
+ return
+
+ # EXTENDED
+ if c.status == "EXTENDED":
+ # Compute elapsed time
+ extend_time = c.arrived_at-self.circuits[c.circ_id].last_extended_at
+ self.circuits[c.circ_id].extend_times.append(extend_time)
+ plog("INFO", "Circuit " + str(c.circ_id) + " extended in " +
+ str(extend_time) + " sec")
+ self.circuits[c.circ_id].last_extended_at = c.arrived_at
+
+ # FAILED & CLOSED
+ elif c.status == "FAILED" or c.status == "CLOSED":
+ # XXX: Can still get a STREAM FAILED for this circ after this
+ circ = self.circuits[c.circ_id]
+ # Actual removal of the circ
+ del self.circuits[c.circ_id]
+ # Give away pending streams
+ for stream in circ.pending_streams:
+ plog("DEBUG", "Finding new circ for " + str(stream.strm_id))
+ self.attach_stream_any(stream, stream.detached_from)
+ # Check if there are enough circs
+ self.check_circuit_pool()
+ return
+
+ # BUILT
+ elif c.status == "BUILT":
+ circ = self.circuits[c.circ_id]
+ circ.built = True
+ for stream in circ.pending_streams:
+ try:
+ self.c.attach_stream(stream.strm_id, c.circ_id)
+ except TorCtl.ErrorReply, e:
+ # No need to retry here. We should get the failed
+ # event for either the circ or stream next
+ plog("WARN", "Error attaching stream: " + str(e.args))
+ # Compute duration by summing up extend_times
+ duration = reduce(lambda x, y: x+y, circ.extend_times, 0.0)
+ plog("INFO", "Circuit " + str(c.circ_id) + " needed " +
+ str(duration) + " seconds to be built")
+ # Save the duration to the circuit for later use
+ circ.setup_duration = duration
+
+ # OTHER?
+ else:
+ # If this was e.g. a LAUNCHED
+ pass
+
+################### StreamHandler ##############################
+
+class StreamHandler(CircuitHandler):
+ """ StreamHandler that extends from the CircuitHandler
+ to handle attaching streams to an appropriate circuit
+ in the pool. """
+ def __init__(self, c, selmgr, num_circs, RouterClass):
+ CircuitHandler.__init__(self, c, selmgr, num_circs, RouterClass)
+ self.sorted_circs = None # optional sorted list
+
+ def clear_dns_cache(self):
+ """ Send signal CLEARDNSCACHE """
+ lines = self.c.sendAndRecv("SIGNAL CLEARDNSCACHE\r\n")
+ for _, msg, more in lines:
+ plog("DEBUG", "CLEARDNSCACHE: " + msg)
+
+ def close_stream(self, id, reason):
+ """ Close a stream with given id and reason """
+ self.c.close_stream(id, reason)
+
+ def create_and_attach(self, stream, unattached_streams):
+ """ Create a new circuit and attach (stream + unattached_streams) """
+ circ = self.build_circuit(stream.host, stream.port)
+ if circ:
+ for u in unattached_streams:
+ plog("DEBUG", "Attaching " + str(u.strm_id) +
+ " pending build of circuit " + str(circ.circ_id))
+ u.pending_circ = circ
+ circ.pending_streams.extend(unattached_streams)
+ self.circuits[circ.circ_id] = circ
+ self.last_exit = circ.exit
+
+ def attach_stream_any(self, stream, badcircs):
+ """ Attach a regular user stream """
+ unattached_streams = [stream]
+ if self.new_nym:
+ self.new_nym = False
+ plog("DEBUG", "Obeying new nym")
+ for key in self.circuits.keys():
+ if (not self.circuits[key].dirty
+ and len(self.circuits[key].pending_streams)):
+ plog("WARN", "New nym called, destroying circuit "+str(key)
+ +" with "+str(len(self.circuits[key].pending_streams))
+ +" pending streams")
+ unattached_streams.extend(self.circuits[key].pending_streams)
+ del self.circuits[key].pending_streams[:]
+ # FIXME: Consider actually closing circs if no streams
+ self.circuits[key].dirty = True
+
+ # Check if there is a sorted list of circs
+ if self.sorted_circs: list = self.sorted_circs
+ else: list = self.circuits.values()
+ for circ in list:
+ # Check each circuit
+ if circ.built and not circ.closed and circ.circ_id not in badcircs and not circ.dirty:
+ if circ.exit.will_exit_to(stream.host, stream.port):
+ try:
+ self.c.attach_stream(stream.strm_id, circ.circ_id)
+ stream.pending_circ = circ # Only one possible here
+ circ.pending_streams.append(stream)
+ self.last_exit = circ.exit
+ except TorCtl.ErrorReply, e:
+ # No need to retry here. We should get the failed
+ # event for either the circ or stream next
+ plog("WARN", "Error attaching stream: " + str(e.args))
+ return
+ break
+ else:
+ plog("DEBUG", "Circuit " + str(circ.circ_id) + " won't exit")
+ else:
+ self.create_and_attach(stream, unattached_streams)
+
+ def stream_status_event(self, s):
+ """ Catch user stream events """
+ # Construct debugging output
+ output = [s.event_name, str(s.strm_id), s.status, str(s.circ_id), s.target_host+':'+str(s.target_port)]
+ if s.reason: output.append("REASON=" + s.reason)
+ if s.remote_reason: output.append("REMOTE_REASON=" + s.remote_reason)
+ if s.purpose: output.append("PURPOSE=" + s.purpose)
+ plog("DEBUG", " ".join(output))
+
+ # If target_host is not an IP-address
+ if not re.match(r"\d+.\d+.\d+.\d+", s.target_host):
+ s.target_host = "255.255.255.255" # ignore DNS for exit policy check
+
+ # Hack to ignore Tor-handled streams (Currently only directory streams)
+ if s.strm_id in self.streams and self.streams[s.strm_id].ignored:
+ plog("DEBUG", "Ignoring stream: " + str(s.strm_id))
+ return
+
+ # NEW or NEWRESOLVE
+ if s.status == "NEW" or s.status == "NEWRESOLVE":
+ if s.status == "NEWRESOLVE" and not s.target_port:
+ s.target_port = self.resolve_port
+ # Set up the new stream
+ stream = Stream(s.strm_id, s.target_host, s.target_port, s.status)
+
+ self.streams[s.strm_id] = stream
+ if s.purpose and s.purpose.find("DIR_") == 0:
+ stream.ignored = True
+ plog("DEBUG", "Ignoring stream: " + str(s.strm_id))
+ return
+ else:
+ self.attach_stream_any(self.streams[s.strm_id], self.streams[s.strm_id].detached_from)
+
+ # DETACHED
+ elif s.status == "DETACHED":
+ # Stream not found
+ if s.strm_id not in self.streams:
+ plog("WARN", "Detached stream " + str(s.strm_id) + " not found")
+ self.streams[s.strm_id] = Stream(s.strm_id, s.target_host, s.target_port, "NEW")
+ # Circuit not found
+ if not s.circ_id:
+ plog("WARN", "Stream " + str(s.strm_id) + " detached from no circuit!")
+ else:
+ self.streams[s.strm_id].detached_from.append(s.circ_id)
+ # Detect timeouts on user streams
+ if s.reason == "TIMEOUT":
+ # TODO: Count timeouts on streams?
+ #self.streams[s.strm_id].timeout_counter += 1
+ plog("DEBUG", "User stream timed out on circuit " + str(s.circ_id))
+ # Stream was pending
+ if self.streams[s.strm_id] in self.streams[s.strm_id].pending_circ.pending_streams:
+ self.streams[s.strm_id].pending_circ.pending_streams.remove(self.streams[s.strm_id])
+ # Attach to another circ
+ self.streams[s.strm_id].pending_circ = None
+ self.attach_stream_any(self.streams[s.strm_id], self.streams[s.strm_id].detached_from)
+
+ # SUCCEEDED
+ if s.status == "SUCCEEDED":
+ if s.strm_id not in self.streams:
+ plog("NOTICE", "Succeeded stream " + str(s.strm_id) + " not found")
+ return
+ if s.circ_id and self.streams[s.strm_id].pending_circ.circ_id != s.circ_id:
+ # Hrmm.. this can happen on a new-nym.. Very rare, putting warn
+ # in because I'm still not sure this is correct
+ plog("WARN", "Mismatch of pending: "
+ + str(self.streams[s.strm_id].pending_circ.circ_id) + " vs "
+ + str(s.circ_id))
+ self.streams[s.strm_id].circ = self.circuits[s.circ_id]
+ else:
+ self.streams[s.strm_id].circ = self.streams[s.strm_id].pending_circ
+ self.streams[s.strm_id].pending_circ.pending_streams.remove(self.streams[s.strm_id])
+ self.streams[s.strm_id].pending_circ = None
+ self.streams[s.strm_id].attached_at = s.arrived_at
+
+ # FAILED or CLOSED
+ elif s.status == "FAILED" or s.status == "CLOSED":
+ if s.strm_id not in self.streams:
+ plog("NOTICE", "Failed stream " + str(s.strm_id) + " not found")
+ return
+ # if not s.circ_id:
+ # plog("WARN", "Stream " + str(s.strm_id) + " closed/failed from no circuit")
+ # We get failed and closed for each stream, let CLOSED do the cleanup
+ if s.status == "FAILED":
+ # Avoid busted circuits that will not resolve or carry traffic
+ self.streams[s.strm_id].failed = True
+ if s.circ_id in self.circuits: self.circuits[s.circ_id].dirty = True
+ elif self.streams[s.strm_id].attached_at != 0:
+ plog("WARN", "Failed stream on unknown circuit " + str(s.circ_id))
+ return
+ # CLOSED
+ if self.streams[s.strm_id].pending_circ:
+ self.streams[s.strm_id].pending_circ.pending_streams.remove(self.streams[s.strm_id])
+ # Actual removal of the stream
+ del self.streams[s.strm_id]
+
+ # REMAP
+ elif s.status == "REMAP":
+ if s.strm_id not in self.streams:
+ plog("WARN", "Remap id "+str(s.strm_id)+" not found")
+ else:
+ if not re.match(r"\d+.\d+.\d+.\d+", s.target_host):
+ s.target_host = "255.255.255.255"
+ plog("NOTICE", "Non-IP remap for "+str(s.strm_id) +
+ " to " + s.target_host)
+ self.streams[s.strm_id].host = s.target_host
+ self.streams[s.strm_id].port = s.target_port
+
+ def address_mapped_event(self, event):
+ """ It is necessary to listen to ADDRMAP events to be able to
+ perform DNS lookups using Tor """
+ output = [event.event_name, event.from_addr, event.to_addr,
+ time.asctime(event.when)]
+ plog("DEBUG", " ".join(output))
+
+ def unknown_event(self, event):
+ plog("DEBUG", "UNKNOWN EVENT '" + event.event_name + "':" +
+ event.event_string)
+
+########################## Unit tests ##########################
+
+def do_gen_unit(gen, r_list, weight_bw, num_print):
+ trials = 0
+ for r in r_list:
+ if gen.rstr_list.r_is_ok(r):
+ trials += weight_bw(gen, r)
+ trials = int(trials/1024)
+
+ print "Running "+str(trials)+" trials"
+
+ # 0. Reset r.chosen = 0 for all routers
+ for r in r_list:
+ r.chosen = 0
+
+ # 1. Generate 'trials' choices:
+ # 1a. r.chosen++
+
+ loglevel = TorUtil.loglevel
+ TorUtil.loglevel = "INFO"
+
+ #gen.rewind() - Just overhead if we create a fresh generator each time
+ rtrs = gen.generate()
+ for i in xrange(1, trials):
+ r = rtrs.next()
+ r.chosen += 1
+
+ TorUtil.loglevel = loglevel
+
+ # 2. Print top num_print routers choices+bandwidth stats+flags
+ i = 0
+ copy_rlist = copy.copy(r_list)
+ copy_rlist.sort(lambda x, y: cmp(y.chosen, x.chosen))
+ for r in copy_rlist:
+ if not gen.rstr_list.r_is_ok(r): continue
+ flag = ""
+ bw = int(weight_bw(gen, r))
+ if "Exit" in r.flags:
+ flag += "E"
+ if "Guard" in r.flags:
+ flag += "G"
+ print str(r.list_rank)+". "+r.nickname+" "+str(r.bw/1024)+"/"+str(bw/1024)+": "+str(r.chosen)+", "+flag
+ i += 1
+ if i > num_print: break
+
+def do_unit(rst, r_list, plamb):
+ print "\n"
+ print "-----------------------------------"
+ print rst.r_is_ok.im_class
+ above_i = 0
+ above_bw = 0
+ below_i = 0
+ below_bw = 0
+ for r in r_list:
+ if rst.r_is_ok(r):
+ print r.nickname+" "+plamb(r)+"="+str(rst.r_is_ok(r))+" "+str(r.bw)
+ if r.bw > 400000:
+ above_i = above_i + 1
+ above_bw += r.bw
+ else:
+ below_i = below_i + 1
+ below_bw += r.bw
+
+ print "Routers above: " + str(above_i) + " bw: " + str(above_bw)
+ print "Routers below: " + str(below_i) + " bw: " + str(below_bw)
+
+# TODO: Tests:
+# - Test each NodeRestriction and print in/out lines for it
+# - Test NodeGenerator and reapply NodeRestrictions
+# - Same for PathSelector and PathRestrictions
+# - Also Reapply each restriction by hand to path. Verify returns true
+
+if __name__ == '__main__':
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ s.connect(("127.0.0.1",9051))
+ c = Connection(s)
+ c.debug(file("control.log", "w"))
+ c.authenticate()
+ nslist = c.get_network_status()
+ sorted_rlist = c.read_routers(c.get_network_status())
+
+ sorted_rlist.sort(lambda x, y: cmp(y.bw, x.bw))
+ for i in xrange(len(sorted_rlist)): sorted_rlist[i].list_rank = i
+
+ def flag_weighting(bwgen, r):
+ bw = r.bw
+ if "Exit" in r.flags:
+ bw *= bwgen.exit_weight
+ if "Guard" in r.flags:
+ bw *= bwgen.guard_weight
+ return bw
+
+ do_gen_unit(BwWeightedGenerator(sorted_rlist, FlagsRestriction(["Exit"]),
+ 3, exit=True),
+ sorted_rlist, flag_weighting, 500)
+
+ do_gen_unit(BwWeightedGenerator(sorted_rlist, FlagsRestriction(["Guard"]),
+ 3, guard=True),
+ sorted_rlist, flag_weighting, 500)
+
+ do_gen_unit(
+ BwWeightedGenerator(sorted_rlist, FlagsRestriction(["Valid"]), 3),
+ sorted_rlist, flag_weighting, 500)
+
+ exit(0)
+
+ for r in sorted_rlist:
+ if r.will_exit_to("211.11.21.22", 465):
+ print r.nickname+" "+str(r.bw)
+
+ do_unit(FlagsRestriction(["Guard"], []), sorted_rlist, lambda r: " ".join(r.flags))
+ do_unit(FlagsRestriction(["Fast"], []), sorted_rlist, lambda r: " ".join(r.flags))
+
+ do_unit(ExitPolicyRestriction("2.11.2.2", 80), sorted_rlist,
+ lambda r: "exits to 80")
+ do_unit(PercentileRestriction(0, 100, sorted_rlist), sorted_rlist,
+ lambda r: "")
+ do_unit(PercentileRestriction(10, 20, sorted_rlist), sorted_rlist,
+ lambda r: "")
+ do_unit(OSRestriction([r"[lL]inux", r"BSD", "Darwin"], []), sorted_rlist,
+ lambda r: r.os)
+ do_unit(OSRestriction([], ["Windows", "Solaris"]), sorted_rlist,
+ lambda r: r.os)
+
+ do_unit(VersionRangeRestriction("0.1.2.0"), sorted_rlist,
+ lambda r: str(r.version))
+ do_unit(VersionRangeRestriction("0.1.2.0", "0.1.2.5"), sorted_rlist,
+ lambda r: str(r.version))
+ do_unit(VersionIncludeRestriction(["0.1.1.26-alpha", "0.1.2.7-ignored"]),
+ sorted_rlist, lambda r: str(r.version))
+ do_unit(VersionExcludeRestriction(["0.1.1.26"]), sorted_rlist,
+ lambda r: str(r.version))
+
+ do_unit(ConserveExitsRestriction(), sorted_rlist, lambda r: " ".join(r.flags))
+ do_unit(FlagsRestriction([], ["Valid"]), sorted_rlist, lambda r: " ".join(r.flags))
+
+ do_unit(IdHexRestriction("$FFCB46DB1339DA84674C70D7CB586434C4370441"),
+ sorted_rlist, lambda r: r.idhex)
+
+ rl = [AtLeastNNodeRestriction([ExitPolicyRestriction("255.255.255.255", 80), ExitPolicyRestriction("255.255.255.255", 443), ExitPolicyRestriction("255.255.255.255", 6667)], 2), FlagsRestriction([], ["BadExit"])]
+
+ exit_rstr = NodeRestrictionList(rl)
+
+ ug = UniformGenerator(sorted_rlist, exit_rstr)
+
+ rlist = []
+ for r in ug.generate():
+ print "Checking: " + r.nickname
+ for rs in rl:
+ if not rs.r_is_ok(r):
+ raise PathError()
+ if not "Exit" in r.flags:
+ print "No exit in flags of "+r.nickname
+ rlist.append(r)
+ for r in sorted_rlist:
+ if "Exit" in r.flags and not r in rlist:
+ print r.nickname+" is an exit not in rl!"
+
Deleted: torflow/branches/gsoc2008/TorCtl.local/README
===================================================================
--- torflow/branches/gsoc2008/TorCtl/README 2008-07-07 16:40:57 UTC (rev 15734)
+++ torflow/branches/gsoc2008/TorCtl.local/README 2008-07-07 19:37:47 UTC (rev 15742)
@@ -1,10 +0,0 @@
-See the pydoc:
-
-# python
-
->>> import TorCtl
->>> help(TorCtl)
-
->>> import PathSupport
->>> help(PathSupport)
-
Copied: torflow/branches/gsoc2008/TorCtl.local/README (from rev 15741, torflow/branches/gsoc2008/TorCtl/README)
===================================================================
--- torflow/branches/gsoc2008/TorCtl.local/README (rev 0)
+++ torflow/branches/gsoc2008/TorCtl.local/README 2008-07-07 19:37:47 UTC (rev 15742)
@@ -0,0 +1,10 @@
+See the pydoc:
+
+# python
+
+>>> import TorCtl
+>>> help(TorCtl)
+
+>>> import PathSupport
+>>> help(PathSupport)
+
Deleted: torflow/branches/gsoc2008/TorCtl.local/StatsSupport.py
===================================================================
--- torflow/branches/gsoc2008/TorCtl/StatsSupport.py 2008-07-07 16:40:57 UTC (rev 15734)
+++ torflow/branches/gsoc2008/TorCtl.local/StatsSupport.py 2008-07-07 19:37:47 UTC (rev 15742)
@@ -1,676 +0,0 @@
-#!/usr/bin/python
-#StatsSupport.py - functions and classes useful for calculating stream/circuit statistics
-
-"""
-
-Support classes for statisics gathering
-
-The StatsSupport package contains several classes that extend PathSupport to
-gather continuous statistics on the Tor network.
-
-The main entrypoint is to extend or create an instance of the StatsHandler
-class. The StatsHandler extends from TorCtl.PathSupport.PathBuilder, which is
-itself a TorCtl.EventHandler. The StatsHandler listens to CIRC and STREAM
-events and gathers all manner of statics on their creation and failure before
-passing the events back up to the PathBuilder code, which manages the actual
-construction and the attachment of streams to circuits.
-
-The package also contains a number of support classes that help gather
-additional statistics on the reliability and performance of routers.
-
-For the purpose of accounting failures, the code tracks two main classes of
-failure: 'actual' failure and 'suspected' failure. The general rule is that an
-actual failure is attributed to the node that directly handled the circuit or
-stream. For streams, this is considered to be the exit node. For circuits, it
-is both the extender and the extendee. 'Suspected' failures, on the other
-hand, are attributed to every member of the circuit up until the extendee for
-circuits, and all hops for streams.
-
-For bandwidth accounting, the average stream bandwidth and the average ratio
-of stream bandwidth to advertised bandwidth are tracked, and when the
-statistics are written, a Z-test is performed to calculate the probabilities
-of these values assuming a normal distribution. Note, however, that it has not
-been verified that this distribution is actually normal. It is likely to be
-something else (pareto, perhaps?).
-
-"""
-
-import sys
-import re
-import random
-import copy
-import time
-import math
-
-import TorUtil, PathSupport, TorCtl
-from TorUtil import *
-from PathSupport import *
-from TorUtil import meta_port, meta_host, control_port, control_host
-
-class ReasonRouterList:
- "Helper class to track which Routers have failed for a given reason"
- def __init__(self, reason):
- self.reason = reason
- self.rlist = {}
-
- def sort_list(self): raise NotImplemented()
-
- def write_list(self, f):
- "Write the list of failure counts for this reason 'f'"
- rlist = self.sort_list()
- for r in rlist:
- susp = 0
- tot_failed = r.circ_failed+r.strm_failed
- tot_susp = tot_failed+r.circ_suspected+r.strm_suspected
- f.write(r.idhex+" ("+r.nickname+") F=")
- if self.reason in r.reason_failed:
- susp = r.reason_failed[self.reason]
- f.write(str(susp)+"/"+str(tot_failed))
- f.write(" S=")
- if self.reason in r.reason_suspected:
- susp += r.reason_suspected[self.reason]
- f.write(str(susp)+"/"+str(tot_susp)+"\n")
-
- def add_r(self, r):
- "Add a router to the list for this reason"
- self.rlist[r] = 1
-
- def total_suspected(self):
- "Get a list of total suspected failures for this reason"
- # suspected is disjoint from failed. The failed table
- # may not have an entry
- def notlambda(x, y):
- if self.reason in y.reason_suspected:
- if self.reason in y.reason_failed:
- return (x + y.reason_suspected[self.reason]
- + y.reason_failed[self.reason])
- else:
- return (x + y.reason_suspected[self.reason])
- else:
- if self.reason in y.reason_failed:
- return (x + y.reason_failed[self.reason])
- else: return x
- return reduce(notlambda, self.rlist.iterkeys(), 0)
-
- def total_failed(self):
- "Get a list of total failures for this reason"
- def notlambda(x, y):
- if self.reason in y.reason_failed:
- return (x + y.reason_failed[self.reason])
- else: return x
- return reduce(notlambda, self.rlist.iterkeys(), 0)
-
-class SuspectRouterList(ReasonRouterList):
- """Helper class to track all routers suspected of failing for a given
- reason. The main difference between this and the normal
- ReasonRouterList is the sort order and the verification."""
- def __init__(self, reason): ReasonRouterList.__init__(self,reason)
-
- def sort_list(self):
- rlist = self.rlist.keys()
- rlist.sort(lambda x, y: cmp(y.reason_suspected[self.reason],
- x.reason_suspected[self.reason]))
- return rlist
-
- def _verify_suspected(self):
- return reduce(lambda x, y: x + y.reason_suspected[self.reason],
- self.rlist.iterkeys(), 0)
-
-class FailedRouterList(ReasonRouterList):
- """Helper class to track all routers that failed for a given
- reason. The main difference between this and the normal
- ReasonRouterList is the sort order and the verification."""
- def __init__(self, reason): ReasonRouterList.__init__(self,reason)
-
- def sort_list(self):
- rlist = self.rlist.keys()
- rlist.sort(lambda x, y: cmp(y.reason_failed[self.reason],
- x.reason_failed[self.reason]))
- return rlist
-
- def _verify_failed(self):
- return reduce(lambda x, y: x + y.reason_failed[self.reason],
- self.rlist.iterkeys(), 0)
-class BandwidthStats:
- "Class that manages observed bandwidth through a Router"
- def __init__(self):
- self.byte_list = []
- self.duration_list = []
- self.min_bw = 1e10
- self.max_bw = 0
- self.mean = 0
- self.dev = 0
-
- def _exp(self): # Weighted avg
- "Expectation - weighted average of the bandwidth through this node"
- tot_bw = reduce(lambda x, y: x+y, self.byte_list, 0.0)
- EX = 0.0
- for i in xrange(len(self.byte_list)):
- EX += (self.byte_list[i]*self.byte_list[i])/self.duration_list[i]
- if tot_bw == 0.0: return 0.0
- EX /= tot_bw
- return EX
-
- def _exp2(self): # E[X^2]
- "Second moment of the bandwidth"
- tot_bw = reduce(lambda x, y: x+y, self.byte_list, 0.0)
- EX = 0.0
- for i in xrange(len(self.byte_list)):
- EX += (self.byte_list[i]**3)/(self.duration_list[i]**2)
- if tot_bw == 0.0: return 0.0
- EX /= tot_bw
- return EX
-
- def _dev(self): # Weighted dev
- "Standard deviation of bandwidth"
- EX = self.mean
- EX2 = self._exp2()
- arg = EX2 - (EX*EX)
- if arg < -0.05:
- plog("WARN", "Diff of "+str(EX2)+" and "+str(EX)+"^2 is "+str(arg))
- return math.sqrt(abs(arg))
-
- def add_bw(self, bytes, duration):
- "Add an observed transfer of 'bytes' for 'duration' seconds"
- if not bytes: plog("WARN", "No bytes for bandwidth")
- bytes /= 1024.
- self.byte_list.append(bytes)
- self.duration_list.append(duration)
- bw = bytes/duration
- plog("DEBUG", "Got bandwidth "+str(bw))
- if self.min_bw > bw: self.min_bw = bw
- if self.max_bw < bw: self.max_bw = bw
- self.mean = self._exp()
- self.dev = self._dev()
-
-
-
-class StatsRouter(TorCtl.Router):
- "Extended Router to handle statistics markup"
- def __init__(self, router): # Promotion constructor :)
- """'Promotion Constructor' that converts a Router directly into a
- StatsRouter without a copy."""
- self.__dict__ = router.__dict__
- self.reset()
-
- def reset(self):
- "Reset all stats on this Router"
- self.circ_uncounted = 0
- self.circ_failed = 0
- self.circ_succeeded = 0 # disjoint from failed
- self.circ_suspected = 0
- self.circ_chosen = 0 # above 4 should add to this
- self.strm_failed = 0 # Only exits should have these
- self.strm_succeeded = 0
- self.strm_suspected = 0 # disjoint from failed
- self.strm_uncounted = 0
- self.strm_chosen = 0 # above 4 should add to this
- self.reason_suspected = {}
- self.reason_failed = {}
- self.first_seen = time.time()
- if "Running" in self.flags:
- self.became_active_at = self.first_seen
- self.hibernated_at = 0
- else:
- self.became_active_at = 0
- self.hibernated_at = self.first_seen
- self.total_hibernation_time = 0
- self.total_active_uptime = 0
- self.total_extend_time = 0
- self.total_extended = 0
- self.bwstats = BandwidthStats()
- self.z_ratio = 0
- self.prob_zr = 0
- self.z_bw = 0
- self.prob_zb = 0
-
- def avg_extend_time(self):
- """Return the average amount of time it took for this router
- to extend a circuit one hop"""
- if self.total_extended:
- return self.total_extend_time/self.total_extended
- else: return 0
-
- def bw_ratio(self):
- """Return the ratio of the Router's advertised bandwidth to its
- observed average stream bandwidth"""
- bw = self.bwstats.mean
- if bw == 0.0: return 0
- else: return self.bw/(1024.*bw)
-
- def current_uptime(self):
- if self.became_active_at:
- ret = (self.total_active_uptime+(time.time()-self.became_active_at))
- else:
- ret = self.total_active_uptime
- if ret == 0: return 0.000005 # eh..
- else: return ret
-
- def failed_per_hour(self):
- """Return the number of circuit extend failures per hour for this
- Router"""
- return (3600.*(self.circ_failed+self.strm_failed))/self.current_uptime()
-
- # XXX: Seperate suspected from failed in totals
- def suspected_per_hour(self):
- """Return the number of circuits that failed with this router as an
- earlier hop"""
- return (3600.*(self.circ_suspected+self.strm_suspected
- +self.circ_failed+self.strm_failed))/self.current_uptime()
-
- # These four are for sanity checking
- def _suspected_per_hour(self):
- return (3600.*(self.circ_suspected+self.strm_suspected))/self.current_uptime()
-
- def _uncounted_per_hour(self):
- return (3600.*(self.circ_uncounted+self.strm_uncounted))/self.current_uptime()
-
- def _chosen_per_hour(self):
- return (3600.*(self.circ_chosen+self.strm_chosen))/self.current_uptime()
-
- def _succeeded_per_hour(self):
- return (3600.*(self.circ_succeeded+self.strm_succeeded))/self.current_uptime()
-
- key = """Metatroller Statistics:
- CC=Circuits Chosen CF=Circuits Failed CS=Circuit Suspected
- SC=Streams Chosen SF=Streams Failed SS=Streams Suspected
- FH=Failed per Hour SH=Suspected per Hour ET=avg circuit Extend Time (s)
- EB=mean BW (K) BD=BW std Dev (K) BR=Ratio of observed to avg BW
- ZB=BW z-test value PB=Probability(z-bw) ZR=Ratio z-test value
- PR=Prob(z-ratio) U=Uptime (h)\n"""
-
- def __str__(self):
- return (self.idhex+" ("+self.nickname+")\n"
- +" CC="+str(self.circ_chosen)
- +" CF="+str(self.circ_failed)
- +" CS="+str(self.circ_suspected+self.circ_failed)
- +" SC="+str(self.strm_chosen)
- +" SF="+str(self.strm_failed)
- +" SS="+str(self.strm_suspected+self.strm_failed)
- +" FH="+str(round(self.failed_per_hour(),1))
- +" SH="+str(round(self.suspected_per_hour(),1))+"\n"
- +" ET="+str(round(self.avg_extend_time(),1))
- +" EB="+str(round(self.bwstats.mean,1))
- +" BD="+str(round(self.bwstats.dev,1))
- +" ZB="+str(round(self.z_bw,1))
- +" PB="+(str(round(self.prob_zb,3))[1:])
- +" BR="+str(round(self.bw_ratio(),1))
- +" ZR="+str(round(self.z_ratio,1))
- +" PR="+(str(round(self.prob_zr,3))[1:])
- +" U="+str(round(self.current_uptime()/3600, 1))+"\n")
-
- def sanity_check(self):
- "Makes sure all stats are self-consistent"
- if (self.circ_failed + self.circ_succeeded + self.circ_suspected
- + self.circ_uncounted != self.circ_chosen):
- plog("ERROR", self.nickname+" does not add up for circs")
- if (self.strm_failed + self.strm_succeeded + self.strm_suspected
- + self.strm_uncounted != self.strm_chosen):
- plog("ERROR", self.nickname+" does not add up for streams")
- def check_reasons(reasons, expected, which, rtype):
- count = 0
- for rs in reasons.iterkeys():
- if re.search(r"^"+which, rs): count += reasons[rs]
- if count != expected:
- plog("ERROR", "Mismatch "+which+" "+rtype+" for "+self.nickname)
- check_reasons(self.reason_suspected,self.strm_suspected,"STREAM","susp")
- check_reasons(self.reason_suspected,self.circ_suspected,"CIRC","susp")
- check_reasons(self.reason_failed,self.strm_failed,"STREAM","failed")
- check_reasons(self.reason_failed,self.circ_failed,"CIRC","failed")
- now = time.time()
- tot_hib_time = self.total_hibernation_time
- tot_uptime = self.total_active_uptime
- if self.hibernated_at: tot_hib_time += now - self.hibernated_at
- if self.became_active_at: tot_uptime += now - self.became_active_at
- if round(tot_hib_time+tot_uptime) != round(now-self.first_seen):
- plog("ERROR", "Mismatch of uptimes for "+self.nickname)
-
- per_hour_tot = round(self._uncounted_per_hour()+self.failed_per_hour()+
- self._suspected_per_hour()+self._succeeded_per_hour(), 2)
- chosen_tot = round(self._chosen_per_hour(), 2)
- if per_hour_tot != chosen_tot:
- plog("ERROR", self.nickname+" has mismatch of per hour counts: "
- +str(per_hour_tot) +" vs "+str(chosen_tot))
-
-
-
-class StatsHandler(PathSupport.PathBuilder):
- """An extension of PathSupport.PathBuilder that keeps track of
- router statistics for every circuit and stream"""
- def __init__(self, c, slmgr):
- PathBuilder.__init__(self, c, slmgr, StatsRouter)
- self.circ_count = 0
- self.strm_count = 0
- self.strm_failed = 0
- self.circ_failed = 0
- self.failed_reasons = {}
- self.suspect_reasons = {}
-
- def run_zbtest(self): # Unweighted z-test
- """Run unweighted z-test to calculate the probabilities of a node
- having a given stream bandwidth based on the Normal distribution"""
- n = reduce(lambda x, y: x+(y.bwstats.mean > 0), self.sorted_r, 0)
- if n == 0: return (0, 0)
- avg = reduce(lambda x, y: x+y.bwstats.mean, self.sorted_r, 0)/float(n)
- def notlambda(x, y):
- if y.bwstats.mean <= 0: return x+0
- else: return x+(y.bwstats.mean-avg)*(y.bwstats.mean-avg)
- stddev = math.sqrt(reduce(notlambda, self.sorted_r, 0)/float(n))
- if not stddev: return (avg, stddev)
- for r in self.sorted_r:
- if r.bwstats.mean > 0:
- r.z_bw = abs((r.bwstats.mean-avg)/stddev)
- r.prob_zb = TorUtil.zprob(-r.z_bw)
- return (avg, stddev)
-
- def run_zrtest(self): # Unweighted z-test
- """Run unweighted z-test to calculate the probabilities of a node
- having a given ratio of stream bandwidth to advertised bandwidth
- based on the Normal distribution"""
- n = reduce(lambda x, y: x+(y.bw_ratio() > 0), self.sorted_r, 0)
- if n == 0: return (0, 0)
- avg = reduce(lambda x, y: x+y.bw_ratio(), self.sorted_r, 0)/float(n)
- def notlambda(x, y):
- if y.bw_ratio() <= 0: return x+0
- else: return x+(y.bw_ratio()-avg)*(y.bw_ratio()-avg)
- stddev = math.sqrt(reduce(notlambda, self.sorted_r, 0)/float(n))
- if not stddev: return (avg, stddev)
- for r in self.sorted_r:
- if r.bw_ratio() > 0:
- r.z_ratio = abs((r.bw_ratio()-avg)/stddev)
- r.prob_zr = TorUtil.zprob(-r.z_ratio)
- return (avg, stddev)
-
- def write_reasons(self, f, reasons, name):
- "Write out all the failure reasons and statistics for all Routers"
- f.write("\n\n\t----------------- "+name+" -----------------\n")
- for rsn in reasons:
- f.write("\n"+rsn.reason+". Failed: "+str(rsn.total_failed())
- +", Suspected: "+str(rsn.total_suspected())+"\n")
- rsn.write_list(f)
-
- def write_routers(self, f, rlist, name):
- "Write out all the usage statistics for all Routers"
- f.write("\n\n\t----------------- "+name+" -----------------\n\n")
- for r in rlist:
- # only print it if we've used it.
- if r.circ_chosen+r.strm_chosen > 0: f.write(str(r))
-
- def write_stats(self, filename):
- "Write out all the statistics the StatsHandler has gathered"
- # TODO: all this shit should be configurable. Some of it only makes
- # sense when scanning in certain modes.
- plog("DEBUG", "Writing stats")
- # Sanity check routers
- for r in self.sorted_r: r.sanity_check()
-
- # Sanity check the router reason lists.
- for r in self.sorted_r:
- for rsn in r.reason_failed:
- if r not in self.failed_reasons[rsn].rlist:
- plog("ERROR", "Router missing from reason table")
- for rsn in r.reason_suspected:
- if r not in self.suspect_reasons[rsn].rlist:
- plog("ERROR", "Router missing from reason table")
-
- # Sanity check the lists the other way
- for rsn in self.failed_reasons.itervalues(): rsn._verify_failed()
- for rsn in self.suspect_reasons.itervalues(): rsn._verify_suspected()
-
- f = file(filename, "w")
- f.write(StatsRouter.key)
- (avg, dev) = self.run_zbtest()
- f.write("\n\nBW stats: u="+str(round(avg,1))+" s="+str(round(dev,1))+"\n")
-
- (avg, dev) = self.run_zrtest()
- f.write("BW ratio stats: u="+str(round(avg,1))+" s="+str(round(dev,1))+"\n")
-
- # Circ, strm infoz
- f.write("Circ failure ratio: "+str(self.circ_failed)
- +"/"+str(self.circ_count)+"\n")
-
- f.write("Stream failure ratio: "+str(self.strm_failed)
- +"/"+str(self.strm_count)+"\n")
-
- # Extend times
- n = 0.01+reduce(lambda x, y: x+(y.avg_extend_time() > 0), self.sorted_r, 0)
- avg_extend = reduce(lambda x, y: x+y.avg_extend_time(), self.sorted_r, 0)/n
- def notlambda(x, y):
- return x+(y.avg_extend_time()-avg_extend)*(y.avg_extend_time()-avg_extend)
- dev_extend = math.sqrt(reduce(notlambda, self.sorted_r, 0)/float(n))
-
- f.write("Extend time: u="+str(round(avg_extend,1))
- +" s="+str(round(dev_extend,1)))
-
- # sort+print by bandwidth
- bw_rate = copy.copy(self.sorted_r)
- bw_rate.sort(lambda x, y: cmp(y.bw_ratio(), x.bw_ratio()))
- self.write_routers(f, bw_rate, "Bandwidth Ratios")
-
- failed = copy.copy(self.sorted_r)
- failed.sort(lambda x, y:
- cmp(y.circ_failed+y.strm_failed,
- x.circ_failed+x.strm_failed))
- self.write_routers(f, failed, "Failed Counts")
-
- suspected = copy.copy(self.sorted_r)
- suspected.sort(lambda x, y: # Suspected includes failed
- cmp(y.circ_failed+y.strm_failed+y.circ_suspected+y.strm_suspected,
- x.circ_failed+x.strm_failed+x.circ_suspected+x.strm_suspected))
- self.write_routers(f, suspected, "Suspected Counts")
-
- fail_rate = copy.copy(failed)
- fail_rate.sort(lambda x, y: cmp(y.failed_per_hour(), x.failed_per_hour()))
- self.write_routers(f, fail_rate, "Fail Rates")
-
- suspect_rate = copy.copy(suspected)
- suspect_rate.sort(lambda x, y:
- cmp(y.suspected_per_hour(), x.suspected_per_hour()))
- self.write_routers(f, suspect_rate, "Suspect Rates")
-
- # TODO: Sort by failed/selected and suspect/selected ratios
- # if we ever want to do non-uniform scanning..
-
- # FIXME: Add failed in here somehow..
- susp_reasons = self.suspect_reasons.values()
- susp_reasons.sort(lambda x, y:
- cmp(y.total_suspected(), x.total_suspected()))
- self.write_reasons(f, susp_reasons, "Suspect Reasons")
-
- fail_reasons = self.failed_reasons.values()
- fail_reasons.sort(lambda x, y:
- cmp(y.total_failed(), x.total_failed()))
- self.write_reasons(f, fail_reasons, "Failed Reasons")
- f.close()
-
- # FIXME: sort+print by circ extend time
-
- def reset_stats(self):
- plog("DEBUG", "Resetting stats")
- self.circ_count = 0
- self.strm_count = 0
- self.strm_failed = 0
- self.circ_failed = 0
- self.suspect_reasons.clear()
- self.failed_reasons.clear()
- for r in self.sorted_r: r.reset()
-
- def circ_status_event(self, c):
- if c.circ_id in self.circuits:
- # TODO: Hrmm, consider making this sane in TorCtl.
- if c.reason: lreason = c.reason
- else: lreason = "NONE"
- if c.remote_reason: rreason = c.remote_reason
- else: rreason = "NONE"
- reason = c.event_name+":"+c.status+":"+lreason+":"+rreason
- if c.status == "LAUNCHED":
- # Update circ_chosen count
- self.circ_count += 1
- elif c.status == "EXTENDED":
- delta = c.arrived_at - self.circuits[c.circ_id].last_extended_at
- r_ext = c.path[-1]
- if r_ext[0] != '$': r_ext = self.name_to_key[r_ext]
- self.routers[r_ext[1:]].total_extend_time += delta
- self.routers[r_ext[1:]].total_extended += 1
- elif c.status == "FAILED":
- for r in self.circuits[c.circ_id].path: r.circ_chosen += 1
-
- if len(c.path)-1 < 0: start_f = 0
- else: start_f = len(c.path)-1
-
- # Count failed
- self.circ_failed += 1
- # XXX: Differentiate between extender and extendee
- for r in self.circuits[c.circ_id].path[start_f:len(c.path)+1]:
- r.circ_failed += 1
- if not reason in r.reason_failed:
- r.reason_failed[reason] = 1
- else: r.reason_failed[reason]+=1
- if reason not in self.failed_reasons:
- self.failed_reasons[reason] = FailedRouterList(reason)
- self.failed_reasons[reason].add_r(r)
-
- for r in self.circuits[c.circ_id].path[len(c.path)+1:]:
- r.circ_uncounted += 1
-
- # Don't count if failed was set this round, don't set
- # suspected..
- for r in self.circuits[c.circ_id].path[:start_f]:
- r.circ_suspected += 1
- if not reason in r.reason_suspected:
- r.reason_suspected[reason] = 1
- else: r.reason_suspected[reason]+=1
- if reason not in self.suspect_reasons:
- self.suspect_reasons[reason] = SuspectRouterList(reason)
- self.suspect_reasons[reason].add_r(r)
- elif c.status == "CLOSED":
- # Since PathBuilder deletes the circuit on a failed,
- # we only get this for a clean close
- for r in self.circuits[c.circ_id].path:
- r.circ_chosen += 1
- if lreason in ("REQUESTED", "FINISHED", "ORIGIN"):
- r.circ_succeeded += 1
- else:
- if not reason in r.reason_suspected:
- r.reason_suspected[reason] = 1
- else: r.reason_suspected[reason] += 1
- r.circ_suspected+= 1
- if reason not in self.suspect_reasons:
- self.suspect_reasons[reason] = SuspectRouterList(reason)
- self.suspect_reasons[reason].add_r(r)
- PathBuilder.circ_status_event(self, c)
-
- def count_stream_reason_failed(self, s, reason):
- "Count the routers involved in a failure"
- # Update failed count,reason_failed for exit
- r = self.circuits[s.circ_id].exit
- if not reason in r.reason_failed: r.reason_failed[reason] = 1
- else: r.reason_failed[reason]+=1
- r.strm_failed += 1
- if reason not in self.failed_reasons:
- self.failed_reasons[reason] = FailedRouterList(reason)
- self.failed_reasons[reason].add_r(r)
-
- def count_stream_suspects(self, s, lreason, reason):
- "Count the routers 'suspected' of being involved in a failure"
- if lreason in ("TIMEOUT", "INTERNAL", "TORPROTOCOL" "DESTROY"):
- for r in self.circuits[s.circ_id].path[:-1]:
- r.strm_suspected += 1
- if not reason in r.reason_suspected:
- r.reason_suspected[reason] = 1
- else: r.reason_suspected[reason]+=1
- if reason not in self.suspect_reasons:
- self.suspect_reasons[reason] = SuspectRouterList(reason)
- self.suspect_reasons[reason].add_r(r)
- else:
- for r in self.circuits[s.circ_id].path[:-1]:
- r.strm_uncounted += 1
-
- def stream_status_event(self, s):
- if s.strm_id in self.streams and not self.streams[s.strm_id].ignored:
- # TODO: Hrmm, consider making this sane in TorCtl.
- if s.reason: lreason = s.reason
- else: lreason = "NONE"
- if s.remote_reason: rreason = s.remote_reason
- else: rreason = "NONE"
- reason = s.event_name+":"+s.status+":"+lreason+":"+rreason+":"+self.streams[s.strm_id].kind
- if (s.status in ("DETACHED", "FAILED", "CLOSED", "SUCCEEDED")
- and not s.circ_id):
- # XXX: REMAPs can do this (normal). Also REASON=DESTROY (bug?)
- # Also timeouts.. Those should use the pending circ instead
- # of returning..
- plog("WARN", "Stream "+str(s.strm_id)+" detached from no circuit!")
- PathBuilder.stream_status_event(self, s)
- return
-
- # Verify circ id matches stream.circ
- if s.status not in ("NEW", "NEWRESOLVE", "REMAP"):
- circ = self.streams[s.strm_id].circ
- if not circ: circ = self.streams[s.strm_id].pending_circ
- if circ and circ.circ_id != s.circ_id:
- plog("WARN", str(s.strm_id) + " has mismatch of "
- +str(s.circ_id)+" v "+str(circ.circ_id))
- if s.circ_id and s.circ_id not in self.circuits:
- plog("NOTICE", "Unknown circuit "+str(s.circ_id)
- +" for stream "+str(s.strm_id))
- return
-
- if s.status == "DETACHED":
- if self.streams[s.strm_id].attached_at:
- plog("WARN", str(s.strm_id)+" detached after succeeded")
- # Update strm_chosen count
- self.strm_count += 1
- for r in self.circuits[s.circ_id].path: r.strm_chosen += 1
- self.strm_failed += 1
- self.count_stream_suspects(s, lreason, reason)
- self.count_stream_reason_failed(s, reason)
- elif s.status == "FAILED":
- # HACK. We get both failed and closed for the same stream,
- # with different reasons. Might as well record both, since they
- # often differ.
- self.streams[s.strm_id].failed_reason = reason
- elif s.status == "CLOSED":
- # Always get both a closed and a failed..
- # - Check if the circuit exists still
- # Update strm_chosen count
- self.strm_count += 1
- for r in self.circuits[s.circ_id].path: r.strm_chosen += 1
-
- # Update bw stats. XXX: Don't do this for resolve streams
- if self.streams[s.strm_id].attached_at:
- lifespan = self.streams[s.strm_id].lifespan(s.arrived_at)
- for r in self.streams[s.strm_id].circ.path:
- r.bwstats.add_bw(self.streams[s.strm_id].bytes_written+
- self.streams[s.strm_id].bytes_read, lifespan)
-
- if self.streams[s.strm_id].failed:
- reason = self.streams[s.strm_id].failed_reason+":"+lreason+":"+rreason
-
- self.count_stream_suspects(s, lreason, reason)
-
- r = self.circuits[s.circ_id].exit
- if (not self.streams[s.strm_id].failed
- and (lreason == "DONE" or (lreason == "END" and rreason == "DONE"))):
- r.strm_succeeded += 1
- else:
- self.strm_failed += 1
- self.count_stream_reason_failed(s, reason)
- PathBuilder.stream_status_event(self, s)
-
- def ns_event(self, n):
- PathBuilder.ns_event(self, n)
- now = n.arrived_at
- for ns in n.nslist:
- if not ns.idhex in self.routers:
- continue
- r = self.routers[ns.idhex]
- if "Running" in ns.flags:
- if not r.became_active_at:
- r.became_active_at = now
- r.total_hibernation_time += now - r.hibernated_at
- r.hibernated_at = 0
- else:
- if not r.hibernated_at:
- r.hibernated_at = now
- r.total_active_uptime += now - r.became_active_at
- r.became_active_at = 0
Copied: torflow/branches/gsoc2008/TorCtl.local/StatsSupport.py (from rev 15741, torflow/branches/gsoc2008/TorCtl/StatsSupport.py)
===================================================================
--- torflow/branches/gsoc2008/TorCtl.local/StatsSupport.py (rev 0)
+++ torflow/branches/gsoc2008/TorCtl.local/StatsSupport.py 2008-07-07 19:37:47 UTC (rev 15742)
@@ -0,0 +1,676 @@
+#!/usr/bin/python
+#StatsSupport.py - functions and classes useful for calculating stream/circuit statistics
+
+"""
+
+Support classes for statisics gathering
+
+The StatsSupport package contains several classes that extend PathSupport to
+gather continuous statistics on the Tor network.
+
+The main entrypoint is to extend or create an instance of the StatsHandler
+class. The StatsHandler extends from TorCtl.PathSupport.PathBuilder, which is
+itself a TorCtl.EventHandler. The StatsHandler listens to CIRC and STREAM
+events and gathers all manner of statics on their creation and failure before
+passing the events back up to the PathBuilder code, which manages the actual
+construction and the attachment of streams to circuits.
+
+The package also contains a number of support classes that help gather
+additional statistics on the reliability and performance of routers.
+
+For the purpose of accounting failures, the code tracks two main classes of
+failure: 'actual' failure and 'suspected' failure. The general rule is that an
+actual failure is attributed to the node that directly handled the circuit or
+stream. For streams, this is considered to be the exit node. For circuits, it
+is both the extender and the extendee. 'Suspected' failures, on the other
+hand, are attributed to every member of the circuit up until the extendee for
+circuits, and all hops for streams.
+
+For bandwidth accounting, the average stream bandwidth and the average ratio
+of stream bandwidth to advertised bandwidth are tracked, and when the
+statistics are written, a Z-test is performed to calculate the probabilities
+of these values assuming a normal distribution. Note, however, that it has not
+been verified that this distribution is actually normal. It is likely to be
+something else (pareto, perhaps?).
+
+"""
+
+import sys
+import re
+import random
+import copy
+import time
+import math
+
+import TorUtil, PathSupport, TorCtl
+from TorUtil import *
+from PathSupport import *
+from TorUtil import meta_port, meta_host, control_port, control_host
+
+class ReasonRouterList:
+ "Helper class to track which Routers have failed for a given reason"
+ def __init__(self, reason):
+ self.reason = reason
+ self.rlist = {}
+
+ def sort_list(self): raise NotImplemented()
+
+ def write_list(self, f):
+ "Write the list of failure counts for this reason 'f'"
+ rlist = self.sort_list()
+ for r in rlist:
+ susp = 0
+ tot_failed = r.circ_failed+r.strm_failed
+ tot_susp = tot_failed+r.circ_suspected+r.strm_suspected
+ f.write(r.idhex+" ("+r.nickname+") F=")
+ if self.reason in r.reason_failed:
+ susp = r.reason_failed[self.reason]
+ f.write(str(susp)+"/"+str(tot_failed))
+ f.write(" S=")
+ if self.reason in r.reason_suspected:
+ susp += r.reason_suspected[self.reason]
+ f.write(str(susp)+"/"+str(tot_susp)+"\n")
+
+ def add_r(self, r):
+ "Add a router to the list for this reason"
+ self.rlist[r] = 1
+
+ def total_suspected(self):
+ "Get a list of total suspected failures for this reason"
+ # suspected is disjoint from failed. The failed table
+ # may not have an entry
+ def notlambda(x, y):
+ if self.reason in y.reason_suspected:
+ if self.reason in y.reason_failed:
+ return (x + y.reason_suspected[self.reason]
+ + y.reason_failed[self.reason])
+ else:
+ return (x + y.reason_suspected[self.reason])
+ else:
+ if self.reason in y.reason_failed:
+ return (x + y.reason_failed[self.reason])
+ else: return x
+ return reduce(notlambda, self.rlist.iterkeys(), 0)
+
+ def total_failed(self):
+ "Get a list of total failures for this reason"
+ def notlambda(x, y):
+ if self.reason in y.reason_failed:
+ return (x + y.reason_failed[self.reason])
+ else: return x
+ return reduce(notlambda, self.rlist.iterkeys(), 0)
+
+class SuspectRouterList(ReasonRouterList):
+ """Helper class to track all routers suspected of failing for a given
+ reason. The main difference between this and the normal
+ ReasonRouterList is the sort order and the verification."""
+ def __init__(self, reason): ReasonRouterList.__init__(self,reason)
+
+ def sort_list(self):
+ rlist = self.rlist.keys()
+ rlist.sort(lambda x, y: cmp(y.reason_suspected[self.reason],
+ x.reason_suspected[self.reason]))
+ return rlist
+
+ def _verify_suspected(self):
+ return reduce(lambda x, y: x + y.reason_suspected[self.reason],
+ self.rlist.iterkeys(), 0)
+
+class FailedRouterList(ReasonRouterList):
+ """Helper class to track all routers that failed for a given
+ reason. The main difference between this and the normal
+ ReasonRouterList is the sort order and the verification."""
+ def __init__(self, reason): ReasonRouterList.__init__(self,reason)
+
+ def sort_list(self):
+ rlist = self.rlist.keys()
+ rlist.sort(lambda x, y: cmp(y.reason_failed[self.reason],
+ x.reason_failed[self.reason]))
+ return rlist
+
+ def _verify_failed(self):
+ return reduce(lambda x, y: x + y.reason_failed[self.reason],
+ self.rlist.iterkeys(), 0)
+class BandwidthStats:
+ "Class that manages observed bandwidth through a Router"
+ def __init__(self):
+ self.byte_list = []
+ self.duration_list = []
+ self.min_bw = 1e10
+ self.max_bw = 0
+ self.mean = 0
+ self.dev = 0
+
+ def _exp(self): # Weighted avg
+ "Expectation - weighted average of the bandwidth through this node"
+ tot_bw = reduce(lambda x, y: x+y, self.byte_list, 0.0)
+ EX = 0.0
+ for i in xrange(len(self.byte_list)):
+ EX += (self.byte_list[i]*self.byte_list[i])/self.duration_list[i]
+ if tot_bw == 0.0: return 0.0
+ EX /= tot_bw
+ return EX
+
+ def _exp2(self): # E[X^2]
+ "Second moment of the bandwidth"
+ tot_bw = reduce(lambda x, y: x+y, self.byte_list, 0.0)
+ EX = 0.0
+ for i in xrange(len(self.byte_list)):
+ EX += (self.byte_list[i]**3)/(self.duration_list[i]**2)
+ if tot_bw == 0.0: return 0.0
+ EX /= tot_bw
+ return EX
+
+ def _dev(self): # Weighted dev
+ "Standard deviation of bandwidth"
+ EX = self.mean
+ EX2 = self._exp2()
+ arg = EX2 - (EX*EX)
+ if arg < -0.05:
+ plog("WARN", "Diff of "+str(EX2)+" and "+str(EX)+"^2 is "+str(arg))
+ return math.sqrt(abs(arg))
+
+ def add_bw(self, bytes, duration):
+ "Add an observed transfer of 'bytes' for 'duration' seconds"
+ if not bytes: plog("WARN", "No bytes for bandwidth")
+ bytes /= 1024.
+ self.byte_list.append(bytes)
+ self.duration_list.append(duration)
+ bw = bytes/duration
+ plog("DEBUG", "Got bandwidth "+str(bw))
+ if self.min_bw > bw: self.min_bw = bw
+ if self.max_bw < bw: self.max_bw = bw
+ self.mean = self._exp()
+ self.dev = self._dev()
+
+
+
+class StatsRouter(TorCtl.Router):
+ "Extended Router to handle statistics markup"
+ def __init__(self, router): # Promotion constructor :)
+ """'Promotion Constructor' that converts a Router directly into a
+ StatsRouter without a copy."""
+ self.__dict__ = router.__dict__
+ self.reset()
+
+ def reset(self):
+ "Reset all stats on this Router"
+ self.circ_uncounted = 0
+ self.circ_failed = 0
+ self.circ_succeeded = 0 # disjoint from failed
+ self.circ_suspected = 0
+ self.circ_chosen = 0 # above 4 should add to this
+ self.strm_failed = 0 # Only exits should have these
+ self.strm_succeeded = 0
+ self.strm_suspected = 0 # disjoint from failed
+ self.strm_uncounted = 0
+ self.strm_chosen = 0 # above 4 should add to this
+ self.reason_suspected = {}
+ self.reason_failed = {}
+ self.first_seen = time.time()
+ if "Running" in self.flags:
+ self.became_active_at = self.first_seen
+ self.hibernated_at = 0
+ else:
+ self.became_active_at = 0
+ self.hibernated_at = self.first_seen
+ self.total_hibernation_time = 0
+ self.total_active_uptime = 0
+ self.total_extend_time = 0
+ self.total_extended = 0
+ self.bwstats = BandwidthStats()
+ self.z_ratio = 0
+ self.prob_zr = 0
+ self.z_bw = 0
+ self.prob_zb = 0
+
+ def avg_extend_time(self):
+ """Return the average amount of time it took for this router
+ to extend a circuit one hop"""
+ if self.total_extended:
+ return self.total_extend_time/self.total_extended
+ else: return 0
+
+ def bw_ratio(self):
+ """Return the ratio of the Router's advertised bandwidth to its
+ observed average stream bandwidth"""
+ bw = self.bwstats.mean
+ if bw == 0.0: return 0
+ else: return self.bw/(1024.*bw)
+
+ def current_uptime(self):
+ if self.became_active_at:
+ ret = (self.total_active_uptime+(time.time()-self.became_active_at))
+ else:
+ ret = self.total_active_uptime
+ if ret == 0: return 0.000005 # eh..
+ else: return ret
+
+ def failed_per_hour(self):
+ """Return the number of circuit extend failures per hour for this
+ Router"""
+ return (3600.*(self.circ_failed+self.strm_failed))/self.current_uptime()
+
+ # XXX: Seperate suspected from failed in totals
+ def suspected_per_hour(self):
+ """Return the number of circuits that failed with this router as an
+ earlier hop"""
+ return (3600.*(self.circ_suspected+self.strm_suspected
+ +self.circ_failed+self.strm_failed))/self.current_uptime()
+
+ # These four are for sanity checking
+ def _suspected_per_hour(self):
+ return (3600.*(self.circ_suspected+self.strm_suspected))/self.current_uptime()
+
+ def _uncounted_per_hour(self):
+ return (3600.*(self.circ_uncounted+self.strm_uncounted))/self.current_uptime()
+
+ def _chosen_per_hour(self):
+ return (3600.*(self.circ_chosen+self.strm_chosen))/self.current_uptime()
+
+ def _succeeded_per_hour(self):
+ return (3600.*(self.circ_succeeded+self.strm_succeeded))/self.current_uptime()
+
+ key = """Metatroller Statistics:
+ CC=Circuits Chosen CF=Circuits Failed CS=Circuit Suspected
+ SC=Streams Chosen SF=Streams Failed SS=Streams Suspected
+ FH=Failed per Hour SH=Suspected per Hour ET=avg circuit Extend Time (s)
+ EB=mean BW (K) BD=BW std Dev (K) BR=Ratio of observed to avg BW
+ ZB=BW z-test value PB=Probability(z-bw) ZR=Ratio z-test value
+ PR=Prob(z-ratio) U=Uptime (h)\n"""
+
+ def __str__(self):
+ return (self.idhex+" ("+self.nickname+")\n"
+ +" CC="+str(self.circ_chosen)
+ +" CF="+str(self.circ_failed)
+ +" CS="+str(self.circ_suspected+self.circ_failed)
+ +" SC="+str(self.strm_chosen)
+ +" SF="+str(self.strm_failed)
+ +" SS="+str(self.strm_suspected+self.strm_failed)
+ +" FH="+str(round(self.failed_per_hour(),1))
+ +" SH="+str(round(self.suspected_per_hour(),1))+"\n"
+ +" ET="+str(round(self.avg_extend_time(),1))
+ +" EB="+str(round(self.bwstats.mean,1))
+ +" BD="+str(round(self.bwstats.dev,1))
+ +" ZB="+str(round(self.z_bw,1))
+ +" PB="+(str(round(self.prob_zb,3))[1:])
+ +" BR="+str(round(self.bw_ratio(),1))
+ +" ZR="+str(round(self.z_ratio,1))
+ +" PR="+(str(round(self.prob_zr,3))[1:])
+ +" U="+str(round(self.current_uptime()/3600, 1))+"\n")
+
+ def sanity_check(self):
+ "Makes sure all stats are self-consistent"
+ if (self.circ_failed + self.circ_succeeded + self.circ_suspected
+ + self.circ_uncounted != self.circ_chosen):
+ plog("ERROR", self.nickname+" does not add up for circs")
+ if (self.strm_failed + self.strm_succeeded + self.strm_suspected
+ + self.strm_uncounted != self.strm_chosen):
+ plog("ERROR", self.nickname+" does not add up for streams")
+ def check_reasons(reasons, expected, which, rtype):
+ count = 0
+ for rs in reasons.iterkeys():
+ if re.search(r"^"+which, rs): count += reasons[rs]
+ if count != expected:
+ plog("ERROR", "Mismatch "+which+" "+rtype+" for "+self.nickname)
+ check_reasons(self.reason_suspected,self.strm_suspected,"STREAM","susp")
+ check_reasons(self.reason_suspected,self.circ_suspected,"CIRC","susp")
+ check_reasons(self.reason_failed,self.strm_failed,"STREAM","failed")
+ check_reasons(self.reason_failed,self.circ_failed,"CIRC","failed")
+ now = time.time()
+ tot_hib_time = self.total_hibernation_time
+ tot_uptime = self.total_active_uptime
+ if self.hibernated_at: tot_hib_time += now - self.hibernated_at
+ if self.became_active_at: tot_uptime += now - self.became_active_at
+ if round(tot_hib_time+tot_uptime) != round(now-self.first_seen):
+ plog("ERROR", "Mismatch of uptimes for "+self.nickname)
+
+ per_hour_tot = round(self._uncounted_per_hour()+self.failed_per_hour()+
+ self._suspected_per_hour()+self._succeeded_per_hour(), 2)
+ chosen_tot = round(self._chosen_per_hour(), 2)
+ if per_hour_tot != chosen_tot:
+ plog("ERROR", self.nickname+" has mismatch of per hour counts: "
+ +str(per_hour_tot) +" vs "+str(chosen_tot))
+
+
+
+class StatsHandler(PathSupport.PathBuilder):
+ """An extension of PathSupport.PathBuilder that keeps track of
+ router statistics for every circuit and stream"""
+ def __init__(self, c, slmgr):
+ PathBuilder.__init__(self, c, slmgr, StatsRouter)
+ self.circ_count = 0
+ self.strm_count = 0
+ self.strm_failed = 0
+ self.circ_failed = 0
+ self.failed_reasons = {}
+ self.suspect_reasons = {}
+
+ def run_zbtest(self): # Unweighted z-test
+ """Run unweighted z-test to calculate the probabilities of a node
+ having a given stream bandwidth based on the Normal distribution"""
+ n = reduce(lambda x, y: x+(y.bwstats.mean > 0), self.sorted_r, 0)
+ if n == 0: return (0, 0)
+ avg = reduce(lambda x, y: x+y.bwstats.mean, self.sorted_r, 0)/float(n)
+ def notlambda(x, y):
+ if y.bwstats.mean <= 0: return x+0
+ else: return x+(y.bwstats.mean-avg)*(y.bwstats.mean-avg)
+ stddev = math.sqrt(reduce(notlambda, self.sorted_r, 0)/float(n))
+ if not stddev: return (avg, stddev)
+ for r in self.sorted_r:
+ if r.bwstats.mean > 0:
+ r.z_bw = abs((r.bwstats.mean-avg)/stddev)
+ r.prob_zb = TorUtil.zprob(-r.z_bw)
+ return (avg, stddev)
+
+ def run_zrtest(self): # Unweighted z-test
+ """Run unweighted z-test to calculate the probabilities of a node
+ having a given ratio of stream bandwidth to advertised bandwidth
+ based on the Normal distribution"""
+ n = reduce(lambda x, y: x+(y.bw_ratio() > 0), self.sorted_r, 0)
+ if n == 0: return (0, 0)
+ avg = reduce(lambda x, y: x+y.bw_ratio(), self.sorted_r, 0)/float(n)
+ def notlambda(x, y):
+ if y.bw_ratio() <= 0: return x+0
+ else: return x+(y.bw_ratio()-avg)*(y.bw_ratio()-avg)
+ stddev = math.sqrt(reduce(notlambda, self.sorted_r, 0)/float(n))
+ if not stddev: return (avg, stddev)
+ for r in self.sorted_r:
+ if r.bw_ratio() > 0:
+ r.z_ratio = abs((r.bw_ratio()-avg)/stddev)
+ r.prob_zr = TorUtil.zprob(-r.z_ratio)
+ return (avg, stddev)
+
+ def write_reasons(self, f, reasons, name):
+ "Write out all the failure reasons and statistics for all Routers"
+ f.write("\n\n\t----------------- "+name+" -----------------\n")
+ for rsn in reasons:
+ f.write("\n"+rsn.reason+". Failed: "+str(rsn.total_failed())
+ +", Suspected: "+str(rsn.total_suspected())+"\n")
+ rsn.write_list(f)
+
+ def write_routers(self, f, rlist, name):
+ "Write out all the usage statistics for all Routers"
+ f.write("\n\n\t----------------- "+name+" -----------------\n\n")
+ for r in rlist:
+ # only print it if we've used it.
+ if r.circ_chosen+r.strm_chosen > 0: f.write(str(r))
+
+ def write_stats(self, filename):
+ "Write out all the statistics the StatsHandler has gathered"
+ # TODO: all this shit should be configurable. Some of it only makes
+ # sense when scanning in certain modes.
+ plog("DEBUG", "Writing stats")
+ # Sanity check routers
+ for r in self.sorted_r: r.sanity_check()
+
+ # Sanity check the router reason lists.
+ for r in self.sorted_r:
+ for rsn in r.reason_failed:
+ if r not in self.failed_reasons[rsn].rlist:
+ plog("ERROR", "Router missing from reason table")
+ for rsn in r.reason_suspected:
+ if r not in self.suspect_reasons[rsn].rlist:
+ plog("ERROR", "Router missing from reason table")
+
+ # Sanity check the lists the other way
+ for rsn in self.failed_reasons.itervalues(): rsn._verify_failed()
+ for rsn in self.suspect_reasons.itervalues(): rsn._verify_suspected()
+
+ f = file(filename, "w")
+ f.write(StatsRouter.key)
+ (avg, dev) = self.run_zbtest()
+ f.write("\n\nBW stats: u="+str(round(avg,1))+" s="+str(round(dev,1))+"\n")
+
+ (avg, dev) = self.run_zrtest()
+ f.write("BW ratio stats: u="+str(round(avg,1))+" s="+str(round(dev,1))+"\n")
+
+ # Circ, strm infoz
+ f.write("Circ failure ratio: "+str(self.circ_failed)
+ +"/"+str(self.circ_count)+"\n")
+
+ f.write("Stream failure ratio: "+str(self.strm_failed)
+ +"/"+str(self.strm_count)+"\n")
+
+ # Extend times
+ n = 0.01+reduce(lambda x, y: x+(y.avg_extend_time() > 0), self.sorted_r, 0)
+ avg_extend = reduce(lambda x, y: x+y.avg_extend_time(), self.sorted_r, 0)/n
+ def notlambda(x, y):
+ return x+(y.avg_extend_time()-avg_extend)*(y.avg_extend_time()-avg_extend)
+ dev_extend = math.sqrt(reduce(notlambda, self.sorted_r, 0)/float(n))
+
+ f.write("Extend time: u="+str(round(avg_extend,1))
+ +" s="+str(round(dev_extend,1)))
+
+ # sort+print by bandwidth
+ bw_rate = copy.copy(self.sorted_r)
+ bw_rate.sort(lambda x, y: cmp(y.bw_ratio(), x.bw_ratio()))
+ self.write_routers(f, bw_rate, "Bandwidth Ratios")
+
+ failed = copy.copy(self.sorted_r)
+ failed.sort(lambda x, y:
+ cmp(y.circ_failed+y.strm_failed,
+ x.circ_failed+x.strm_failed))
+ self.write_routers(f, failed, "Failed Counts")
+
+ suspected = copy.copy(self.sorted_r)
+ suspected.sort(lambda x, y: # Suspected includes failed
+ cmp(y.circ_failed+y.strm_failed+y.circ_suspected+y.strm_suspected,
+ x.circ_failed+x.strm_failed+x.circ_suspected+x.strm_suspected))
+ self.write_routers(f, suspected, "Suspected Counts")
+
+ fail_rate = copy.copy(failed)
+ fail_rate.sort(lambda x, y: cmp(y.failed_per_hour(), x.failed_per_hour()))
+ self.write_routers(f, fail_rate, "Fail Rates")
+
+ suspect_rate = copy.copy(suspected)
+ suspect_rate.sort(lambda x, y:
+ cmp(y.suspected_per_hour(), x.suspected_per_hour()))
+ self.write_routers(f, suspect_rate, "Suspect Rates")
+
+ # TODO: Sort by failed/selected and suspect/selected ratios
+ # if we ever want to do non-uniform scanning..
+
+ # FIXME: Add failed in here somehow..
+ susp_reasons = self.suspect_reasons.values()
+ susp_reasons.sort(lambda x, y:
+ cmp(y.total_suspected(), x.total_suspected()))
+ self.write_reasons(f, susp_reasons, "Suspect Reasons")
+
+ fail_reasons = self.failed_reasons.values()
+ fail_reasons.sort(lambda x, y:
+ cmp(y.total_failed(), x.total_failed()))
+ self.write_reasons(f, fail_reasons, "Failed Reasons")
+ f.close()
+
+ # FIXME: sort+print by circ extend time
+
+ def reset_stats(self):
+ plog("DEBUG", "Resetting stats")
+ self.circ_count = 0
+ self.strm_count = 0
+ self.strm_failed = 0
+ self.circ_failed = 0
+ self.suspect_reasons.clear()
+ self.failed_reasons.clear()
+ for r in self.sorted_r: r.reset()
+
+ def circ_status_event(self, c):
+ if c.circ_id in self.circuits:
+ # TODO: Hrmm, consider making this sane in TorCtl.
+ if c.reason: lreason = c.reason
+ else: lreason = "NONE"
+ if c.remote_reason: rreason = c.remote_reason
+ else: rreason = "NONE"
+ reason = c.event_name+":"+c.status+":"+lreason+":"+rreason
+ if c.status == "LAUNCHED":
+ # Update circ_chosen count
+ self.circ_count += 1
+ elif c.status == "EXTENDED":
+ delta = c.arrived_at - self.circuits[c.circ_id].last_extended_at
+ r_ext = c.path[-1]
+ if r_ext[0] != '$': r_ext = self.name_to_key[r_ext]
+ self.routers[r_ext[1:]].total_extend_time += delta
+ self.routers[r_ext[1:]].total_extended += 1
+ elif c.status == "FAILED":
+ for r in self.circuits[c.circ_id].path: r.circ_chosen += 1
+
+ if len(c.path)-1 < 0: start_f = 0
+ else: start_f = len(c.path)-1
+
+ # Count failed
+ self.circ_failed += 1
+ # XXX: Differentiate between extender and extendee
+ for r in self.circuits[c.circ_id].path[start_f:len(c.path)+1]:
+ r.circ_failed += 1
+ if not reason in r.reason_failed:
+ r.reason_failed[reason] = 1
+ else: r.reason_failed[reason]+=1
+ if reason not in self.failed_reasons:
+ self.failed_reasons[reason] = FailedRouterList(reason)
+ self.failed_reasons[reason].add_r(r)
+
+ for r in self.circuits[c.circ_id].path[len(c.path)+1:]:
+ r.circ_uncounted += 1
+
+ # Don't count if failed was set this round, don't set
+ # suspected..
+ for r in self.circuits[c.circ_id].path[:start_f]:
+ r.circ_suspected += 1
+ if not reason in r.reason_suspected:
+ r.reason_suspected[reason] = 1
+ else: r.reason_suspected[reason]+=1
+ if reason not in self.suspect_reasons:
+ self.suspect_reasons[reason] = SuspectRouterList(reason)
+ self.suspect_reasons[reason].add_r(r)
+ elif c.status == "CLOSED":
+ # Since PathBuilder deletes the circuit on a failed,
+ # we only get this for a clean close
+ for r in self.circuits[c.circ_id].path:
+ r.circ_chosen += 1
+ if lreason in ("REQUESTED", "FINISHED", "ORIGIN"):
+ r.circ_succeeded += 1
+ else:
+ if not reason in r.reason_suspected:
+ r.reason_suspected[reason] = 1
+ else: r.reason_suspected[reason] += 1
+ r.circ_suspected+= 1
+ if reason not in self.suspect_reasons:
+ self.suspect_reasons[reason] = SuspectRouterList(reason)
+ self.suspect_reasons[reason].add_r(r)
+ PathBuilder.circ_status_event(self, c)
+
+ def count_stream_reason_failed(self, s, reason):
+ "Count the routers involved in a failure"
+ # Update failed count,reason_failed for exit
+ r = self.circuits[s.circ_id].exit
+ if not reason in r.reason_failed: r.reason_failed[reason] = 1
+ else: r.reason_failed[reason]+=1
+ r.strm_failed += 1
+ if reason not in self.failed_reasons:
+ self.failed_reasons[reason] = FailedRouterList(reason)
+ self.failed_reasons[reason].add_r(r)
+
+ def count_stream_suspects(self, s, lreason, reason):
+ "Count the routers 'suspected' of being involved in a failure"
+ if lreason in ("TIMEOUT", "INTERNAL", "TORPROTOCOL" "DESTROY"):
+ for r in self.circuits[s.circ_id].path[:-1]:
+ r.strm_suspected += 1
+ if not reason in r.reason_suspected:
+ r.reason_suspected[reason] = 1
+ else: r.reason_suspected[reason]+=1
+ if reason not in self.suspect_reasons:
+ self.suspect_reasons[reason] = SuspectRouterList(reason)
+ self.suspect_reasons[reason].add_r(r)
+ else:
+ for r in self.circuits[s.circ_id].path[:-1]:
+ r.strm_uncounted += 1
+
+ def stream_status_event(self, s):
+ if s.strm_id in self.streams and not self.streams[s.strm_id].ignored:
+ # TODO: Hrmm, consider making this sane in TorCtl.
+ if s.reason: lreason = s.reason
+ else: lreason = "NONE"
+ if s.remote_reason: rreason = s.remote_reason
+ else: rreason = "NONE"
+ reason = s.event_name+":"+s.status+":"+lreason+":"+rreason+":"+self.streams[s.strm_id].kind
+ if (s.status in ("DETACHED", "FAILED", "CLOSED", "SUCCEEDED")
+ and not s.circ_id):
+ # XXX: REMAPs can do this (normal). Also REASON=DESTROY (bug?)
+ # Also timeouts.. Those should use the pending circ instead
+ # of returning..
+ plog("WARN", "Stream "+str(s.strm_id)+" detached from no circuit!")
+ PathBuilder.stream_status_event(self, s)
+ return
+
+ # Verify circ id matches stream.circ
+ if s.status not in ("NEW", "NEWRESOLVE", "REMAP"):
+ circ = self.streams[s.strm_id].circ
+ if not circ: circ = self.streams[s.strm_id].pending_circ
+ if circ and circ.circ_id != s.circ_id:
+ plog("WARN", str(s.strm_id) + " has mismatch of "
+ +str(s.circ_id)+" v "+str(circ.circ_id))
+ if s.circ_id and s.circ_id not in self.circuits:
+ plog("NOTICE", "Unknown circuit "+str(s.circ_id)
+ +" for stream "+str(s.strm_id))
+ return
+
+ if s.status == "DETACHED":
+ if self.streams[s.strm_id].attached_at:
+ plog("WARN", str(s.strm_id)+" detached after succeeded")
+ # Update strm_chosen count
+ self.strm_count += 1
+ for r in self.circuits[s.circ_id].path: r.strm_chosen += 1
+ self.strm_failed += 1
+ self.count_stream_suspects(s, lreason, reason)
+ self.count_stream_reason_failed(s, reason)
+ elif s.status == "FAILED":
+ # HACK. We get both failed and closed for the same stream,
+ # with different reasons. Might as well record both, since they
+ # often differ.
+ self.streams[s.strm_id].failed_reason = reason
+ elif s.status == "CLOSED":
+ # Always get both a closed and a failed..
+ # - Check if the circuit exists still
+ # Update strm_chosen count
+ self.strm_count += 1
+ for r in self.circuits[s.circ_id].path: r.strm_chosen += 1
+
+ # Update bw stats. XXX: Don't do this for resolve streams
+ if self.streams[s.strm_id].attached_at:
+ lifespan = self.streams[s.strm_id].lifespan(s.arrived_at)
+ for r in self.streams[s.strm_id].circ.path:
+ r.bwstats.add_bw(self.streams[s.strm_id].bytes_written+
+ self.streams[s.strm_id].bytes_read, lifespan)
+
+ if self.streams[s.strm_id].failed:
+ reason = self.streams[s.strm_id].failed_reason+":"+lreason+":"+rreason
+
+ self.count_stream_suspects(s, lreason, reason)
+
+ r = self.circuits[s.circ_id].exit
+ if (not self.streams[s.strm_id].failed
+ and (lreason == "DONE" or (lreason == "END" and rreason == "DONE"))):
+ r.strm_succeeded += 1
+ else:
+ self.strm_failed += 1
+ self.count_stream_reason_failed(s, reason)
+ PathBuilder.stream_status_event(self, s)
+
+ def ns_event(self, n):
+ PathBuilder.ns_event(self, n)
+ now = n.arrived_at
+ for ns in n.nslist:
+ if not ns.idhex in self.routers:
+ continue
+ r = self.routers[ns.idhex]
+ if "Running" in ns.flags:
+ if not r.became_active_at:
+ r.became_active_at = now
+ r.total_hibernation_time += now - r.hibernated_at
+ r.hibernated_at = 0
+ else:
+ if not r.hibernated_at:
+ r.hibernated_at = now
+ r.total_active_uptime += now - r.became_active_at
+ r.became_active_at = 0
Deleted: torflow/branches/gsoc2008/TorCtl.local/TorCtl.py
===================================================================
--- torflow/branches/gsoc2008/TorCtl/TorCtl.py 2008-07-07 16:40:57 UTC (rev 15734)
+++ torflow/branches/gsoc2008/TorCtl.local/TorCtl.py 2008-07-07 19:37:47 UTC (rev 15742)
@@ -1,1091 +0,0 @@
-#!/usr/bin/python
-# TorCtl.py -- Python module to interface with Tor Control interface.
-# Copyright 2005 Nick Mathewson
-# Copyright 2007 Mike Perry. See LICENSE file.
-
-"""
-Library to control Tor processes.
-
-This library handles sending commands, parsing responses, and delivering
-events to and from the control port. The basic usage is to create a
-socket, wrap that in a TorCtl.Connection, and then add an EventHandler
-to that connection. A simple example with a DebugEventHandler (that just
-echoes the events back to stdout) is present in run_example().
-
-Note that the TorCtl.Connection is fully compatible with the more
-advanced EventHandlers in TorCtl.PathSupport (and of course any other
-custom event handlers that you may extend off of those).
-
-This package also contains a helper class for representing Routers, and
-classes and constants for each event.
-
-"""
-
-__all__ = ["EVENT_TYPE", "TorCtlError", "TorCtlClosed", "ProtocolError",
- "ErrorReply", "NetworkStatus", "ExitPolicyLine", "Router",
- "RouterVersion", "Connection", "parse_ns_body",
- "EventHandler", "DebugEventHandler", "NetworkStatusEvent",
- "NewDescEvent", "CircuitEvent", "StreamEvent", "ORConnEvent",
- "StreamBwEvent", "LogEvent", "AddrMapEvent", "BWEvent",
- "UnknownEvent" ]
-
-import os
-import re
-import struct
-import sys
-import threading
-import Queue
-import datetime
-import traceback
-import socket
-import binascii
-import types
-import time
-from TorUtil import *
-
-# Types of "EVENT" message.
-EVENT_TYPE = Enum2(
- CIRC="CIRC",
- STREAM="STREAM",
- ORCONN="ORCONN",
- STREAM_BW="STREAM_BW",
- BW="BW",
- NS="NS",
- NEWDESC="NEWDESC",
- ADDRMAP="ADDRMAP",
- DEBUG="DEBUG",
- INFO="INFO",
- NOTICE="NOTICE",
- WARN="WARN",
- ERR="ERR")
-
-class TorCtlError(Exception):
- "Generic error raised by TorControl code."
- pass
-
-class TorCtlClosed(TorCtlError):
- "Raised when the controller connection is closed by Tor (not by us.)"
- pass
-
-class ProtocolError(TorCtlError):
- "Raised on violations in Tor controller protocol"
- pass
-
-class ErrorReply(TorCtlError):
- "Raised when Tor controller returns an error"
- pass
-
-class NetworkStatus:
- "Filled in during NS events"
- def __init__(self, nickname, idhash, orhash, updated, ip, orport, dirport, flags):
- self.nickname = nickname
- self.idhash = idhash
- self.orhash = orhash
- self.ip = ip
- self.orport = int(orport)
- self.dirport = int(dirport)
- self.flags = flags
- self.idhex = (self.idhash + "=").decode("base64").encode("hex").upper()
- m = re.search(r"(\d+)-(\d+)-(\d+) (\d+):(\d+):(\d+)", updated)
- self.updated = datetime.datetime(*map(int, m.groups()))
-
-class NetworkStatusEvent:
- def __init__(self, event_name, nslist):
- self.event_name = event_name
- self.arrived_at = 0
- self.nslist = nslist # List of NetworkStatus objects
-
-class NewDescEvent:
- def __init__(self, event_name, idlist):
- self.event_name = event_name
- self.arrived_at = 0
- self.idlist = idlist
-
-class CircuitEvent:
- def __init__(self, event_name, circ_id, status, path, reason,
- remote_reason):
- self.event_name = event_name
- self.arrived_at = 0
- self.circ_id = circ_id
- self.status = status
- self.path = path
- self.reason = reason
- self.remote_reason = remote_reason
-
-class StreamEvent:
- def __init__(self, event_name, strm_id, status, circ_id, target_host,
- target_port, reason, remote_reason, source, source_addr, purpose):
- self.event_name = event_name
- self.arrived_at = 0
- self.strm_id = strm_id
- self.status = status
- self.circ_id = circ_id
- self.target_host = target_host
- self.target_port = int(target_port)
- self.reason = reason
- self.remote_reason = remote_reason
- self.source = source
- self.source_addr = source_addr
- self.purpose = purpose
-
-class ORConnEvent:
- def __init__(self, event_name, status, endpoint, age, read_bytes,
- wrote_bytes, reason, ncircs):
- self.event_name = event_name
- self.arrived_at = 0
- self.status = status
- self.endpoint = endpoint
- self.age = age
- self.read_bytes = read_bytes
- self.wrote_bytes = wrote_bytes
- self.reason = reason
- self.ncircs = ncircs
-
-class StreamBwEvent:
- def __init__(self, event_name, strm_id, read, written):
- self.event_name = event_name
- self.strm_id = int(strm_id)
- self.bytes_read = int(read)
- self.bytes_written = int(written)
-
-class LogEvent:
- def __init__(self, level, msg):
- self.event_name = self.level = level
- self.msg = msg
-
-class AddrMapEvent:
- def __init__(self, event_name, from_addr, to_addr, when):
- self.event_name = event_name
- self.from_addr = from_addr
- self.to_addr = to_addr
- self.when = when
-
-class BWEvent:
- def __init__(self, event_name, read, written):
- self.event_name = event_name
- self.read = read
- self.written = written
-
-class UnknownEvent:
- def __init__(self, event_name, event_string):
- self.event_name = event_name
- self.event_string = event_string
-
-class ExitPolicyLine:
- """ Class to represent a line in a Router's exit policy in a way
- that can be easily checked. """
- def __init__(self, match, ip_mask, port_low, port_high):
- self.match = match
- if ip_mask == "*":
- self.ip = 0
- self.netmask = 0
- else:
- if not "/" in ip_mask:
- self.netmask = 0xFFFFFFFF
- ip = ip_mask
- else:
- ip, mask = ip_mask.split("/")
- if re.match(r"\d+.\d+.\d+.\d+", mask):
- self.netmask=struct.unpack(">I", socket.inet_aton(mask))[0]
- else:
- self.netmask = ~(2**(32 - int(mask)) - 1)
- self.ip = struct.unpack(">I", socket.inet_aton(ip))[0]
- self.ip &= self.netmask
- if port_low == "*":
- self.port_low,self.port_high = (0,65535)
- else:
- if not port_high:
- port_high = port_low
- self.port_low = int(port_low)
- self.port_high = int(port_high)
-
- def check(self, ip, port):
- """Check to see if an ip and port is matched by this line.
- Returns true if the line is an Accept, and False if it is a Reject. """
- ip = struct.unpack(">I", socket.inet_aton(ip))[0]
- if (ip & self.netmask) == self.ip:
- if self.port_low <= port and port <= self.port_high:
- return self.match
- return -1
-
-class RouterVersion:
- """ Represents a Router's version. Overloads all comparison operators
- to check for newer, older, or equivalent versions. """
- def __init__(self, version):
- if version:
- v = re.search("^(\d+).(\d+).(\d+).(\d+)", version).groups()
- self.version = int(v[0])*0x1000000 + int(v[1])*0x10000 + int(v[2])*0x100 + int(v[3])
- self.ver_string = version
- else:
- self.version = version
- self.ver_string = "unknown"
-
- def __lt__(self, other): return self.version < other.version
- def __gt__(self, other): return self.version > other.version
- def __ge__(self, other): return self.version >= other.version
- def __le__(self, other): return self.version <= other.version
- def __eq__(self, other): return self.version == other.version
- def __ne__(self, other): return self.version != other.version
- def __str__(self): return self.ver_string
-
-class Router:
- """
- Class to represent a router from a descriptor. Can either be
- created from the parsed fields, or can be built from a
- descriptor+NetworkStatus
- """
- def __init__(self, idhex, name, bw, down, exitpolicy, flags, ip, version, os, uptime):
- self.idhex = idhex
- self.nickname = name
- self.bw = bw
- self.exitpolicy = exitpolicy
- self.flags = flags
- self.down = down
- self.ip = struct.unpack(">I", socket.inet_aton(ip))[0]
- self.version = RouterVersion(version)
- self.os = os
- self.list_rank = 0 # position in a sorted list of routers.
- self.uptime = uptime
-
- def __str__(self):
- s = self.idhex, self.nickname
- return s.__str__()
-
- def build_from_desc(desc, ns):
- """
- Static method of Router that parses a descriptor string into this class.
- 'desc' is a full descriptor as a string.
- 'ns' is a TorCtl.NetworkStatus instance for this router (needed for
- the flags, the nickname, and the idhex string).
- Returns a Router instance.
- """
- # XXX: Compile these regular expressions? This is an expensive process
- # Use http://docs.python.org/lib/profile.html to verify this is
- # the part of startup that is slow
- exitpolicy = []
- dead = not ("Running" in ns.flags)
- bw_observed = 0
- version = None
- os = None
- uptime = 0
- ip = 0
- router = "[none]"
-
- for line in desc:
- rt = re.search(r"^router (\S+) (\S+)", line)
- fp = re.search(r"^opt fingerprint (.+).*on (\S+)", line)
- pl = re.search(r"^platform Tor (\S+).*on (\S+)", line)
- ac = re.search(r"^accept (\S+):([^-]+)(?:-(\d+))?", line)
- rj = re.search(r"^reject (\S+):([^-]+)(?:-(\d+))?", line)
- bw = re.search(r"^bandwidth \d+ \d+ (\d+)", line)
- up = re.search(r"^uptime (\d+)", line)
- if re.search(r"^opt hibernating 1", line):
- #dead = 1 # XXX: Technically this may be stale..
- if ("Running" in ns.flags):
- plog("INFO", "Hibernating router "+ns.nickname+" is running..")
- if ac:
- exitpolicy.append(ExitPolicyLine(True, *ac.groups()))
- elif rj:
- exitpolicy.append(ExitPolicyLine(False, *rj.groups()))
- elif bw:
- bw_observed = int(bw.group(1))
- elif pl:
- version, os = pl.groups()
- elif up:
- uptime = int(up.group(1))
- elif rt:
- router,ip = rt.groups()
- if router != ns.nickname:
- plog("NOTICE", "Got different names " + ns.nickname + " vs " +
- router + " for " + ns.idhex)
- if not bw_observed and not dead and ("Valid" in ns.flags):
- plog("INFO", "No bandwidth for live router " + ns.nickname)
- if not version or not os:
- plog("INFO", "No version and/or OS for router " + ns.nickname)
- return Router(ns.idhex, ns.nickname, bw_observed, dead, exitpolicy,
- ns.flags, ip, version, os, uptime)
- build_from_desc = Callable(build_from_desc)
-
- def update_to(self, new):
- """ Somewhat hackish method to update this router to be a copy of
- 'new' """
- if self.idhex != new.idhex:
- plog("ERROR", "Update of router "+self.nickname+"changes idhex!")
- self.idhex = new.idhex
- self.nickname = new.nickname
- self.bw = new.bw
- self.exitpolicy = new.exitpolicy
- self.flags = new.flags
- self.ip = new.ip
- self.version = new.version
- self.os = new.os
- self.uptime = new.uptime
-
- def will_exit_to(self, ip, port):
- """ Check the entire exitpolicy to see if the router will allow
- connections to 'ip':'port' """
- for line in self.exitpolicy:
- ret = line.check(ip, port)
- if ret != -1:
- return ret
- plog("WARN", "No matching exit line for "+self.nickname)
- return False
-
-class Connection:
- """A Connection represents a connection to the Tor process via the
- control port."""
- def __init__(self, sock):
- """Create a Connection to communicate with the Tor process over the
- socket 'sock'.
- """
- self._handler = None
- self._handleFn = None
- self._sendLock = threading.RLock()
- self._queue = Queue.Queue()
- self._thread = None
- self._closedEx = None
- self._closed = 0
- self._closeHandler = None
- self._eventThread = None
- self._eventQueue = Queue.Queue()
- self._s = BufSock(sock)
- self._debugFile = None
-
- def set_close_handler(self, handler):
- """Call 'handler' when the Tor process has closed its connection or
- given us an exception. If we close normally, no arguments are
- provided; otherwise, it will be called with an exception as its
- argument.
- """
- self._closeHandler = handler
-
- def close(self):
- """Shut down this controller connection"""
- self._sendLock.acquire()
- try:
- self._queue.put("CLOSE")
- self._eventQueue.put((time.time(), "CLOSE"))
- finally:
- self._sendLock.release()
-
- def launch_thread(self, daemon=1):
- """Launch a background thread to handle messages from the Tor process."""
- assert self._thread is None
- t = threading.Thread(target=self._loop)
- if daemon:
- t.setDaemon(daemon)
- t.start()
- self._thread = t
- t = threading.Thread(target=self._eventLoop)
- if daemon:
- t.setDaemon(daemon)
- t.start()
- self._eventThread = t
- return self._thread
-
- def _loop(self):
- """Main subthread loop: Read commands from Tor, and handle them either
- as events or as responses to other commands.
- """
- while 1:
- try:
- isEvent, reply = self._read_reply()
- except:
- self._err(sys.exc_info())
- return
-
- if isEvent:
- if self._handler is not None:
- self._eventQueue.put((time.time(), reply))
- else:
- cb = self._queue.get() # atomic..
- if cb == "CLOSE":
- self._s.close()
- self._s = None
- self._closed = 1
- return
- else:
- cb(reply)
-
- def _err(self, (tp, ex, tb), fromEventLoop=0):
- """DOCDOC"""
- # silent death is bad :(
- traceback.print_exception(tp, ex, tb)
- if self._s:
- try:
- self.close()
- except:
- pass
- self._sendLock.acquire()
- try:
- self._closedEx = ex
- self._closed = 1
- finally:
- self._sendLock.release()
- while 1:
- try:
- cb = self._queue.get(timeout=0)
- if cb != "CLOSE":
- cb("EXCEPTION")
- except Queue.Empty:
- break
- if self._closeHandler is not None:
- self._closeHandler(ex)
- return
-
- def _eventLoop(self):
- """DOCDOC"""
- while 1:
- (timestamp, reply) = self._eventQueue.get()
- if reply[0][0] == "650" and reply[0][1] == "OK":
- plog("DEBUG", "Ignoring incompatible syntactic sugar: 650 OK")
- continue
- if reply == "CLOSE":
- return
- try:
- self._handleFn(timestamp, reply)
- except:
- for code, msg, data in reply:
- plog("WARN", "No event for: "+str(code)+" "+str(msg))
- self._err(sys.exc_info(), 1)
- return
-
- def _sendImpl(self, sendFn, msg):
- """DOCDOC"""
- if self._thread is None:
- self.launch_thread(1)
- # This condition will get notified when we've got a result...
- condition = threading.Condition()
- # Here's where the result goes...
- result = []
-
- if self._closedEx is not None:
- raise self._closedEx
- elif self._closed:
- raise TorCtlClosed()
-
- def cb(reply,condition=condition,result=result):
- condition.acquire()
- try:
- result.append(reply)
- condition.notify()
- finally:
- condition.release()
-
- # Sends a message to Tor...
- self._sendLock.acquire() # ensure queue+sendmsg is atomic
- try:
- self._queue.put(cb)
- sendFn(msg) # _doSend(msg)
- finally:
- self._sendLock.release()
-
- # Now wait till the answer is in...
- condition.acquire()
- try:
- while not result:
- condition.wait()
- finally:
- condition.release()
-
- # ...And handle the answer appropriately.
- assert len(result) == 1
- reply = result[0]
- if reply == "EXCEPTION":
- raise self._closedEx
-
- return reply
-
-
- def debug(self, f):
- """DOCDOC"""
- self._debugFile = f
-
- def set_event_handler(self, handler):
- """Cause future events from the Tor process to be sent to 'handler'.
- """
- self._handler = handler
- self._handleFn = handler._handle1
-
- def _read_reply(self):
- lines = []
- while 1:
- line = self._s.readline().strip()
- if self._debugFile:
- self._debugFile.write(" %s\n" % line)
- if len(line)<4:
- raise ProtocolError("Badly formatted reply line: Too short")
- code = line[:3]
- tp = line[3]
- s = line[4:]
- if tp == "-":
- lines.append((code, s, None))
- elif tp == " ":
- lines.append((code, s, None))
- isEvent = (lines and lines[0][0][0] == '6')
- return isEvent, lines
- elif tp != "+":
- raise ProtocolError("Badly formatted reply line: unknown type %r"%tp)
- else:
- more = []
- while 1:
- line = self._s.readline()
- if self._debugFile:
- self._debugFile.write("+++ %s" % line)
- if line in (".\r\n", ".\n", "650 OK\n", "650 OK\r\n"):
- break
- more.append(line)
- lines.append((code, s, unescape_dots("".join(more))))
- isEvent = (lines and lines[0][0][0] == '6')
- if isEvent: # Need "250 OK" if it's not an event. Otherwise, end
- return (isEvent, lines)
-
- # Notreached
- raise TorCtlError()
-
- def _doSend(self, msg):
- if self._debugFile:
- amsg = msg
- lines = amsg.split("\n")
- if len(lines) > 2:
- amsg = "\n".join(lines[:2]) + "\n"
- self._debugFile.write(">>> %s" % amsg)
- self._s.write(msg)
-
- def sendAndRecv(self, msg="", expectedTypes=("250", "251")):
- """Helper: Send a command 'msg' to Tor, and wait for a command
- in response. If the response type is in expectedTypes,
- return a list of (tp,body,extra) tuples. If it is an
- error, raise ErrorReply. Otherwise, raise ProtocolError.
- """
- if type(msg) == types.ListType:
- msg = "".join(msg)
- assert msg.endswith("\r\n")
-
- lines = self._sendImpl(self._doSend, msg)
- # print lines
- for tp, msg, _ in lines:
- if tp[0] in '45':
- raise ErrorReply("%s %s"%(tp, msg))
- if tp not in expectedTypes:
- raise ProtocolError("Unexpectd message type %r"%tp)
-
- return lines
-
- def authenticate(self, secret=""):
- """Send an authenticating secret to Tor. You'll need to call this
- method before Tor can start.
- """
- #hexstr = binascii.b2a_hex(secret)
- self.sendAndRecv("AUTHENTICATE \"%s\"\r\n"%secret)
-
- def get_option(self, name):
- """Get the value of the configuration option named 'name'. To
- retrieve multiple values, pass a list for 'name' instead of
- a string. Returns a list of (key,value) pairs.
- Refer to section 3.3 of control-spec.txt for a list of valid names.
- """
- if not isinstance(name, str):
- name = " ".join(name)
- lines = self.sendAndRecv("GETCONF %s\r\n" % name)
-
- r = []
- for _,line,_ in lines:
- try:
- key, val = line.split("=", 1)
- r.append((key,val))
- except ValueError:
- r.append((line, None))
-
- return r
-
- def set_option(self, key, value):
- """Set the value of the configuration option 'key' to the value 'value'.
- """
- self.set_options([(key, value)])
-
- def set_options(self, kvlist):
- """Given a list of (key,value) pairs, set them as configuration
- options.
- """
- if not kvlist:
- return
- msg = " ".join(["%s=%s"%(k,quote(v)) for k,v in kvlist])
- self.sendAndRecv("SETCONF %s\r\n"%msg)
-
- def reset_options(self, keylist):
- """Reset the options listed in 'keylist' to their default values.
-
- Tor started implementing this command in version 0.1.1.7-alpha;
- previous versions wanted you to set configuration keys to "".
- That no longer works.
- """
- self.sendAndRecv("RESETCONF %s\r\n"%(" ".join(keylist)))
-
- def get_network_status(self, who="all"):
- """Get the entire network status list. Returns a list of
- TorCtl.NetworkStatus instances."""
- return parse_ns_body(self.sendAndRecv("GETINFO ns/"+who+"\r\n")[0][2])
-
- def get_router(self, ns):
- """Fill in a Router class corresponding to a given NS class"""
- desc = self.sendAndRecv("GETINFO desc/id/" + ns.idhex + "\r\n")[0][2].split("\n")
- return Router.build_from_desc(desc, ns)
-
-
- def read_routers(self, nslist):
- """ Given a list a NetworkStatuses in 'nslist', this function will
- return a list of new Router instances.
- """
- bad_key = 0
- new = []
- for ns in nslist:
- try:
- r = self.get_router(ns)
- new.append(r)
- except ErrorReply:
- bad_key += 1
- if "Running" in ns.flags:
- plog("NOTICE", "Running router "+ns.nickname+"="
- +ns.idhex+" has no descriptor")
- except:
- traceback.print_exception(*sys.exc_info())
- continue
-
- return new
-
- def get_info(self, name):
- """Return the value of the internal information field named 'name'.
- Refer to section 3.9 of control-spec.txt for a list of valid names.
- DOCDOC
- """
- if not isinstance(name, str):
- name = " ".join(name)
- lines = self.sendAndRecv("GETINFO %s\r\n"%name)
- d = {}
- for _,msg,more in lines:
- if msg == "OK":
- break
- try:
- k,rest = msg.split("=",1)
- except ValueError:
- raise ProtocolError("Bad info line %r",msg)
- if more:
- d[k] = more
- else:
- d[k] = rest
- return d
-
- def set_events(self, events, extended=False):
- """Change the list of events that the event handler is interested
- in to those in 'events', which is a list of event names.
- Recognized event names are listed in section 3.3 of the control-spec
- """
- if extended:
- plog ("DEBUG", "SETEVENTS EXTENDED %s\r\n" % " ".join(events))
- self.sendAndRecv("SETEVENTS EXTENDED %s\r\n" % " ".join(events))
- else:
- self.sendAndRecv("SETEVENTS %s\r\n" % " ".join(events))
-
- def save_conf(self):
- """Flush all configuration changes to disk.
- """
- self.sendAndRecv("SAVECONF\r\n")
-
- def send_signal(self, sig):
- """Send the signal 'sig' to the Tor process; The allowed values for
- 'sig' are listed in section 3.6 of control-spec.
- """
- sig = { 0x01 : "HUP",
- 0x02 : "INT",
- 0x03 : "NEWNYM",
- 0x0A : "USR1",
- 0x0C : "USR2",
- 0x0F : "TERM" }.get(sig,sig)
- self.sendAndRecv("SIGNAL %s\r\n"%sig)
-
- def resolve(self, host):
- """ Launch a remote hostname lookup request:
- 'host' may be a hostname or IPv4 address
- """
- # TODO: handle "mode=reverse"
- self.sendAndRecv("RESOLVE %s\r\n"%host)
-
- def map_address(self, kvList):
- """ Sends the MAPADDRESS command for each of the tuples in kvList """
- if not kvList:
- return
- m = " ".join([ "%s=%s" for k,v in kvList])
- lines = self.sendAndRecv("MAPADDRESS %s\r\n"%m)
- r = []
- for _,line,_ in lines:
- try:
- key, val = line.split("=", 1)
- except ValueError:
- raise ProtocolError("Bad address line %r",v)
- r.append((key,val))
- return r
-
- def extend_circuit(self, circid, hops):
- """Tell Tor to extend the circuit identified by 'circid' through the
- servers named in the list 'hops'.
- """
- if circid is None:
- circid = "0"
- plog("DEBUG", "Extending circuit")
- lines = self.sendAndRecv("EXTENDCIRCUIT %d %s\r\n"
- %(circid, ",".join(hops)))
- tp,msg,_ = lines[0]
- m = re.match(r'EXTENDED (\S*)', msg)
- if not m:
- raise ProtocolError("Bad extended line %r",msg)
- plog("DEBUG", "Circuit extended")
- return int(m.group(1))
-
- def redirect_stream(self, streamid, newaddr, newport=""):
- """DOCDOC"""
- if newport:
- self.sendAndRecv("REDIRECTSTREAM %d %s %s\r\n"%(streamid, newaddr, newport))
- else:
- self.sendAndRecv("REDIRECTSTREAM %d %s\r\n"%(streamid, newaddr))
-
- def attach_stream(self, streamid, circid, hop=None):
- """Attach a stream to a circuit, specify both by IDs. If hop is given,
- try to use the specified hop in the circuit as the exit node for
- this stream.
- """
- if hop:
- self.sendAndRecv("ATTACHSTREAM %d %d HOP=%d\r\n"%(streamid, circid, hop))
- plog("DEBUG", "Attaching stream: "+str(streamid)+" to hop "+str(hop)+" of circuit "+str(circid))
- else:
- self.sendAndRecv("ATTACHSTREAM %d %d\r\n"%(streamid, circid))
- plog("DEBUG", "Attaching stream: "+str(streamid)+" to circuit "+str(circid))
-
- def close_stream(self, streamid, reason=0, flags=()):
- """DOCDOC"""
- self.sendAndRecv("CLOSESTREAM %d %s %s\r\n"
- %(streamid, reason, "".join(flags)))
-
- def close_circuit(self, circid, reason=0, flags=()):
- """DOCDOC"""
- self.sendAndRecv("CLOSECIRCUIT %d %s %s\r\n"
- %(circid, reason, "".join(flags)))
-
- def post_descriptor(self, desc):
- self.sendAndRecv("+POSTDESCRIPTOR\r\n%s"%escape_dots(desc))
-
-def parse_ns_body(data):
- """Parse the body of an NS event or command into a list of
- NetworkStatus instances"""
- nsgroups = re.compile(r"^r ", re.M).split(data)
- nsgroups.pop(0)
- nslist = []
- for nsline in nsgroups:
- m = re.search(r"^s((?:\s\S*)+)", nsline, re.M)
- flags = m.groups()
- flags = flags[0].strip().split(" ")
- m = re.match(r"(\S+)\s(\S+)\s(\S+)\s(\S+\s\S+)\s(\S+)\s(\d+)\s(\d+)", nsline)
- nslist.append(NetworkStatus(*(m.groups() + (flags,))))
- return nslist
-
-class EventHandler:
- """An 'EventHandler' wraps callbacks for the events Tor can return.
- Each event argument is an instance of the corresponding event
- class."""
- def __init__(self):
- """Create a new EventHandler."""
- self._map1 = {
- "CIRC" : self.circ_status_event,
- "STREAM" : self.stream_status_event,
- "ORCONN" : self.or_conn_status_event,
- "STREAM_BW" : self.stream_bw_event,
- "BW" : self.bandwidth_event,
- "DEBUG" : self.msg_event,
- "INFO" : self.msg_event,
- "NOTICE" : self.msg_event,
- "WARN" : self.msg_event,
- "ERR" : self.msg_event,
- "NEWDESC" : self.new_desc_event,
- "ADDRMAP" : self.address_mapped_event,
- "NS" : self.ns_event
- }
-
- def _handle1(self, timestamp, lines):
- """Dispatcher: called from Connection when an event is received."""
- for code, msg, data in lines:
- event = self._decode1(msg, data)
- event.arrived_at = timestamp
- self.heartbeat_event(event)
- self._map1.get(event.event_name, self.unknown_event)(event)
-
- def _decode1(self, body, data):
- """Unpack an event message into a type/arguments-tuple tuple."""
- if " " in body:
- evtype,body = body.split(" ",1)
- else:
- evtype,body = body,""
- evtype = evtype.upper()
- if evtype == "CIRC":
- m = re.match(r"(\d+)\s+(\S+)(\s\S+)?(\s\S+)?(\s\S+)?", body)
- if not m:
- raise ProtocolError("CIRC event misformatted.")
- ident,status,path,reason,remote = m.groups()
- ident = int(ident)
- if path:
- if "REASON=" in path:
- remote = reason
- reason = path
- path=[]
- else:
- path = path.strip().split(",")
- else:
- path = []
- if reason: reason = reason[8:]
- if remote: remote = remote[15:]
- event = CircuitEvent(evtype, ident, status, path, reason, remote)
- elif evtype == "STREAM":
- #plog("DEBUG", "STREAM: "+body)
- m = re.match(r"(\S+)\s+(\S+)\s+(\S+)\s+(\S+):(\d+)(\sREASON=\S+)?(\sREMOTE_REASON=\S+)?(\sSOURCE=\S+)?(\sSOURCE_ADDR=\S+)?(\sPURPOSE=\S+)?", body)
- if not m:
- raise ProtocolError("STREAM event misformatted.")
- ident,status,circ,target_host,target_port,reason,remote,source,source_addr,purpose = m.groups()
- ident,circ = map(int, (ident,circ))
- if reason: reason = reason[8:]
- if remote: remote = remote[15:]
- if source: source = source[8:]
- if source_addr: source_addr = source_addr[13:]
- if purpose: purpose = purpose[9:]
- event = StreamEvent(evtype, ident, status, circ, target_host,
- int(target_port), reason, remote, source, source_addr, purpose)
- elif evtype == "ORCONN":
- m = re.match(r"(\S+)\s+(\S+)(\sAGE=\S+)?(\sREAD=\S+)?(\sWRITTEN=\S+)?(\sREASON=\S+)?(\sNCIRCS=\S+)?", body)
- if not m:
- raise ProtocolError("ORCONN event misformatted.")
- target, status, age, read, wrote, reason, ncircs = m.groups()
-
- #plog("DEBUG", "ORCONN: "+body)
- if ncircs: ncircs = int(ncircs[8:])
- else: ncircs = 0
- if reason: reason = reason[8:]
- if age: age = int(age[5:])
- else: age = 0
- if read: read = int(read[6:])
- else: read = 0
- if wrote: wrote = int(wrote[9:])
- else: wrote = 0
- event = ORConnEvent(evtype, status, target, age, read, wrote,
- reason, ncircs)
- elif evtype == "STREAM_BW":
- m = re.match(r"(\d+)\s+(\d+)\s+(\d+)", body)
- if not m:
- raise ProtocolError("STREAM_BW event misformatted.")
- event = StreamBwEvent(evtype, *m.groups())
- elif evtype == "BW":
- m = re.match(r"(\d+)\s+(\d+)", body)
- if not m:
- raise ProtocolError("BANDWIDTH event misformatted.")
- read, written = map(long, m.groups())
- event = BWEvent(evtype, read, written)
- elif evtype in ("DEBUG", "INFO", "NOTICE", "WARN", "ERR"):
- event = LogEvent(evtype, body)
- elif evtype == "NEWDESC":
- event = NewDescEvent(evtype, body.split(" "))
- elif evtype == "ADDRMAP":
- # TODO: Also parse errors and GMTExpiry
- m = re.match(r'(\S+)\s+(\S+)\s+(\"[^"]+\"|\w+)', body)
- if not m:
- raise ProtocolError("ADDRMAP event misformatted.")
- fromaddr, toaddr, when = m.groups()
- if when.upper() == "NEVER":
- when = None
- else:
- when = time.strptime(when[1:-1], "%Y-%m-%d %H:%M:%S")
- event = AddrMapEvent(evtype, fromaddr, toaddr, when)
- elif evtype == "NS":
- event = NetworkStatusEvent(evtype, parse_ns_body(data))
- else:
- event = UnknownEvent(evtype, body)
-
- return event
-
- def heartbeat_event(self, event):
- """Called before any event is recieved. Convenience function
- for any cleanup/setup/reconfiguration you may need to do.
- """
- pass
-
- def unknown_event(self, event):
- """Called when we get an event type we don't recognize. This
- is almost alwyas an error.
- """
- raise NotImplemented()
-
- def circ_status_event(self, event):
- """Called when a circuit status changes if listening to CIRCSTATUS
- events."""
- raise NotImplemented()
-
- def stream_status_event(self, event):
- """Called when a stream status changes if listening to STREAMSTATUS
- events. """
- raise NotImplemented()
-
- def stream_bw_event(self, event):
- raise NotImplemented()
-
- def or_conn_status_event(self, event):
- """Called when an OR connection's status changes if listening to
- ORCONNSTATUS events."""
- raise NotImplemented()
-
- def bandwidth_event(self, event):
- """Called once a second if listening to BANDWIDTH events.
- """
- raise NotImplemented()
-
- def new_desc_event(self, event):
- """Called when Tor learns a new server descriptor if listenting to
- NEWDESC events.
- """
- raise NotImplemented()
-
- def msg_event(self, event):
- """Called when a log message of a given severity arrives if listening
- to INFO_MSG, NOTICE_MSG, WARN_MSG, or ERR_MSG events."""
- raise NotImplemented()
-
- def ns_event(self, event):
- raise NotImplemented()
-
- def address_mapped_event(self, event):
- """Called when Tor adds a mapping for an address if listening
- to ADDRESSMAPPED events.
- """
- raise NotImplemented()
-
-
-class DebugEventHandler(EventHandler):
- """Trivial debug event handler: reassembles all parsed events to stdout."""
- def circ_status_event(self, circ_event): # CircuitEvent()
- output = [circ_event.event_name, str(circ_event.circ_id),
- circ_event.status]
- if circ_event.path:
- output.append(",".join(circ_event.path))
- if circ_event.reason:
- output.append("REASON=" + circ_event.reason)
- if circ_event.remote_reason:
- output.append("REMOTE_REASON=" + circ_event.remote_reason)
- print " ".join(output)
-
- def stream_status_event(self, strm_event):
- output = [strm_event.event_name, str(strm_event.strm_id),
- strm_event.status, str(strm_event.circ_id),
- strm_event.target_host, str(strm_event.target_port)]
- if strm_event.reason:
- output.append("REASON=" + strm_event.reason)
- if strm_event.remote_reason:
- output.append("REMOTE_REASON=" + strm_event.remote_reason)
- print " ".join(output)
-
- def ns_event(self, ns_event):
- for ns in ns_event.nslist:
- print " ".join((ns_event.event_name, ns.nickname, ns.idhash,
- ns.updated.isoformat(), ns.ip, str(ns.orport),
- str(ns.dirport), " ".join(ns.flags)))
-
- def new_desc_event(self, newdesc_event):
- print " ".join((newdesc_event.event_name, " ".join(newdesc_event.idlist)))
-
- def or_conn_status_event(self, orconn_event):
- if orconn_event.age: age = "AGE="+str(orconn_event.age)
- else: age = ""
- if orconn_event.read_bytes: read = "READ="+str(orconn_event.read_bytes)
- else: read = ""
- if orconn_event.wrote_bytes: wrote = "WRITTEN="+str(orconn_event.wrote_bytes)
- else: wrote = ""
- if orconn_event.reason: reason = "REASON="+orconn_event.reason
- else: reason = ""
- if orconn_event.ncircs: ncircs = "NCIRCS="+str(orconn_event.ncircs)
- else: ncircs = ""
- print " ".join((orconn_event.event_name, orconn_event.endpoint,
- orconn_event.status, age, read, wrote, reason, ncircs))
-
- def msg_event(self, log_event):
- print log_event.event_name+" "+log_event.msg
-
- def bandwidth_event(self, bw_event):
- print bw_event.event_name+" "+str(bw_event.read)+" "+str(bw_event.written)
-
-def parseHostAndPort(h):
- """Given a string of the form 'address:port' or 'address' or
- 'port' or '', return a two-tuple of (address, port)
- """
- host, port = "localhost", 9100
- if ":" in h:
- i = h.index(":")
- host = h[:i]
- try:
- port = int(h[i+1:])
- except ValueError:
- print "Bad hostname %r"%h
- sys.exit(1)
- elif h:
- try:
- port = int(h)
- except ValueError:
- host = h
-
- return host, port
-
-def run_example(host,port):
- """ Example of basic TorCtl usage. See PathSupport for more advanced
- usage.
- """
- print "host is %s:%d"%(host,port)
- s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- s.connect((host,port))
- c = Connection(s)
- c.set_event_handler(DebugEventHandler())
- th = c.launch_thread()
- c.authenticate()
- print "nick",`c.get_option("nickname")`
- print `c.get_info("version")`
- #print `c.get_info("desc/name/moria1")`
- print `c.get_info("network-status")`
- print `c.get_info("addr-mappings/all")`
- print `c.get_info("addr-mappings/config")`
- print `c.get_info("addr-mappings/cache")`
- print `c.get_info("addr-mappings/control")`
-
- print `c.extend_circuit(0,["moria1"])`
- try:
- print `c.extend_circuit(0,[""])`
- except ErrorReply: # wtf?
- print "got error. good."
- except:
- print "Strange error", sys.exc_info()[0]
-
- #send_signal(s,1)
- #save_conf(s)
-
- #set_option(s,"1")
- #set_option(s,"bandwidthburstbytes 100000")
- #set_option(s,"runasdaemon 1")
- #set_events(s,[EVENT_TYPE.WARN])
-# c.set_events([EVENT_TYPE.ORCONN], True)
- c.set_events([EVENT_TYPE.STREAM, EVENT_TYPE.CIRC,
- EVENT_TYPE.NS, EVENT_TYPE.NEWDESC,
- EVENT_TYPE.ORCONN, EVENT_TYPE.BW], True)
-
- th.join()
- return
-
-if __name__ == '__main__':
- if len(sys.argv) > 2:
- print "Syntax: TorControl.py torhost:torport"
- sys.exit(0)
- else:
- sys.argv.append("localhost:9051")
- sh,sp = parseHostAndPort(sys.argv[1])
- run_example(sh,sp)
-
Copied: torflow/branches/gsoc2008/TorCtl.local/TorCtl.py (from rev 15741, torflow/branches/gsoc2008/TorCtl/TorCtl.py)
===================================================================
--- torflow/branches/gsoc2008/TorCtl.local/TorCtl.py (rev 0)
+++ torflow/branches/gsoc2008/TorCtl.local/TorCtl.py 2008-07-07 19:37:47 UTC (rev 15742)
@@ -0,0 +1,1091 @@
+#!/usr/bin/python
+# TorCtl.py -- Python module to interface with Tor Control interface.
+# Copyright 2005 Nick Mathewson
+# Copyright 2007 Mike Perry. See LICENSE file.
+
+"""
+Library to control Tor processes.
+
+This library handles sending commands, parsing responses, and delivering
+events to and from the control port. The basic usage is to create a
+socket, wrap that in a TorCtl.Connection, and then add an EventHandler
+to that connection. A simple example with a DebugEventHandler (that just
+echoes the events back to stdout) is present in run_example().
+
+Note that the TorCtl.Connection is fully compatible with the more
+advanced EventHandlers in TorCtl.PathSupport (and of course any other
+custom event handlers that you may extend off of those).
+
+This package also contains a helper class for representing Routers, and
+classes and constants for each event.
+
+"""
+
+__all__ = ["EVENT_TYPE", "TorCtlError", "TorCtlClosed", "ProtocolError",
+ "ErrorReply", "NetworkStatus", "ExitPolicyLine", "Router",
+ "RouterVersion", "Connection", "parse_ns_body",
+ "EventHandler", "DebugEventHandler", "NetworkStatusEvent",
+ "NewDescEvent", "CircuitEvent", "StreamEvent", "ORConnEvent",
+ "StreamBwEvent", "LogEvent", "AddrMapEvent", "BWEvent",
+ "UnknownEvent" ]
+
+import os
+import re
+import struct
+import sys
+import threading
+import Queue
+import datetime
+import traceback
+import socket
+import binascii
+import types
+import time
+from TorUtil import *
+
+# Types of "EVENT" message.
+EVENT_TYPE = Enum2(
+ CIRC="CIRC",
+ STREAM="STREAM",
+ ORCONN="ORCONN",
+ STREAM_BW="STREAM_BW",
+ BW="BW",
+ NS="NS",
+ NEWDESC="NEWDESC",
+ ADDRMAP="ADDRMAP",
+ DEBUG="DEBUG",
+ INFO="INFO",
+ NOTICE="NOTICE",
+ WARN="WARN",
+ ERR="ERR")
+
+class TorCtlError(Exception):
+ "Generic error raised by TorControl code."
+ pass
+
+class TorCtlClosed(TorCtlError):
+ "Raised when the controller connection is closed by Tor (not by us.)"
+ pass
+
+class ProtocolError(TorCtlError):
+ "Raised on violations in Tor controller protocol"
+ pass
+
+class ErrorReply(TorCtlError):
+ "Raised when Tor controller returns an error"
+ pass
+
+class NetworkStatus:
+ "Filled in during NS events"
+ def __init__(self, nickname, idhash, orhash, updated, ip, orport, dirport, flags):
+ self.nickname = nickname
+ self.idhash = idhash
+ self.orhash = orhash
+ self.ip = ip
+ self.orport = int(orport)
+ self.dirport = int(dirport)
+ self.flags = flags
+ self.idhex = (self.idhash + "=").decode("base64").encode("hex").upper()
+ m = re.search(r"(\d+)-(\d+)-(\d+) (\d+):(\d+):(\d+)", updated)
+ self.updated = datetime.datetime(*map(int, m.groups()))
+
+class NetworkStatusEvent:
+ def __init__(self, event_name, nslist):
+ self.event_name = event_name
+ self.arrived_at = 0
+ self.nslist = nslist # List of NetworkStatus objects
+
+class NewDescEvent:
+ def __init__(self, event_name, idlist):
+ self.event_name = event_name
+ self.arrived_at = 0
+ self.idlist = idlist
+
+class CircuitEvent:
+ def __init__(self, event_name, circ_id, status, path, reason,
+ remote_reason):
+ self.event_name = event_name
+ self.arrived_at = 0
+ self.circ_id = circ_id
+ self.status = status
+ self.path = path
+ self.reason = reason
+ self.remote_reason = remote_reason
+
+class StreamEvent:
+ def __init__(self, event_name, strm_id, status, circ_id, target_host,
+ target_port, reason, remote_reason, source, source_addr, purpose):
+ self.event_name = event_name
+ self.arrived_at = 0
+ self.strm_id = strm_id
+ self.status = status
+ self.circ_id = circ_id
+ self.target_host = target_host
+ self.target_port = int(target_port)
+ self.reason = reason
+ self.remote_reason = remote_reason
+ self.source = source
+ self.source_addr = source_addr
+ self.purpose = purpose
+
+class ORConnEvent:
+ def __init__(self, event_name, status, endpoint, age, read_bytes,
+ wrote_bytes, reason, ncircs):
+ self.event_name = event_name
+ self.arrived_at = 0
+ self.status = status
+ self.endpoint = endpoint
+ self.age = age
+ self.read_bytes = read_bytes
+ self.wrote_bytes = wrote_bytes
+ self.reason = reason
+ self.ncircs = ncircs
+
+class StreamBwEvent:
+ def __init__(self, event_name, strm_id, read, written):
+ self.event_name = event_name
+ self.strm_id = int(strm_id)
+ self.bytes_read = int(read)
+ self.bytes_written = int(written)
+
+class LogEvent:
+ def __init__(self, level, msg):
+ self.event_name = self.level = level
+ self.msg = msg
+
+class AddrMapEvent:
+ def __init__(self, event_name, from_addr, to_addr, when):
+ self.event_name = event_name
+ self.from_addr = from_addr
+ self.to_addr = to_addr
+ self.when = when
+
+class BWEvent:
+ def __init__(self, event_name, read, written):
+ self.event_name = event_name
+ self.read = read
+ self.written = written
+
+class UnknownEvent:
+ def __init__(self, event_name, event_string):
+ self.event_name = event_name
+ self.event_string = event_string
+
+class ExitPolicyLine:
+ """ Class to represent a line in a Router's exit policy in a way
+ that can be easily checked. """
+ def __init__(self, match, ip_mask, port_low, port_high):
+ self.match = match
+ if ip_mask == "*":
+ self.ip = 0
+ self.netmask = 0
+ else:
+ if not "/" in ip_mask:
+ self.netmask = 0xFFFFFFFF
+ ip = ip_mask
+ else:
+ ip, mask = ip_mask.split("/")
+ if re.match(r"\d+.\d+.\d+.\d+", mask):
+ self.netmask=struct.unpack(">I", socket.inet_aton(mask))[0]
+ else:
+ self.netmask = ~(2**(32 - int(mask)) - 1)
+ self.ip = struct.unpack(">I", socket.inet_aton(ip))[0]
+ self.ip &= self.netmask
+ if port_low == "*":
+ self.port_low,self.port_high = (0,65535)
+ else:
+ if not port_high:
+ port_high = port_low
+ self.port_low = int(port_low)
+ self.port_high = int(port_high)
+
+ def check(self, ip, port):
+ """Check to see if an ip and port is matched by this line.
+ Returns true if the line is an Accept, and False if it is a Reject. """
+ ip = struct.unpack(">I", socket.inet_aton(ip))[0]
+ if (ip & self.netmask) == self.ip:
+ if self.port_low <= port and port <= self.port_high:
+ return self.match
+ return -1
+
+class RouterVersion:
+ """ Represents a Router's version. Overloads all comparison operators
+ to check for newer, older, or equivalent versions. """
+ def __init__(self, version):
+ if version:
+ v = re.search("^(\d+).(\d+).(\d+).(\d+)", version).groups()
+ self.version = int(v[0])*0x1000000 + int(v[1])*0x10000 + int(v[2])*0x100 + int(v[3])
+ self.ver_string = version
+ else:
+ self.version = version
+ self.ver_string = "unknown"
+
+ def __lt__(self, other): return self.version < other.version
+ def __gt__(self, other): return self.version > other.version
+ def __ge__(self, other): return self.version >= other.version
+ def __le__(self, other): return self.version <= other.version
+ def __eq__(self, other): return self.version == other.version
+ def __ne__(self, other): return self.version != other.version
+ def __str__(self): return self.ver_string
+
+class Router:
+ """
+ Class to represent a router from a descriptor. Can either be
+ created from the parsed fields, or can be built from a
+ descriptor+NetworkStatus
+ """
+ def __init__(self, idhex, name, bw, down, exitpolicy, flags, ip, version, os, uptime):
+ self.idhex = idhex
+ self.nickname = name
+ self.bw = bw
+ self.exitpolicy = exitpolicy
+ self.flags = flags
+ self.down = down
+ self.ip = struct.unpack(">I", socket.inet_aton(ip))[0]
+ self.version = RouterVersion(version)
+ self.os = os
+ self.list_rank = 0 # position in a sorted list of routers.
+ self.uptime = uptime
+
+ def __str__(self):
+ s = self.idhex, self.nickname
+ return s.__str__()
+
+ def build_from_desc(desc, ns):
+ """
+ Static method of Router that parses a descriptor string into this class.
+ 'desc' is a full descriptor as a string.
+ 'ns' is a TorCtl.NetworkStatus instance for this router (needed for
+ the flags, the nickname, and the idhex string).
+ Returns a Router instance.
+ """
+ # XXX: Compile these regular expressions? This is an expensive process
+ # Use http://docs.python.org/lib/profile.html to verify this is
+ # the part of startup that is slow
+ exitpolicy = []
+ dead = not ("Running" in ns.flags)
+ bw_observed = 0
+ version = None
+ os = None
+ uptime = 0
+ ip = 0
+ router = "[none]"
+
+ for line in desc:
+ rt = re.search(r"^router (\S+) (\S+)", line)
+ fp = re.search(r"^opt fingerprint (.+).*on (\S+)", line)
+ pl = re.search(r"^platform Tor (\S+).*on (\S+)", line)
+ ac = re.search(r"^accept (\S+):([^-]+)(?:-(\d+))?", line)
+ rj = re.search(r"^reject (\S+):([^-]+)(?:-(\d+))?", line)
+ bw = re.search(r"^bandwidth \d+ \d+ (\d+)", line)
+ up = re.search(r"^uptime (\d+)", line)
+ if re.search(r"^opt hibernating 1", line):
+ #dead = 1 # XXX: Technically this may be stale..
+ if ("Running" in ns.flags):
+ plog("INFO", "Hibernating router "+ns.nickname+" is running..")
+ if ac:
+ exitpolicy.append(ExitPolicyLine(True, *ac.groups()))
+ elif rj:
+ exitpolicy.append(ExitPolicyLine(False, *rj.groups()))
+ elif bw:
+ bw_observed = int(bw.group(1))
+ elif pl:
+ version, os = pl.groups()
+ elif up:
+ uptime = int(up.group(1))
+ elif rt:
+ router,ip = rt.groups()
+ if router != ns.nickname:
+ plog("NOTICE", "Got different names " + ns.nickname + " vs " +
+ router + " for " + ns.idhex)
+ if not bw_observed and not dead and ("Valid" in ns.flags):
+ plog("INFO", "No bandwidth for live router " + ns.nickname)
+ if not version or not os:
+ plog("INFO", "No version and/or OS for router " + ns.nickname)
+ return Router(ns.idhex, ns.nickname, bw_observed, dead, exitpolicy,
+ ns.flags, ip, version, os, uptime)
+ build_from_desc = Callable(build_from_desc)
+
+ def update_to(self, new):
+ """ Somewhat hackish method to update this router to be a copy of
+ 'new' """
+ if self.idhex != new.idhex:
+ plog("ERROR", "Update of router "+self.nickname+"changes idhex!")
+ self.idhex = new.idhex
+ self.nickname = new.nickname
+ self.bw = new.bw
+ self.exitpolicy = new.exitpolicy
+ self.flags = new.flags
+ self.ip = new.ip
+ self.version = new.version
+ self.os = new.os
+ self.uptime = new.uptime
+
+ def will_exit_to(self, ip, port):
+ """ Check the entire exitpolicy to see if the router will allow
+ connections to 'ip':'port' """
+ for line in self.exitpolicy:
+ ret = line.check(ip, port)
+ if ret != -1:
+ return ret
+ plog("WARN", "No matching exit line for "+self.nickname)
+ return False
+
+class Connection:
+ """A Connection represents a connection to the Tor process via the
+ control port."""
+ def __init__(self, sock):
+ """Create a Connection to communicate with the Tor process over the
+ socket 'sock'.
+ """
+ self._handler = None
+ self._handleFn = None
+ self._sendLock = threading.RLock()
+ self._queue = Queue.Queue()
+ self._thread = None
+ self._closedEx = None
+ self._closed = 0
+ self._closeHandler = None
+ self._eventThread = None
+ self._eventQueue = Queue.Queue()
+ self._s = BufSock(sock)
+ self._debugFile = None
+
+ def set_close_handler(self, handler):
+ """Call 'handler' when the Tor process has closed its connection or
+ given us an exception. If we close normally, no arguments are
+ provided; otherwise, it will be called with an exception as its
+ argument.
+ """
+ self._closeHandler = handler
+
+ def close(self):
+ """Shut down this controller connection"""
+ self._sendLock.acquire()
+ try:
+ self._queue.put("CLOSE")
+ self._eventQueue.put((time.time(), "CLOSE"))
+ finally:
+ self._sendLock.release()
+
+ def launch_thread(self, daemon=1):
+ """Launch a background thread to handle messages from the Tor process."""
+ assert self._thread is None
+ t = threading.Thread(target=self._loop)
+ if daemon:
+ t.setDaemon(daemon)
+ t.start()
+ self._thread = t
+ t = threading.Thread(target=self._eventLoop)
+ if daemon:
+ t.setDaemon(daemon)
+ t.start()
+ self._eventThread = t
+ return self._thread
+
+ def _loop(self):
+ """Main subthread loop: Read commands from Tor, and handle them either
+ as events or as responses to other commands.
+ """
+ while 1:
+ try:
+ isEvent, reply = self._read_reply()
+ except:
+ self._err(sys.exc_info())
+ return
+
+ if isEvent:
+ if self._handler is not None:
+ self._eventQueue.put((time.time(), reply))
+ else:
+ cb = self._queue.get() # atomic..
+ if cb == "CLOSE":
+ self._s.close()
+ self._s = None
+ self._closed = 1
+ return
+ else:
+ cb(reply)
+
+ def _err(self, (tp, ex, tb), fromEventLoop=0):
+ """DOCDOC"""
+ # silent death is bad :(
+ traceback.print_exception(tp, ex, tb)
+ if self._s:
+ try:
+ self.close()
+ except:
+ pass
+ self._sendLock.acquire()
+ try:
+ self._closedEx = ex
+ self._closed = 1
+ finally:
+ self._sendLock.release()
+ while 1:
+ try:
+ cb = self._queue.get(timeout=0)
+ if cb != "CLOSE":
+ cb("EXCEPTION")
+ except Queue.Empty:
+ break
+ if self._closeHandler is not None:
+ self._closeHandler(ex)
+ return
+
+ def _eventLoop(self):
+ """DOCDOC"""
+ while 1:
+ (timestamp, reply) = self._eventQueue.get()
+ if reply[0][0] == "650" and reply[0][1] == "OK":
+ plog("DEBUG", "Ignoring incompatible syntactic sugar: 650 OK")
+ continue
+ if reply == "CLOSE":
+ return
+ try:
+ self._handleFn(timestamp, reply)
+ except:
+ for code, msg, data in reply:
+ plog("WARN", "No event for: "+str(code)+" "+str(msg))
+ self._err(sys.exc_info(), 1)
+ return
+
+ def _sendImpl(self, sendFn, msg):
+ """DOCDOC"""
+ if self._thread is None:
+ self.launch_thread(1)
+ # This condition will get notified when we've got a result...
+ condition = threading.Condition()
+ # Here's where the result goes...
+ result = []
+
+ if self._closedEx is not None:
+ raise self._closedEx
+ elif self._closed:
+ raise TorCtlClosed()
+
+ def cb(reply,condition=condition,result=result):
+ condition.acquire()
+ try:
+ result.append(reply)
+ condition.notify()
+ finally:
+ condition.release()
+
+ # Sends a message to Tor...
+ self._sendLock.acquire() # ensure queue+sendmsg is atomic
+ try:
+ self._queue.put(cb)
+ sendFn(msg) # _doSend(msg)
+ finally:
+ self._sendLock.release()
+
+ # Now wait till the answer is in...
+ condition.acquire()
+ try:
+ while not result:
+ condition.wait()
+ finally:
+ condition.release()
+
+ # ...And handle the answer appropriately.
+ assert len(result) == 1
+ reply = result[0]
+ if reply == "EXCEPTION":
+ raise self._closedEx
+
+ return reply
+
+
+ def debug(self, f):
+ """DOCDOC"""
+ self._debugFile = f
+
+ def set_event_handler(self, handler):
+ """Cause future events from the Tor process to be sent to 'handler'.
+ """
+ self._handler = handler
+ self._handleFn = handler._handle1
+
+ def _read_reply(self):
+ lines = []
+ while 1:
+ line = self._s.readline().strip()
+ if self._debugFile:
+ self._debugFile.write(" %s\n" % line)
+ if len(line)<4:
+ raise ProtocolError("Badly formatted reply line: Too short")
+ code = line[:3]
+ tp = line[3]
+ s = line[4:]
+ if tp == "-":
+ lines.append((code, s, None))
+ elif tp == " ":
+ lines.append((code, s, None))
+ isEvent = (lines and lines[0][0][0] == '6')
+ return isEvent, lines
+ elif tp != "+":
+ raise ProtocolError("Badly formatted reply line: unknown type %r"%tp)
+ else:
+ more = []
+ while 1:
+ line = self._s.readline()
+ if self._debugFile:
+ self._debugFile.write("+++ %s" % line)
+ if line in (".\r\n", ".\n", "650 OK\n", "650 OK\r\n"):
+ break
+ more.append(line)
+ lines.append((code, s, unescape_dots("".join(more))))
+ isEvent = (lines and lines[0][0][0] == '6')
+ if isEvent: # Need "250 OK" if it's not an event. Otherwise, end
+ return (isEvent, lines)
+
+ # Notreached
+ raise TorCtlError()
+
+ def _doSend(self, msg):
+ if self._debugFile:
+ amsg = msg
+ lines = amsg.split("\n")
+ if len(lines) > 2:
+ amsg = "\n".join(lines[:2]) + "\n"
+ self._debugFile.write(">>> %s" % amsg)
+ self._s.write(msg)
+
+ def sendAndRecv(self, msg="", expectedTypes=("250", "251")):
+ """Helper: Send a command 'msg' to Tor, and wait for a command
+ in response. If the response type is in expectedTypes,
+ return a list of (tp,body,extra) tuples. If it is an
+ error, raise ErrorReply. Otherwise, raise ProtocolError.
+ """
+ if type(msg) == types.ListType:
+ msg = "".join(msg)
+ assert msg.endswith("\r\n")
+
+ lines = self._sendImpl(self._doSend, msg)
+ # print lines
+ for tp, msg, _ in lines:
+ if tp[0] in '45':
+ raise ErrorReply("%s %s"%(tp, msg))
+ if tp not in expectedTypes:
+ raise ProtocolError("Unexpectd message type %r"%tp)
+
+ return lines
+
+ def authenticate(self, secret=""):
+ """Send an authenticating secret to Tor. You'll need to call this
+ method before Tor can start.
+ """
+ #hexstr = binascii.b2a_hex(secret)
+ self.sendAndRecv("AUTHENTICATE \"%s\"\r\n"%secret)
+
+ def get_option(self, name):
+ """Get the value of the configuration option named 'name'. To
+ retrieve multiple values, pass a list for 'name' instead of
+ a string. Returns a list of (key,value) pairs.
+ Refer to section 3.3 of control-spec.txt for a list of valid names.
+ """
+ if not isinstance(name, str):
+ name = " ".join(name)
+ lines = self.sendAndRecv("GETCONF %s\r\n" % name)
+
+ r = []
+ for _,line,_ in lines:
+ try:
+ key, val = line.split("=", 1)
+ r.append((key,val))
+ except ValueError:
+ r.append((line, None))
+
+ return r
+
+ def set_option(self, key, value):
+ """Set the value of the configuration option 'key' to the value 'value'.
+ """
+ self.set_options([(key, value)])
+
+ def set_options(self, kvlist):
+ """Given a list of (key,value) pairs, set them as configuration
+ options.
+ """
+ if not kvlist:
+ return
+ msg = " ".join(["%s=%s"%(k,quote(v)) for k,v in kvlist])
+ self.sendAndRecv("SETCONF %s\r\n"%msg)
+
+ def reset_options(self, keylist):
+ """Reset the options listed in 'keylist' to their default values.
+
+ Tor started implementing this command in version 0.1.1.7-alpha;
+ previous versions wanted you to set configuration keys to "".
+ That no longer works.
+ """
+ self.sendAndRecv("RESETCONF %s\r\n"%(" ".join(keylist)))
+
+ def get_network_status(self, who="all"):
+ """Get the entire network status list. Returns a list of
+ TorCtl.NetworkStatus instances."""
+ return parse_ns_body(self.sendAndRecv("GETINFO ns/"+who+"\r\n")[0][2])
+
+ def get_router(self, ns):
+ """Fill in a Router class corresponding to a given NS class"""
+ desc = self.sendAndRecv("GETINFO desc/id/" + ns.idhex + "\r\n")[0][2].split("\n")
+ return Router.build_from_desc(desc, ns)
+
+
+ def read_routers(self, nslist):
+ """ Given a list a NetworkStatuses in 'nslist', this function will
+ return a list of new Router instances.
+ """
+ bad_key = 0
+ new = []
+ for ns in nslist:
+ try:
+ r = self.get_router(ns)
+ new.append(r)
+ except ErrorReply:
+ bad_key += 1
+ if "Running" in ns.flags:
+ plog("NOTICE", "Running router "+ns.nickname+"="
+ +ns.idhex+" has no descriptor")
+ except:
+ traceback.print_exception(*sys.exc_info())
+ continue
+
+ return new
+
+ def get_info(self, name):
+ """Return the value of the internal information field named 'name'.
+ Refer to section 3.9 of control-spec.txt for a list of valid names.
+ DOCDOC
+ """
+ if not isinstance(name, str):
+ name = " ".join(name)
+ lines = self.sendAndRecv("GETINFO %s\r\n"%name)
+ d = {}
+ for _,msg,more in lines:
+ if msg == "OK":
+ break
+ try:
+ k,rest = msg.split("=",1)
+ except ValueError:
+ raise ProtocolError("Bad info line %r",msg)
+ if more:
+ d[k] = more
+ else:
+ d[k] = rest
+ return d
+
+ def set_events(self, events, extended=False):
+ """Change the list of events that the event handler is interested
+ in to those in 'events', which is a list of event names.
+ Recognized event names are listed in section 3.3 of the control-spec
+ """
+ if extended:
+ plog ("DEBUG", "SETEVENTS EXTENDED %s\r\n" % " ".join(events))
+ self.sendAndRecv("SETEVENTS EXTENDED %s\r\n" % " ".join(events))
+ else:
+ self.sendAndRecv("SETEVENTS %s\r\n" % " ".join(events))
+
+ def save_conf(self):
+ """Flush all configuration changes to disk.
+ """
+ self.sendAndRecv("SAVECONF\r\n")
+
+ def send_signal(self, sig):
+ """Send the signal 'sig' to the Tor process; The allowed values for
+ 'sig' are listed in section 3.6 of control-spec.
+ """
+ sig = { 0x01 : "HUP",
+ 0x02 : "INT",
+ 0x03 : "NEWNYM",
+ 0x0A : "USR1",
+ 0x0C : "USR2",
+ 0x0F : "TERM" }.get(sig,sig)
+ self.sendAndRecv("SIGNAL %s\r\n"%sig)
+
+ def resolve(self, host):
+ """ Launch a remote hostname lookup request:
+ 'host' may be a hostname or IPv4 address
+ """
+ # TODO: handle "mode=reverse"
+ self.sendAndRecv("RESOLVE %s\r\n"%host)
+
+ def map_address(self, kvList):
+ """ Sends the MAPADDRESS command for each of the tuples in kvList """
+ if not kvList:
+ return
+ m = " ".join([ "%s=%s" for k,v in kvList])
+ lines = self.sendAndRecv("MAPADDRESS %s\r\n"%m)
+ r = []
+ for _,line,_ in lines:
+ try:
+ key, val = line.split("=", 1)
+ except ValueError:
+ raise ProtocolError("Bad address line %r",v)
+ r.append((key,val))
+ return r
+
+ def extend_circuit(self, circid, hops):
+ """Tell Tor to extend the circuit identified by 'circid' through the
+ servers named in the list 'hops'.
+ """
+ if circid is None:
+ circid = "0"
+ plog("DEBUG", "Extending circuit")
+ lines = self.sendAndRecv("EXTENDCIRCUIT %d %s\r\n"
+ %(circid, ",".join(hops)))
+ tp,msg,_ = lines[0]
+ m = re.match(r'EXTENDED (\S*)', msg)
+ if not m:
+ raise ProtocolError("Bad extended line %r",msg)
+ plog("DEBUG", "Circuit extended")
+ return int(m.group(1))
+
+ def redirect_stream(self, streamid, newaddr, newport=""):
+ """DOCDOC"""
+ if newport:
+ self.sendAndRecv("REDIRECTSTREAM %d %s %s\r\n"%(streamid, newaddr, newport))
+ else:
+ self.sendAndRecv("REDIRECTSTREAM %d %s\r\n"%(streamid, newaddr))
+
+ def attach_stream(self, streamid, circid, hop=None):
+ """Attach a stream to a circuit, specify both by IDs. If hop is given,
+ try to use the specified hop in the circuit as the exit node for
+ this stream.
+ """
+ if hop:
+ self.sendAndRecv("ATTACHSTREAM %d %d HOP=%d\r\n"%(streamid, circid, hop))
+ plog("DEBUG", "Attaching stream: "+str(streamid)+" to hop "+str(hop)+" of circuit "+str(circid))
+ else:
+ self.sendAndRecv("ATTACHSTREAM %d %d\r\n"%(streamid, circid))
+ plog("DEBUG", "Attaching stream: "+str(streamid)+" to circuit "+str(circid))
+
+ def close_stream(self, streamid, reason=0, flags=()):
+ """DOCDOC"""
+ self.sendAndRecv("CLOSESTREAM %d %s %s\r\n"
+ %(streamid, reason, "".join(flags)))
+
+ def close_circuit(self, circid, reason=0, flags=()):
+ """DOCDOC"""
+ self.sendAndRecv("CLOSECIRCUIT %d %s %s\r\n"
+ %(circid, reason, "".join(flags)))
+
+ def post_descriptor(self, desc):
+ self.sendAndRecv("+POSTDESCRIPTOR\r\n%s"%escape_dots(desc))
+
+def parse_ns_body(data):
+ """Parse the body of an NS event or command into a list of
+ NetworkStatus instances"""
+ nsgroups = re.compile(r"^r ", re.M).split(data)
+ nsgroups.pop(0)
+ nslist = []
+ for nsline in nsgroups:
+ m = re.search(r"^s((?:\s\S*)+)", nsline, re.M)
+ flags = m.groups()
+ flags = flags[0].strip().split(" ")
+ m = re.match(r"(\S+)\s(\S+)\s(\S+)\s(\S+\s\S+)\s(\S+)\s(\d+)\s(\d+)", nsline)
+ nslist.append(NetworkStatus(*(m.groups() + (flags,))))
+ return nslist
+
+class EventHandler:
+ """An 'EventHandler' wraps callbacks for the events Tor can return.
+ Each event argument is an instance of the corresponding event
+ class."""
+ def __init__(self):
+ """Create a new EventHandler."""
+ self._map1 = {
+ "CIRC" : self.circ_status_event,
+ "STREAM" : self.stream_status_event,
+ "ORCONN" : self.or_conn_status_event,
+ "STREAM_BW" : self.stream_bw_event,
+ "BW" : self.bandwidth_event,
+ "DEBUG" : self.msg_event,
+ "INFO" : self.msg_event,
+ "NOTICE" : self.msg_event,
+ "WARN" : self.msg_event,
+ "ERR" : self.msg_event,
+ "NEWDESC" : self.new_desc_event,
+ "ADDRMAP" : self.address_mapped_event,
+ "NS" : self.ns_event
+ }
+
+ def _handle1(self, timestamp, lines):
+ """Dispatcher: called from Connection when an event is received."""
+ for code, msg, data in lines:
+ event = self._decode1(msg, data)
+ event.arrived_at = timestamp
+ self.heartbeat_event(event)
+ self._map1.get(event.event_name, self.unknown_event)(event)
+
+ def _decode1(self, body, data):
+ """Unpack an event message into a type/arguments-tuple tuple."""
+ if " " in body:
+ evtype,body = body.split(" ",1)
+ else:
+ evtype,body = body,""
+ evtype = evtype.upper()
+ if evtype == "CIRC":
+ m = re.match(r"(\d+)\s+(\S+)(\s\S+)?(\s\S+)?(\s\S+)?", body)
+ if not m:
+ raise ProtocolError("CIRC event misformatted.")
+ ident,status,path,reason,remote = m.groups()
+ ident = int(ident)
+ if path:
+ if "REASON=" in path:
+ remote = reason
+ reason = path
+ path=[]
+ else:
+ path = path.strip().split(",")
+ else:
+ path = []
+ if reason: reason = reason[8:]
+ if remote: remote = remote[15:]
+ event = CircuitEvent(evtype, ident, status, path, reason, remote)
+ elif evtype == "STREAM":
+ #plog("DEBUG", "STREAM: "+body)
+ m = re.match(r"(\S+)\s+(\S+)\s+(\S+)\s+(\S+):(\d+)(\sREASON=\S+)?(\sREMOTE_REASON=\S+)?(\sSOURCE=\S+)?(\sSOURCE_ADDR=\S+)?(\sPURPOSE=\S+)?", body)
+ if not m:
+ raise ProtocolError("STREAM event misformatted.")
+ ident,status,circ,target_host,target_port,reason,remote,source,source_addr,purpose = m.groups()
+ ident,circ = map(int, (ident,circ))
+ if reason: reason = reason[8:]
+ if remote: remote = remote[15:]
+ if source: source = source[8:]
+ if source_addr: source_addr = source_addr[13:]
+ if purpose: purpose = purpose[9:]
+ event = StreamEvent(evtype, ident, status, circ, target_host,
+ int(target_port), reason, remote, source, source_addr, purpose)
+ elif evtype == "ORCONN":
+ m = re.match(r"(\S+)\s+(\S+)(\sAGE=\S+)?(\sREAD=\S+)?(\sWRITTEN=\S+)?(\sREASON=\S+)?(\sNCIRCS=\S+)?", body)
+ if not m:
+ raise ProtocolError("ORCONN event misformatted.")
+ target, status, age, read, wrote, reason, ncircs = m.groups()
+
+ #plog("DEBUG", "ORCONN: "+body)
+ if ncircs: ncircs = int(ncircs[8:])
+ else: ncircs = 0
+ if reason: reason = reason[8:]
+ if age: age = int(age[5:])
+ else: age = 0
+ if read: read = int(read[6:])
+ else: read = 0
+ if wrote: wrote = int(wrote[9:])
+ else: wrote = 0
+ event = ORConnEvent(evtype, status, target, age, read, wrote,
+ reason, ncircs)
+ elif evtype == "STREAM_BW":
+ m = re.match(r"(\d+)\s+(\d+)\s+(\d+)", body)
+ if not m:
+ raise ProtocolError("STREAM_BW event misformatted.")
+ event = StreamBwEvent(evtype, *m.groups())
+ elif evtype == "BW":
+ m = re.match(r"(\d+)\s+(\d+)", body)
+ if not m:
+ raise ProtocolError("BANDWIDTH event misformatted.")
+ read, written = map(long, m.groups())
+ event = BWEvent(evtype, read, written)
+ elif evtype in ("DEBUG", "INFO", "NOTICE", "WARN", "ERR"):
+ event = LogEvent(evtype, body)
+ elif evtype == "NEWDESC":
+ event = NewDescEvent(evtype, body.split(" "))
+ elif evtype == "ADDRMAP":
+ # TODO: Also parse errors and GMTExpiry
+ m = re.match(r'(\S+)\s+(\S+)\s+(\"[^"]+\"|\w+)', body)
+ if not m:
+ raise ProtocolError("ADDRMAP event misformatted.")
+ fromaddr, toaddr, when = m.groups()
+ if when.upper() == "NEVER":
+ when = None
+ else:
+ when = time.strptime(when[1:-1], "%Y-%m-%d %H:%M:%S")
+ event = AddrMapEvent(evtype, fromaddr, toaddr, when)
+ elif evtype == "NS":
+ event = NetworkStatusEvent(evtype, parse_ns_body(data))
+ else:
+ event = UnknownEvent(evtype, body)
+
+ return event
+
+ def heartbeat_event(self, event):
+ """Called before any event is recieved. Convenience function
+ for any cleanup/setup/reconfiguration you may need to do.
+ """
+ pass
+
+ def unknown_event(self, event):
+ """Called when we get an event type we don't recognize. This
+ is almost alwyas an error.
+ """
+ raise NotImplemented()
+
+ def circ_status_event(self, event):
+ """Called when a circuit status changes if listening to CIRCSTATUS
+ events."""
+ raise NotImplemented()
+
+ def stream_status_event(self, event):
+ """Called when a stream status changes if listening to STREAMSTATUS
+ events. """
+ raise NotImplemented()
+
+ def stream_bw_event(self, event):
+ raise NotImplemented()
+
+ def or_conn_status_event(self, event):
+ """Called when an OR connection's status changes if listening to
+ ORCONNSTATUS events."""
+ raise NotImplemented()
+
+ def bandwidth_event(self, event):
+ """Called once a second if listening to BANDWIDTH events.
+ """
+ raise NotImplemented()
+
+ def new_desc_event(self, event):
+ """Called when Tor learns a new server descriptor if listenting to
+ NEWDESC events.
+ """
+ raise NotImplemented()
+
+ def msg_event(self, event):
+ """Called when a log message of a given severity arrives if listening
+ to INFO_MSG, NOTICE_MSG, WARN_MSG, or ERR_MSG events."""
+ raise NotImplemented()
+
+ def ns_event(self, event):
+ raise NotImplemented()
+
+ def address_mapped_event(self, event):
+ """Called when Tor adds a mapping for an address if listening
+ to ADDRESSMAPPED events.
+ """
+ raise NotImplemented()
+
+
+class DebugEventHandler(EventHandler):
+ """Trivial debug event handler: reassembles all parsed events to stdout."""
+ def circ_status_event(self, circ_event): # CircuitEvent()
+ output = [circ_event.event_name, str(circ_event.circ_id),
+ circ_event.status]
+ if circ_event.path:
+ output.append(",".join(circ_event.path))
+ if circ_event.reason:
+ output.append("REASON=" + circ_event.reason)
+ if circ_event.remote_reason:
+ output.append("REMOTE_REASON=" + circ_event.remote_reason)
+ print " ".join(output)
+
+ def stream_status_event(self, strm_event):
+ output = [strm_event.event_name, str(strm_event.strm_id),
+ strm_event.status, str(strm_event.circ_id),
+ strm_event.target_host, str(strm_event.target_port)]
+ if strm_event.reason:
+ output.append("REASON=" + strm_event.reason)
+ if strm_event.remote_reason:
+ output.append("REMOTE_REASON=" + strm_event.remote_reason)
+ print " ".join(output)
+
+ def ns_event(self, ns_event):
+ for ns in ns_event.nslist:
+ print " ".join((ns_event.event_name, ns.nickname, ns.idhash,
+ ns.updated.isoformat(), ns.ip, str(ns.orport),
+ str(ns.dirport), " ".join(ns.flags)))
+
+ def new_desc_event(self, newdesc_event):
+ print " ".join((newdesc_event.event_name, " ".join(newdesc_event.idlist)))
+
+ def or_conn_status_event(self, orconn_event):
+ if orconn_event.age: age = "AGE="+str(orconn_event.age)
+ else: age = ""
+ if orconn_event.read_bytes: read = "READ="+str(orconn_event.read_bytes)
+ else: read = ""
+ if orconn_event.wrote_bytes: wrote = "WRITTEN="+str(orconn_event.wrote_bytes)
+ else: wrote = ""
+ if orconn_event.reason: reason = "REASON="+orconn_event.reason
+ else: reason = ""
+ if orconn_event.ncircs: ncircs = "NCIRCS="+str(orconn_event.ncircs)
+ else: ncircs = ""
+ print " ".join((orconn_event.event_name, orconn_event.endpoint,
+ orconn_event.status, age, read, wrote, reason, ncircs))
+
+ def msg_event(self, log_event):
+ print log_event.event_name+" "+log_event.msg
+
+ def bandwidth_event(self, bw_event):
+ print bw_event.event_name+" "+str(bw_event.read)+" "+str(bw_event.written)
+
+def parseHostAndPort(h):
+ """Given a string of the form 'address:port' or 'address' or
+ 'port' or '', return a two-tuple of (address, port)
+ """
+ host, port = "localhost", 9100
+ if ":" in h:
+ i = h.index(":")
+ host = h[:i]
+ try:
+ port = int(h[i+1:])
+ except ValueError:
+ print "Bad hostname %r"%h
+ sys.exit(1)
+ elif h:
+ try:
+ port = int(h)
+ except ValueError:
+ host = h
+
+ return host, port
+
+def run_example(host,port):
+ """ Example of basic TorCtl usage. See PathSupport for more advanced
+ usage.
+ """
+ print "host is %s:%d"%(host,port)
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ s.connect((host,port))
+ c = Connection(s)
+ c.set_event_handler(DebugEventHandler())
+ th = c.launch_thread()
+ c.authenticate()
+ print "nick",`c.get_option("nickname")`
+ print `c.get_info("version")`
+ #print `c.get_info("desc/name/moria1")`
+ print `c.get_info("network-status")`
+ print `c.get_info("addr-mappings/all")`
+ print `c.get_info("addr-mappings/config")`
+ print `c.get_info("addr-mappings/cache")`
+ print `c.get_info("addr-mappings/control")`
+
+ print `c.extend_circuit(0,["moria1"])`
+ try:
+ print `c.extend_circuit(0,[""])`
+ except ErrorReply: # wtf?
+ print "got error. good."
+ except:
+ print "Strange error", sys.exc_info()[0]
+
+ #send_signal(s,1)
+ #save_conf(s)
+
+ #set_option(s,"1")
+ #set_option(s,"bandwidthburstbytes 100000")
+ #set_option(s,"runasdaemon 1")
+ #set_events(s,[EVENT_TYPE.WARN])
+# c.set_events([EVENT_TYPE.ORCONN], True)
+ c.set_events([EVENT_TYPE.STREAM, EVENT_TYPE.CIRC,
+ EVENT_TYPE.NS, EVENT_TYPE.NEWDESC,
+ EVENT_TYPE.ORCONN, EVENT_TYPE.BW], True)
+
+ th.join()
+ return
+
+if __name__ == '__main__':
+ if len(sys.argv) > 2:
+ print "Syntax: TorControl.py torhost:torport"
+ sys.exit(0)
+ else:
+ sys.argv.append("localhost:9051")
+ sh,sp = parseHostAndPort(sys.argv[1])
+ run_example(sh,sp)
+
Deleted: torflow/branches/gsoc2008/TorCtl.local/TorUtil.py
===================================================================
--- torflow/branches/gsoc2008/TorCtl/TorUtil.py 2008-07-07 16:40:57 UTC (rev 15734)
+++ torflow/branches/gsoc2008/TorCtl.local/TorUtil.py 2008-07-07 19:37:47 UTC (rev 15742)
@@ -1,260 +0,0 @@
-#!/usr/bin/python
-# TorCtl.py -- Python module to interface with Tor Control interface.
-# Copyright 2007 Mike Perry -- See LICENSE for licensing information.
-# Portions Copyright 2005 Nick Matthewson
-
-"""
-TorUtil -- Support functions for TorCtl.py and metatroller
-"""
-
-import os
-import re
-import sys
-import socket
-import binascii
-import sha
-import math
-import time
-
-__all__ = ["Enum", "Enum2", "Callable", "sort_list", "quote", "escape_dots", "unescape_dots",
- "BufSock", "secret_to_key", "urandom_rng", "s2k_gen", "s2k_check", "plog",
- "ListenSocket", "zprob"]
-
-# TODO: Make functions to read these from a config file. This isn't
-# the right place for them either.. But at least it's unified.
-tor_port = 9050
-tor_host = '127.0.0.1'
-
-meta_port = 9052
-meta_host = '127.0.0.1'
-
-control_port = 9051
-control_host = '127.0.0.1'
-
-class Enum:
- """ Defines an ordered dense name-to-number 1-1 mapping """
- def __init__(self, start, names):
- self.nameOf = {}
- idx = start
- for name in names:
- setattr(self,name,idx)
- self.nameOf[idx] = name
- idx += 1
-
-class Enum2:
- """ Defines an ordered sparse name-to-number 1-1 mapping """
- def __init__(self, **args):
- self.__dict__.update(args)
- self.nameOf = {}
- for k,v in args.items():
- self.nameOf[v] = k
-
-class Callable:
- def __init__(self, anycallable):
- self.__call__ = anycallable
-
-def sort_list(list, key):
- """ Sort a list by a specified key """
- list.sort(lambda x,y: cmp(key(x), key(y))) # Python < 2.4 hack
- return list
-
-def quote(s):
- return re.sub(r'([\r\n\\\"])', r'\\\1', s)
-
-def escape_dots(s, translate_nl=1):
- if translate_nl:
- lines = re.split(r"\r?\n", s)
- else:
- lines = s.split("\r\n")
- if lines and not lines[-1]:
- del lines[-1]
- for i in xrange(len(lines)):
- if lines[i].startswith("."):
- lines[i] = "."+lines[i]
- lines.append(".\r\n")
- return "\r\n".join(lines)
-
-def unescape_dots(s, translate_nl=1):
- lines = s.split("\r\n")
-
- for i in xrange(len(lines)):
- if lines[i].startswith("."):
- lines[i] = lines[i][1:]
-
- if lines and lines[-1]:
- lines.append("")
-
- if translate_nl:
- return "\n".join(lines)
- else:
- return "\r\n".join(lines)
-
-# XXX: Exception handling
-class BufSock:
- def __init__(self, s):
- self._s = s
- self._buf = []
-
- def readline(self):
- if self._buf:
- idx = self._buf[0].find('\n')
- if idx >= 0:
- result = self._buf[0][:idx+1]
- self._buf[0] = self._buf[0][idx+1:]
- return result
-
- while 1:
- s = self._s.recv(128)
- if not s: return None
- # XXX: This really does need an exception
- # raise ConnectionClosed()
- idx = s.find('\n')
- if idx >= 0:
- self._buf.append(s[:idx+1])
- result = "".join(self._buf)
- rest = s[idx+1:]
- if rest:
- self._buf = [ rest ]
- else:
- del self._buf[:]
- return result
- else:
- self._buf.append(s)
-
- def write(self, s):
- self._s.send(s)
-
- def close(self):
- self._s.close()
-
-# SocketServer.TCPServer is nuts..
-class ListenSocket:
- def __init__(self, listen_ip, port):
- msg = None
- self.s = None
- for res in socket.getaddrinfo(listen_ip, port, socket.AF_UNSPEC,
- socket.SOCK_STREAM, 0, socket.AI_PASSIVE):
- af, socktype, proto, canonname, sa = res
- try:
- self.s = socket.socket(af, socktype, proto)
- self.s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
- except socket.error, msg:
- self.s = None
- continue
- try:
- self.s.bind(sa)
- self.s.listen(1)
- except socket.error, msg:
- self.s.close()
- self.s = None
- continue
- break
- if self.s is None:
- raise socket.error(msg)
-
- def accept(self):
- conn, addr = self.s.accept()
- return conn
-
- def close(self):
- self.s.close()
-
-
-def secret_to_key(secret, s2k_specifier):
- """Used to generate a hashed password string. DOCDOC."""
- c = ord(s2k_specifier[8])
- EXPBIAS = 6
- count = (16+(c&15)) << ((c>>4) + EXPBIAS)
-
- d = sha.new()
- tmp = s2k_specifier[:8]+secret
- slen = len(tmp)
- while count:
- if count > slen:
- d.update(tmp)
- count -= slen
- else:
- d.update(tmp[:count])
- count = 0
- return d.digest()
-
-def urandom_rng(n):
- """Try to read some entropy from the platform entropy source."""
- f = open('/dev/urandom', 'rb')
- try:
- return f.read(n)
- finally:
- f.close()
-
-def s2k_gen(secret, rng=None):
- """DOCDOC"""
- if rng is None:
- if hasattr(os, "urandom"):
- rng = os.urandom
- else:
- rng = urandom_rng
- spec = "%s%s"%(rng(8), chr(96))
- return "16:%s"%(
- binascii.b2a_hex(spec + secret_to_key(secret, spec)))
-
-def s2k_check(secret, k):
- """DOCDOC"""
- assert k[:3] == "16:"
-
- k = binascii.a2b_hex(k[3:])
- return secret_to_key(secret, k[:9]) == k[9:]
-
-
-## XXX: Make this a class?
-loglevel = "DEBUG"
-loglevels = {"DEBUG" : 0, "INFO" : 1, "NOTICE" : 2, "WARN" : 3, "ERROR" : 4}
-
-def plog(level, msg): # XXX: Timestamps
- if(loglevels[level] >= loglevels[loglevel]):
- t = time.strftime("%a %b %d %H:%M:%S %Y")
- print level, '[', t, ']:', msg
- sys.stdout.flush()
-
-# Stolen from
-# http://www.nmr.mgh.harvard.edu/Neural_Systems_Group/gary/python/stats.py
-def zprob(z):
- """
-Returns the area under the normal curve 'to the left of' the given z value.
-Thus,
- for z<0, zprob(z) = 1-tail probability
- for z>0, 1.0-zprob(z) = 1-tail probability
- for any z, 2.0*(1.0-zprob(abs(z))) = 2-tail probability
-Adapted from z.c in Gary Perlman's |Stat.
-
-Usage: lzprob(z)
-"""
- Z_MAX = 6.0 # maximum meaningful z-value
- if z == 0.0:
- x = 0.0
- else:
- y = 0.5 * math.fabs(z)
- if y >= (Z_MAX*0.5):
- x = 1.0
- elif (y < 1.0):
- w = y*y
- x = ((((((((0.000124818987 * w
- -0.001075204047) * w +0.005198775019) * w
- -0.019198292004) * w +0.059054035642) * w
- -0.151968751364) * w +0.319152932694) * w
- -0.531923007300) * w +0.797884560593) * y * 2.0
- else:
- y = y - 2.0
- x = (((((((((((((-0.000045255659 * y
- +0.000152529290) * y -0.000019538132) * y
- -0.000676904986) * y +0.001390604284) * y
- -0.000794620820) * y -0.002034254874) * y
- +0.006549791214) * y -0.010557625006) * y
- +0.011630447319) * y -0.009279453341) * y
- +0.005353579108) * y -0.002141268741) * y
- +0.000535310849) * y +0.999936657524
- if z > 0.0:
- prob = ((x+1.0)*0.5)
- else:
- prob = ((1.0-x)*0.5)
- return prob
-
Copied: torflow/branches/gsoc2008/TorCtl.local/TorUtil.py (from rev 15741, torflow/branches/gsoc2008/TorCtl/TorUtil.py)
===================================================================
--- torflow/branches/gsoc2008/TorCtl.local/TorUtil.py (rev 0)
+++ torflow/branches/gsoc2008/TorCtl.local/TorUtil.py 2008-07-07 19:37:47 UTC (rev 15742)
@@ -0,0 +1,260 @@
+#!/usr/bin/python
+# TorCtl.py -- Python module to interface with Tor Control interface.
+# Copyright 2007 Mike Perry -- See LICENSE for licensing information.
+# Portions Copyright 2005 Nick Matthewson
+
+"""
+TorUtil -- Support functions for TorCtl.py and metatroller
+"""
+
+import os
+import re
+import sys
+import socket
+import binascii
+import sha
+import math
+import time
+
+__all__ = ["Enum", "Enum2", "Callable", "sort_list", "quote", "escape_dots", "unescape_dots",
+ "BufSock", "secret_to_key", "urandom_rng", "s2k_gen", "s2k_check", "plog",
+ "ListenSocket", "zprob"]
+
+# TODO: Make functions to read these from a config file. This isn't
+# the right place for them either.. But at least it's unified.
+tor_port = 9050
+tor_host = '127.0.0.1'
+
+meta_port = 9052
+meta_host = '127.0.0.1'
+
+control_port = 9051
+control_host = '127.0.0.1'
+
+class Enum:
+ """ Defines an ordered dense name-to-number 1-1 mapping """
+ def __init__(self, start, names):
+ self.nameOf = {}
+ idx = start
+ for name in names:
+ setattr(self,name,idx)
+ self.nameOf[idx] = name
+ idx += 1
+
+class Enum2:
+ """ Defines an ordered sparse name-to-number 1-1 mapping """
+ def __init__(self, **args):
+ self.__dict__.update(args)
+ self.nameOf = {}
+ for k,v in args.items():
+ self.nameOf[v] = k
+
+class Callable:
+ def __init__(self, anycallable):
+ self.__call__ = anycallable
+
+def sort_list(list, key):
+ """ Sort a list by a specified key """
+ list.sort(lambda x,y: cmp(key(x), key(y))) # Python < 2.4 hack
+ return list
+
+def quote(s):
+ return re.sub(r'([\r\n\\\"])', r'\\\1', s)
+
+def escape_dots(s, translate_nl=1):
+ if translate_nl:
+ lines = re.split(r"\r?\n", s)
+ else:
+ lines = s.split("\r\n")
+ if lines and not lines[-1]:
+ del lines[-1]
+ for i in xrange(len(lines)):
+ if lines[i].startswith("."):
+ lines[i] = "."+lines[i]
+ lines.append(".\r\n")
+ return "\r\n".join(lines)
+
+def unescape_dots(s, translate_nl=1):
+ lines = s.split("\r\n")
+
+ for i in xrange(len(lines)):
+ if lines[i].startswith("."):
+ lines[i] = lines[i][1:]
+
+ if lines and lines[-1]:
+ lines.append("")
+
+ if translate_nl:
+ return "\n".join(lines)
+ else:
+ return "\r\n".join(lines)
+
+# XXX: Exception handling
+class BufSock:
+ def __init__(self, s):
+ self._s = s
+ self._buf = []
+
+ def readline(self):
+ if self._buf:
+ idx = self._buf[0].find('\n')
+ if idx >= 0:
+ result = self._buf[0][:idx+1]
+ self._buf[0] = self._buf[0][idx+1:]
+ return result
+
+ while 1:
+ s = self._s.recv(128)
+ if not s: return None
+ # XXX: This really does need an exception
+ # raise ConnectionClosed()
+ idx = s.find('\n')
+ if idx >= 0:
+ self._buf.append(s[:idx+1])
+ result = "".join(self._buf)
+ rest = s[idx+1:]
+ if rest:
+ self._buf = [ rest ]
+ else:
+ del self._buf[:]
+ return result
+ else:
+ self._buf.append(s)
+
+ def write(self, s):
+ self._s.send(s)
+
+ def close(self):
+ self._s.close()
+
+# SocketServer.TCPServer is nuts..
+class ListenSocket:
+ def __init__(self, listen_ip, port):
+ msg = None
+ self.s = None
+ for res in socket.getaddrinfo(listen_ip, port, socket.AF_UNSPEC,
+ socket.SOCK_STREAM, 0, socket.AI_PASSIVE):
+ af, socktype, proto, canonname, sa = res
+ try:
+ self.s = socket.socket(af, socktype, proto)
+ self.s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ except socket.error, msg:
+ self.s = None
+ continue
+ try:
+ self.s.bind(sa)
+ self.s.listen(1)
+ except socket.error, msg:
+ self.s.close()
+ self.s = None
+ continue
+ break
+ if self.s is None:
+ raise socket.error(msg)
+
+ def accept(self):
+ conn, addr = self.s.accept()
+ return conn
+
+ def close(self):
+ self.s.close()
+
+
+def secret_to_key(secret, s2k_specifier):
+ """Used to generate a hashed password string. DOCDOC."""
+ c = ord(s2k_specifier[8])
+ EXPBIAS = 6
+ count = (16+(c&15)) << ((c>>4) + EXPBIAS)
+
+ d = sha.new()
+ tmp = s2k_specifier[:8]+secret
+ slen = len(tmp)
+ while count:
+ if count > slen:
+ d.update(tmp)
+ count -= slen
+ else:
+ d.update(tmp[:count])
+ count = 0
+ return d.digest()
+
+def urandom_rng(n):
+ """Try to read some entropy from the platform entropy source."""
+ f = open('/dev/urandom', 'rb')
+ try:
+ return f.read(n)
+ finally:
+ f.close()
+
+def s2k_gen(secret, rng=None):
+ """DOCDOC"""
+ if rng is None:
+ if hasattr(os, "urandom"):
+ rng = os.urandom
+ else:
+ rng = urandom_rng
+ spec = "%s%s"%(rng(8), chr(96))
+ return "16:%s"%(
+ binascii.b2a_hex(spec + secret_to_key(secret, spec)))
+
+def s2k_check(secret, k):
+ """DOCDOC"""
+ assert k[:3] == "16:"
+
+ k = binascii.a2b_hex(k[3:])
+ return secret_to_key(secret, k[:9]) == k[9:]
+
+
+## XXX: Make this a class?
+loglevel = "DEBUG"
+loglevels = {"DEBUG" : 0, "INFO" : 1, "NOTICE" : 2, "WARN" : 3, "ERROR" : 4}
+
+def plog(level, msg): # XXX: Timestamps
+ if(loglevels[level] >= loglevels[loglevel]):
+ t = time.strftime("%a %b %d %H:%M:%S %Y")
+ print level, '[', t, ']:', msg
+ sys.stdout.flush()
+
+# Stolen from
+# http://www.nmr.mgh.harvard.edu/Neural_Systems_Group/gary/python/stats.py
+def zprob(z):
+ """
+Returns the area under the normal curve 'to the left of' the given z value.
+Thus,
+ for z<0, zprob(z) = 1-tail probability
+ for z>0, 1.0-zprob(z) = 1-tail probability
+ for any z, 2.0*(1.0-zprob(abs(z))) = 2-tail probability
+Adapted from z.c in Gary Perlman's |Stat.
+
+Usage: lzprob(z)
+"""
+ Z_MAX = 6.0 # maximum meaningful z-value
+ if z == 0.0:
+ x = 0.0
+ else:
+ y = 0.5 * math.fabs(z)
+ if y >= (Z_MAX*0.5):
+ x = 1.0
+ elif (y < 1.0):
+ w = y*y
+ x = ((((((((0.000124818987 * w
+ -0.001075204047) * w +0.005198775019) * w
+ -0.019198292004) * w +0.059054035642) * w
+ -0.151968751364) * w +0.319152932694) * w
+ -0.531923007300) * w +0.797884560593) * y * 2.0
+ else:
+ y = y - 2.0
+ x = (((((((((((((-0.000045255659 * y
+ +0.000152529290) * y -0.000019538132) * y
+ -0.000676904986) * y +0.001390604284) * y
+ -0.000794620820) * y -0.002034254874) * y
+ +0.006549791214) * y -0.010557625006) * y
+ +0.011630447319) * y -0.009279453341) * y
+ +0.005353579108) * y -0.002141268741) * y
+ +0.000535310849) * y +0.999936657524
+ if z > 0.0:
+ prob = ((x+1.0)*0.5)
+ else:
+ prob = ((1.0-x)*0.5)
+ return prob
+
Deleted: torflow/branches/gsoc2008/TorCtl.local/__init__.py
===================================================================
--- torflow/branches/gsoc2008/TorCtl/__init__.py 2008-07-07 16:40:57 UTC (rev 15734)
+++ torflow/branches/gsoc2008/TorCtl.local/__init__.py 2008-07-07 19:37:47 UTC (rev 15742)
@@ -1,2 +0,0 @@
-
-__all__ = ["TorUtil", "GeoIPSupport", "PathSupport", "TorCtl", "StatsSupport"]
Copied: torflow/branches/gsoc2008/TorCtl.local/__init__.py (from rev 15741, torflow/branches/gsoc2008/TorCtl/__init__.py)
===================================================================
--- torflow/branches/gsoc2008/TorCtl.local/__init__.py (rev 0)
+++ torflow/branches/gsoc2008/TorCtl.local/__init__.py 2008-07-07 19:37:47 UTC (rev 15742)
@@ -0,0 +1,2 @@
+
+__all__ = ["TorUtil", "GeoIPSupport", "PathSupport", "TorCtl", "StatsSupport"]
More information about the tor-commits
mailing list