[or-cvs] r10203: Moved all GeoIP-code to a separate file and added some new r (in torflow/trunk: . TorCtl)
renner at seul.org
renner at seul.org
Fri May 18 00:41:43 UTC 2007
Author: renner
Date: 2007-05-17 20:41:06 -0400 (Thu, 17 May 2007)
New Revision: 10203
Added:
torflow/trunk/TorCtl/GeoIPSupport.py
Modified:
torflow/trunk/TorCtl/PathSupport.py
torflow/trunk/TorCtl/__init__.py
torflow/trunk/op-addon.py
Log:
Moved all GeoIP-code to a separate file and added some new restrictions to PathSupport.py
Added: torflow/trunk/TorCtl/GeoIPSupport.py
===================================================================
--- torflow/trunk/TorCtl/GeoIPSupport.py 2007-05-17 01:52:26 UTC (rev 10202)
+++ torflow/trunk/TorCtl/GeoIPSupport.py 2007-05-18 00:41:06 UTC (rev 10203)
@@ -0,0 +1,83 @@
+#!/usr/bin/python
+
+import struct
+import socket
+import GeoIP
+import TorCtl
+from TorUtil import plog
+
+# GeoIP data object: choose database here
+geoip = GeoIP.new(GeoIP.GEOIP_STANDARD)
+#geoip = GeoIP.open("./GeoLiteCity.dat", GeoIP.GEOIP_STANDARD)
+
+# Continent class
+class Continent:
+ def __init__(self, continent_code):
+ self.code = continent_code
+ self.countries = []
+
+ def contains(self, country_code):
+ return country_code in self.countries
+
+# The continents
+africa = Continent("AF")
+# TODO: Add more countries
+africa.countries = ["CI"]
+
+asia = Continent("AS")
+asia.countries = ["AP","AE","AF","AM","AZ","BD","BH","BN","BT","CC","CN","CX","CY","GE",
+ "HK","ID","IL","IN","IO","IQ","IR","JO","JP","KG","KH","KP","KR","KW",
+ "KZ","LA","LB","LK","MM","MN","MO","MV","MY","NP","OM","PH","PK","PS",
+ "QA","RU","SA","SG","SY","TH","TJ","TM","TP","TR","TW","UZ","VN","YE"]
+
+europe = Continent("EU")
+europe.countries = ["EU","AD","AL","AT","BA","BE","BG","BY","CH","CZ","DE","DK","EE","ES",
+ "FI","FO","FR","FX","GB","GI","GR","HR","HU","IE","IS","IT","LI","LT",
+ "LU","LV","MC","MD","MK","MT","NL","NO","PL","PT","RO","SE","SI","SJ",
+ "SK","SM","UA","VA","YU"]
+
+oceania = Continent("OC")
+oceania.countries = ["AS","AU","CK","FJ","FM","GU","KI","MH","MP","NC","NF","NR","NU","NZ",
+ "PF","PG","PN","PW","SB","TK","TO","TV","UM","VU","WF","WS"]
+
+north_america = Continent("NA")
+north_america.countries = ["CA","MX","US"]
+
+south_america = Continent("SA")
+south_america.countries = ["AG","AI","AN","AR","AW","BB","BM","BO","BR","BS","BZ","CL","CO",
+ "CR","CU","DM","DO","EC","FK","GD","GF","GL","GP","GS","GT","GY",
+ "HN","HT","JM","KN","KY","LC","MQ","MS","NI","PA","PE","PM","PR",
+ "PY","SA","SR","SV","TC","TT","UY","VC","VE","VG","VI"]
+
+# List of continents
+continents = [africa, asia, europe, north_america, oceania, south_america]
+
+# Perform country -- continent mapping
+def get_continent(country_code):
+ for c in continents:
+ if c.contains(country_code):
+ return c.code
+ plog("WARN", country_code + " is not on any continent")
+ return None
+
+# Get the country code out of a GeoLiteCity record
+def get_country_from_record(ip):
+ record = geoip.record_by_addr(ip)
+ if record != None:
+ return record['country_code']
+
+# Router class extended to GeoIP
+class GeoIPRouter(TorCtl.Router):
+ def __init__(self, router): # Promotion constructor :)
+ self.__dict__ = router.__dict__
+ # Select method to get the country_code here
+ self.country_code = geoip.country_code_by_addr(self.get_ip_dotted())
+ #self.country_code = get_country_from_record(self.get_ip_dotted())
+ self.continent = None
+ if self.country_code == None:
+ plog("WARN", self.nickname + ": Country code not found")
+ else: self.continent = get_continent(self.country_code)
+
+ # Convert long int back to dotted quad string
+ def get_ip_dotted(self):
+ return socket.inet_ntoa(struct.pack('>I', self.ip))
Modified: torflow/trunk/TorCtl/PathSupport.py
===================================================================
--- torflow/trunk/TorCtl/PathSupport.py 2007-05-17 01:52:26 UTC (rev 10202)
+++ torflow/trunk/TorCtl/PathSupport.py 2007-05-18 00:41:06 UTC (rev 10203)
@@ -17,7 +17,8 @@
"AtLeastNNodeRestriction", "NotNodeRestriction", "Subnet16Restriction",
"UniqueRestriction", "UniformGenerator", "OrderedExitGenerator",
"PathSelector", "Connection", "NickRestriction", "IdHexRestriction",
-"PathBuilder", "SelectionManager"]
+"PathBuilder", "SelectionManager", "CountryCodeRestriction",
+"CountryRestriction", "UniqueCountryRestriction", "ContinentRestriction" ]
#################### Path Support Interfaces #####################
@@ -297,7 +298,52 @@
class UniqueRestriction(PathRestriction):
def r_is_ok(self, path, r): return not r in path
+#################### GeoIP Restrictions ###################
+# Ensure country_code is set
+class CountryCodeRestriction(NodeRestriction):
+ def r_is_ok(self, r):
+ return r.country_code != None
+
+# Ensure a specific country_code
+class CountryRestriction(NodeRestriction):
+ def __init__(self, country_code):
+ self.country_code = country_code
+
+ def r_is_ok(self, r):
+ return r.country_code == self.country_code
+
+# Ensure every router to have distinct country
+class UniqueCountryRestriction(PathRestriction):
+ def r_is_ok(self, path, router):
+ for r in path:
+ if router.country_code == r.country_code:
+ return False
+ return True
+
+# Do not more than n continent crossings
+class ContinentRestriction(PathRestriction):
+ def __init__(self, n):
+ self.n = n
+
+ # TODO: Include our location
+ def r_is_ok(self, path, router):
+ crossings = 0
+ last = None
+ # Compute crossings until now
+ for r in path:
+ # Jump over the first router
+ if last:
+ if r.continent != last.continent:
+ crossings += 1
+ last = r
+ # Check what happens if we add 'router'
+ if len(path)>=1:
+ if router.continent != last.continent:
+ crossings += 1
+ if crossings > self.n: return False
+ else: return True
+
#################### Node Generators ######################
class UniformGenerator(NodeGenerator):
@@ -387,7 +433,7 @@
"""
def __init__(self, pathlen, order_exits,
percent_fast, percent_skip, min_bw, use_all_exits,
- uniform, use_exit, use_guards):
+ uniform, use_exit, use_guards, use_geoip=False):
self.__ordered_exit_gen = None
self.pathlen = pathlen
self.order_exits = order_exits
@@ -398,6 +444,8 @@
self.uniform = uniform
self.exit_name = use_exit
self.use_guards = use_guards
+ # Replace with a geoip-config object?
+ self.use_geoip = use_geoip
def reconfigure(self, sorted_r):
if self.use_all_exits:
@@ -435,6 +483,22 @@
else:
self.exit_rstr.add_restriction(NickRestriction(self.exit_name))
+ # GeoIP configuration
+ # TODO: Make configurable, config-object?
+ if self.use_geoip:
+ # Restrictions for Entry
+ entry_rstr.add_restriction(CountryCodeRestriction())
+ # First hop in our country
+ #entry_rstr.add_restriction(CountryRestriction("DE"))
+ # Middle
+ mid_rstr.add_restriction(CountryCodeRestriction())
+ # Exit
+ self.exit_rstr.add_restriction(CountryCodeRestriction())
+ # Path
+ self.path_rstr.add_restriction(UniqueCountryRestriction())
+ # Specify max number of crossings here
+ self.path_rstr.add_restriction(ContinentRestriction(1))
+
# This is kind of hokey..
if self.order_exits:
if self.__ordered_exit_gen:
Modified: torflow/trunk/TorCtl/__init__.py
===================================================================
--- torflow/trunk/TorCtl/__init__.py 2007-05-17 01:52:26 UTC (rev 10202)
+++ torflow/trunk/TorCtl/__init__.py 2007-05-18 00:41:06 UTC (rev 10203)
@@ -1,2 +1,2 @@
-__all__ = ["TorUtil", "PathSupport", "TorCtl"]
+__all__ = ["TorUtil", "GeoIPSupport", "PathSupport", "TorCtl"]
Modified: torflow/trunk/op-addon.py
===================================================================
--- torflow/trunk/op-addon.py 2007-05-17 01:52:26 UTC (rev 10202)
+++ torflow/trunk/op-addon.py 2007-05-18 00:41:06 UTC (rev 10203)
@@ -10,51 +10,50 @@
# from or-addons/alternate directory, building fast circuits from all
# of these infos and attaching streams to fast circuits.
-# TODO: import 'with'-statement for Lock objects? (with some_lock: do something)
+# TODO: import 'with'-statement for Lock objects (Python 2.5: "with some_lock: do something")
import re
import sys
import math
import time
import sched
-import struct
import socket
import threading
import Queue
# Non-standard packages
import socks
-import GeoIP
#import networkx
+import TorCtl.PathSupport
+import TorCtl.GeoIPSupport
from TorCtl import *
-from TorCtl.TorUtil import *
-from TorCtl.PathSupport import *
+from TorCtl.TorUtil import plog
# Move these to config file
control_host = "127.0.0.1"
control_port = 9051
socks_host = "127.0.0.1"
socks_port = 9050
+
# Any ideas/proposals?
ping_dummy_host = "127.0.0.1"
ping_dummy_port = 100
-# Close circ after n timeouts
-timeout_limit = 3
-# Slow RTT: 1 second??
+# Close circ after n timeouts or slownesses
+timeout_limit = 2
+# Slow RTT := x seconds
slow = 1
-# Set interval between work loads
-sleep_interval = 10
-# No of idle circuits
-idle_circuits = 8
+# Set interval between work loads in sec
+sleep_interval = 30
+# No of idle circuits to build preemptively
+idle_circuits = 6
-# GeoIP data object
-geoip = GeoIP.new(GeoIP.GEOIP_STANDARD)
-# TODO: Load the big database for more detailed info?
-#geoip = GeoIP.open("./GeoLiteCity.dat", GeoIP.GEOIP_STANDARD)
-
# Lock object for regulating access to the circuit list
circs_lock = threading.Lock()
+# Infos about this proxy TODO: Save in some class
+my_ip = None
+my_country = None
+
# Configure Selection Manager here!!
# Do NOT modify this object directly after it is handed to
# PathBuilder, Use PathBuilder.schedule_selmgr instead.
@@ -67,7 +66,8 @@
use_all_exits=False,
uniform=True,
use_exit=None,
- use_guards=False)
+ use_guards=False,
+ use_geoip=True)
######################################### BEGIN: Connection #####################
@@ -90,43 +90,22 @@
######################################### END: Connection #####################
######################################### Router, Circuit, Stream #####################
-# Router class extended to GeoIP
-class GeoIPRouter(TorCtl.Router):
- def __init__(self, router): # Promotion constructor :)
- self.__dict__ = router.__dict__
- # Set the country code
- self.country_code = self.get_country()
-
- # Convert long int to dotted quad string
- def get_ip_dotted(self):
- return socket.inet_ntoa(struct.pack('L', self.ip))
-
- # Get the country-code from GeoIP on the fly
- def get_country(self):
- ip = self.get_ip_dotted()
- country = geoip.country_code_by_addr(ip)
- #record = geoip.record_by_addr(ip)
- #if record != None:
- # country = record['country_code3']
- #plog("DEBUG", "Set country of router " + self.nickname + " (" + ip + "): " + str(country))
- return country
-
# Circuit class extended to RTTs
class Circuit(PathSupport.Circuit):
def __init__(self):
PathSupport.Circuit.__init__(self)
self.total_rtt = None # double (sec), substitute with..
- self.rtts = [] # list of partial rtts: 1-2-3
+ self.rtts = {} # dict of partial rtts, for pathlen 3: 1-2-None
self.timeout_counter = 0 # timeout limit
self.slowness_counter = 0 # slowness limit
self.closed = False # Mark circuits closed
-# Stream class extended to isPing
+# Stream class extended to isPing and hop
class Stream(PathSupport.Stream):
def __init__(self, sid, host, port, kind):
PathSupport.Stream.__init__(self, sid, host, port, kind)
self.isPing = False
- self.hop = None # Save hop if this is a ping, None = complete circ
+ self.hop = None # Save hop if this is a ping, None = complete circ
######################################### Router, Circuit, Stream #####################
######################################### BEGIN: Pinger #####################
@@ -178,16 +157,59 @@
######################################### END: NetworkModel #####################
######################################### BEGIN: EventHandler #####################
+# DRAFT for a new CircuitManager
+class NewCircuitManager:
+ def __init__(self, c):
+ self.conn = c # connection to Tor
+ self.circuits = {} # dict mapping id:circuit
+ self.circs_sorted = [] # list of circs sorted for rtt
+
+ # Sort a list by a specified key
+ def sort_list(self, list, key):
+ list.sort(lambda x,y: cmp(key(x), key(y))) # Python < 2.4 hack
+ return list
+
+ def refresh_sorted_list(self):
+ self.circs_sorted = self.sort_list(self.circuits.values(), lambda x: x.total_rtt)
+
+ def add_circuit(self, circ):
+ self.circuits[circ.circ_id] = circ
+
+ def del_circuit(self, circ_id):
+ # TODO: Test
+ del self.circuits[circ_id]
+
+ def new_circuit(self):
+ circ = None
+ while circ == None:
+ try:
+ # Build the circuit
+ circ = self.conn.build_circuit(self.selmgr.pathlen, self.selmgr.path_selector)
+ self.add_circuit(circ)
+ except TorCtl.ErrorReply, e:
+ # FIXME: How come some routers are non-existant? Shouldn't
+ # we have gotten an NS event to notify us they disappeared?
+ plog("NOTICE", "Error building circ: " + str(e.args))
+
+ def close_circuit(self, circ_id):
+ # try .. except
+ self.conn.close_circuit(circ_id)
+
+ def attach_stream(self, stream):
+ pass
+
+###########################################
+
# We need an EventHandler, this one extends PathBuilder
class EventHandler(PathSupport.PathBuilder):
def __init__(self, c, selmgr):
# Call constructor of superclass
- PathBuilder.__init__(self, c, selmgr, GeoIPRouter)
+ PathSupport.PathBuilder.__init__(self, c, selmgr, GeoIPSupport.GeoIPRouter)
# Additional stuff
self.ping_circs = Queue.Queue() # (circ_id, hop)-pairs
self.start_times = {} # dict mapping (circ_id, hop):start_time TODO: cleanup
self.circs_sorted = [] # sorted list of circs, generated regularly
- # Set up the CircuitManager
+ # Set up the CircuitManager, only pass self.circuits instead of self?
self.circ_manager = CircuitManager(selmgr, c, self)
self.circ_manager.setDaemon(True)
self.circ_manager.start()
@@ -217,13 +239,14 @@
# Do something when circuit-events occur
def circ_status_event(self, c):
- circs_lock.acquire()
# Construct output for logging
output = [c.event_name, str(c.circ_id), c.status]
if c.path: output.append(",".join(c.path))
if c.reason: output.append("REASON=" + c.reason)
if c.remote_reason: output.append("REMOTE_REASON=" + c.remote_reason)
plog("DEBUG", " ".join(output))
+ # Acquire lock here
+ circs_lock.acquire()
# Circuits we don't control get built by Tor
if c.circ_id not in self.circuits:
plog("DEBUG", "Ignoring circuit " + str(c.circ_id) + " (controlled by Tor or not yet in the list)")
@@ -231,17 +254,24 @@
return
if c.status == "EXTENDED":
self.circuits[c.circ_id].last_extended_at = c.arrived_at
+ circs_lock.release()
elif c.status == "FAILED" or c.status == "CLOSED":
# XXX: Can still get a STREAM FAILED for this circ after this
circ = self.circuits[c.circ_id]
+ # Actual removal of the circ
del self.circuits[c.circ_id]
- # Refresh the list
- #self.refresh_sorted_list()
+ circs_lock.release()
+ # Give away pending streams
for stream in circ.pending_streams:
- plog("DEBUG", "Finding new circ for " + str(stream.strm_id))
- # TODO: What to do with pings?
+ if stream.isPing:
+ #plog("DEBUG", "Finding new circ for ping stream " + str(stream.strm_id))
+ pass
if not stream.isPing:
+ plog("DEBUG", "Finding new circ for " + str(stream.strm_id))
self.attach_stream_any(stream, stream.detached_from)
+ # Refresh the list
+ self.refresh_sorted_list()
+ return
# TODO: Check if there are enough circs?
elif c.status == "BUILT":
# TODO: Perform a measuring directly?
@@ -255,9 +285,12 @@
plog("WARN", "Error attaching stream: " + str(e.args))
circs_lock.release()
return
- circs_lock.release()
+ circs_lock.release()
+ else:
+ # If this was e.g. a LAUNCHED
+ circs_lock.release()
- # Attach a regular user stream, moved here to play around
+ # Attach a regular user stream
def attach_stream_any(self, stream, badcircs):
# To be able to always choose the fastest:
# slows down attaching?
@@ -290,7 +323,7 @@
except TorCtl.ErrorReply, e:
# No need to retry here. We should get the failed
# event for either the circ or stream next
- plog("WARN", "Error attaching stream: "+str(e.args))
+ plog("WARN", "Error attaching stream: " + str(e.args))
return
break
else:
@@ -304,30 +337,30 @@
except TorCtl.ErrorReply, e:
# FIXME: How come some routers are non-existant? Shouldn't
# we have gotten an NS event to notify us they disappeared?
- plog("NOTICE", "Error building circ: "+str(e.args))
+ plog("NOTICE", "Error building circ: " + str(e.args))
for u in unattached_streams:
- plog("DEBUG", "Attaching "+str(u.strm_id)+" pending build of "+str(circ.circ_id))
- u.pending_circ = circ
+ plog("DEBUG", "Attaching " + str(u.strm_id) + " pending build of circuit " + str(circ.circ_id))
+ u.pending_circ = circ
circ.pending_streams.extend(unattached_streams)
+ # Problem here??
circs_lock.acquire()
self.circuits[circ.circ_id] = circ
circs_lock.release()
self.last_exit = circ.exit
- # Handle a ping stream
+ # Attach a ping stream to its circuit
def attach_ping(self, stream):
plog("DEBUG", "New ping request")
- # Get info from the Queue
- # TODO: check if empty
+ # Get info from the Queue TODO: check if empty
ping_info = self.ping_circs.get()
- # Extract ping-infos
+ # Extract ping-info
circ_id = ping_info[0]
hop = ping_info[1]
# Set circ to stream
stream.circ = circ_id
try:
circs_lock.acquire()
- # Get the circuit, TODO: Handle circs that do not exist anymore!
+ # Get the circuit
if circ_id in self.circuits:
circ = self.circuits[circ_id]
if circ.built and not circ.closed:
@@ -340,7 +373,9 @@
else:
plog("WARN", "Circuit not built")
else:
- plog("WARN", "Circuit does not exist")
+ # Close stream if circuit is gone
+ plog("WARN", "Circuit does not exist anymore, closing stream " + str(stream.strm_id))
+ self.c.close_stream(stream.strm_id, 5)
circs_lock.release()
except TorCtl.ErrorReply, e:
plog("WARN", "Error attaching stream: " + str(e.args))
@@ -391,7 +426,7 @@
self.circuits[s.circ_id].timeout_counter += 1
self.circuits[s.circ_id].slowness_counter += 1
plog("DEBUG", str(self.circuits[s.circ_id].timeout_counter) + " timeout(s) on circuit " + str(s.circ_id))
- if self.circuits[s.circ_id].timeout_counter >= timeout_limit:
+ if self.circuits[s.circ_id].timeout_counter >= timeout_limit and not self.circuits[s.circ_id].closed:
# Close the circuit
plog("DEBUG", "Reached limit on timeouts --> closing circuit " + str(s.circ_id))
self.circuits[s.circ_id].closed = True
@@ -404,18 +439,23 @@
return
# This is a successful ping: measure here
now = time.time()
- rtt = now - self.start_times[(s.circ_id, self.streams[s.strm_id].hop)]
+ hop = self.streams[s.strm_id].hop
+ rtt = now - self.start_times[(s.circ_id, hop)]
plog("INFO", "Measured RTT: " + str(rtt) + " sec")
# Save RTT to circuit
- self.circuits[s.circ_id].total_rtt = rtt
+ self.circuits[s.circ_id].rtts[hop] = rtt
+ # Additionally save total_rtt ?
+ if hop == None:
+ self.circuits[s.circ_id].total_rtt = rtt
# Close if slow-max is reached
if rtt >= slow:
self.circuits[s.circ_id].slowness_counter += 1
- if self.circuits[s.circ_id].slowness_counter >= timeout_limit:
+ if self.circuits[s.circ_id].slowness_counter >= timeout_limit and not self.circuits[s.circ_id].closed:
plog("DEBUG", "Slow-max is reached --> closing circuit " + str(s.circ_id))
self.circuits[s.circ_id].closed = True
self.c.close_circuit(s.circ_id)
+
circs_lock.release()
# Resort every time ??
self.refresh_sorted_list()
@@ -428,7 +468,7 @@
circs_lock.acquire()
self.circuits[s.circ_id].timeout_counter += 1
plog("DEBUG", str(self.circuits[s.circ_id].timeout_counter) + " timeout(s) on circuit " + str(s.circ_id))
- if self.circuits[s.circ_id].timeout_counter >= timeout_limit:
+ if self.circuits[s.circ_id].timeout_counter >= timeout_limit and not self.circuits[s.circ_id].closed:
# Close the circuit
plog("DEBUG", "Reached limit on timeouts --> closing circuit " + str(s.circ_id))
self.circuits[s.circ_id].closed = True
@@ -467,8 +507,7 @@
if s.strm_id not in self.streams:
plog("NOTICE", "Failed stream " + str(s.strm_id) + " not found")
return
- if (not s.circ_id) & (s.reason != "DONE"):
- plog("WARN", "Stream " + str(s.strm_id) + " closed/failed from no circuit!")
+ #if not s.circ_id: plog("WARN", "Stream " + str(s.strm_id) + " closed/failed from no circuit")
# We get failed and closed for each stream. OK to return
# and let the CLOSED do the cleanup
if s.status == "FAILED":
@@ -477,7 +516,7 @@
circs_lock.acquire()
if s.circ_id in self.circuits: self.circuits[s.circ_id].dirty = True
elif self.streams[s.strm_id].attached_at != 0:
- plog("WARN","Failed stream on unknown circ " + str(s.circ_id))
+ plog("WARN","Failed stream on unknown circuit " + str(s.circ_id))
circs_lock.release()
return
# CLOSED
@@ -507,6 +546,7 @@
# Does work regularly
# TODO: Switch circuit-managing off to get circuits created from Tor
# TODO: Add a NetworkModel to this!
+# TODO: Make this to contain the circuit-list and use a pinger-thread
class CircuitManager(threading.Thread):
@@ -565,9 +605,13 @@
circs_lock.release()
for c in circs:
if c.built:
+ # Get length of c ...
id = c.circ_id
# TODO: Measure for all hops, test if result is
# bigger each time, else start again
+ #self.handler.queue_ping_circ((id, 2))
+ # Trigger ping
+ #self.pinger.ping()
# Put in the queue (circ, hop), XXX: synchronize!
self.handler.queue_ping_circ((id, None))
# Trigger ping
@@ -597,6 +641,15 @@
# Do the configuration
def configure(conn):
+ # Get our own IP and country here, TODO: use try .. except?
+ try:
+ info = conn.get_info("address")
+ my_ip = info["address"]
+ my_country = GeoIPSupport.geoip.country_code_by_addr(my_ip)
+ #my_country = GeoIPSupport.get_country_from_record(my_ip)
+ plog("INFO", "Our IP address is " + str(my_ip) + " [" + my_country + "]")
+ except:
+ plog("INFO", "Could not get our IP")
# Set events to listen to
conn.set_events([TorCtl.EVENT_TYPE.STREAM,
TorCtl.EVENT_TYPE.CIRC,
More information about the tor-commits
mailing list