- tor-commits - lists.torproject.org

[ooni-probe/master] * Minor change to Makefile.
by isis＠torproject.org 16 Nov '12

16 Nov '12

commit 64be6a7163534713ef50b527d47c90443be879fe Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Nov 15 23:06:57 2012 +0000 * Minor change to Makefile. --- Makefile | 28 ++++++++++++++++------------ 1 files changed, 16 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index a965c00..13d277e 100644 --- a/Makefile +++ b/Makefile @@ -158,14 +158,16 @@ libdnet: scapy-all: scapy pypcap libdnet -tags-recursive: - list='$(SUBDIRS)'; for subdir in $$list; do \ - test "$$subdir" = . || (cd $$subdir && make tags); \ - done - ETAGS = etags ETAGSFLAGS = ETAGSARGS = +## or change to ${SUBDIRS} +TAGME = ${topsrc_dir} + +tags-recursive: + list='$(TAGME)'; for subdir in $$list; do \ + test "$$subdir" = . || (cd $$subdir && make tags); \ + done tags: TAGS @@ -182,17 +184,19 @@ TAGS: tags-recursive $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \ $(TAGS_FILES) $(LISP) tags=; \ here=`pwd`; \ - list='$(SUBDIRS)'; for subdir in $$list; do \ + echo ${E_OR_V}; \ + list='$(TAGME)'; for subdir in $$list; do \ if test "$$subdir" = .; then :; else \ test -f $$subdir/TAGS && tags="$$tags -i $$here/$$subdir/TAGS"; \ fi; \ done; \ list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ unique=`for i in $$list; do \ - if test -f "$$i"; fi; \ - done | \ - $(AWK) ' { files[$$0] = 1; } \ - END { for (i in files) print i; }'`; \ + if test -f "$$i"; then echo $$i; fi; \ + done;` test -z "$(ETAGS_ARGS)$$tags$$unique" \ - || $(ETAGS) $(ETAGSFLAGS) $(ETAGS_ARGS) \ - $$tags $$unique + || $(ETAGS) $(ETAGSFLAGS) $(ETAGS_ARGS) $$tags $$unique + +awkward: + | $(AWK) ' { files[$$0] = 1; } \ + END { for (i in files) print i; }'`; \

1 0

[ooni-probe/master] * Moved iface and ipaddr discovery function to ooni.utils.net.
by isis＠torproject.org 16 Nov '12

16 Nov '12

commit 33ba9a28c4f3aae4e72c8b9fce370255e21971b7 Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun Nov 11 13:56:09 2012 +0000 * Moved iface and ipaddr discovery function to ooni.utils.net. --- nettests/bridge_reachability/echo.py | 26 ----- ooni/utils/net.py | 194 +++++++++++++++++++++++++-------- 2 files changed, 147 insertions(+), 73 deletions(-) diff --git a/nettests/bridge_reachability/echo.py b/nettests/bridge_reachability/echo.py index 5060ffd..5198766 100644 --- a/nettests/bridge_reachability/echo.py +++ b/nettests/bridge_reachability/echo.py @@ -59,41 +59,15 @@ class EchoTest(BaseScapyTest): Struct returned from getifaddrs(3) and turned into a tuple in the form (*ifa_name, AF_FAMILY, *ifa_addr) ''' - if self.localOptions: - log.debug("%s: local_options found" % self.name) for key, value in self.localOptions.items(): log.debug("setting self.%s = %s" % (key, value)) setattr(self, key, value) - ## xxx is this now .subOptions? - #self.inputFile = self.localOptions['file'] self.timeout *= 1000 ## convert to milliseconds if not self.interface: log.msg("No network interface specified!") - log.debug("OS detected: %s" % sys.platform) - if LINUX or OPENBSD or NETBSD or FREEBSD or DARWIN or SOLARIS: - from twisted.internet.test import _posixifaces - log.msg("Attempting to discover network interfaces...") - ifaces = _posixifaces._interfaces() - elif WINDOWS: - from twisted.internet.test import _win32ifaces - log.msg("Attempting to discover network interfaces...") - ifaces = _win32ifaces._interfaces() - else: - log.debug("Client OS %s not accounted for!" % sys.platform) - log.debug("Unable to discover network interfaces...") - ifaces = [('lo', '')] - - ## found = {'eth0': '1.1.1.1'} - found = [{i[0]: i[2]} for i in ifaces if i[0] != 'lo'] - log.info("Found interfaces:\n%s" % pprint(found)) - self.interfaces = self.tryInterfaces(found) - else: - ## xxx need a way to check that iface exists, is up, and - ## we have permissions on it - log.debug("Our interface has been set to %s" % self.interface) if self.pcap: try: diff --git a/ooni/utils/net.py b/ooni/utils/net.py index 3ddba61..0cbc095 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -2,58 +2,43 @@ # # net.py # -------- -# OONI utilities for networking related operations +# OONI utilities for network infrastructure and hardware. +# +# :authors: Isis Lovecruft, Arturo Filasto +# :version: 0.0.1-pre-alpha +# :license: (c) 2012 Isis Lovecruft, Arturo Filasto +# see attached LICENCE file import sys -from zope.interface import implements +from zope.interface import implements from twisted.internet import protocol, defer from twisted.internet import threads, reactor from twisted.web.iweb import IBodyProducer - from scapy.all import utils from ooni.utils import log, txscapy -userAgents = [("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6", "Firefox 2.0, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)", "Internet Explorer 7, Windows Vista"), - ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30)", "Internet Explorer 7, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 6, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 5.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 5, Windows XP"), - ("Opera/9.20 (Windows NT 6.0; U; en)", "Opera 9.2, Windows Vista"), - ("Opera/9.00 (Windows NT 5.1; U; en)", "Opera 9.0, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.50", "Opera 8.5, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.0", "Opera 8.0, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1) Opera 7.02 [en]", "Opera 7.02, Windows XP"), - ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20060127 Netscape/8.1", "Netscape 8.1, Windows XP")] - -class StringProducer(object): - implements(IBodyProducer) - - def __init__(self, body): - self.body = body - self.length = len(body) +#if platformm.system() == 'Windows': +# import _winreg as winreg - def startProducing(self, consumer): - consumer.write(self.body) - return defer.succeed(None) +PLATFORMS = {'LINUX': platform.startswith("linux"), + 'OPENBSD': platform.startswith("openbsd"), + 'FREEBSD': platform.startswith("freebsd"), + 'NETBSD': platform.startswith("netbsd"), + 'DARWIN': platform.startswith("darwin"), + 'SOLARIS': platform.startswith("sunos"), + 'WINDOWS': platform.startswith("win32")} - def pauseProducing(self): - pass - def stopProducing(self): - pass +class UnsupportedPlatform(Exception): + """Support for this platform is not currently available.""" -class BodyReceiver(protocol.Protocol): - def __init__(self, finished): - self.finished = finished - self.data = "" +class IfaceError(Exception): + """Could not find default network interface.""" - def dataReceived(self, bytes): - self.data += bytes - - def connectionLost(self, reason): - self.finished.callback(self.data) +class PermissionsError(SystemExit): + """This test requires admin or root privileges to run. Exiting...""" def capturePackets(pcap_filename): from scapy.all import sniff @@ -79,14 +64,129 @@ def capturePackets(pcap_filename): reactor.addSystemEventTrigger('before', 'shutdown', stopCapture) return d -class PermissionsError(SystemExit): - def __init__(self, *args, **kwargs): - if not args and not kwargs: - pe = "This test requires admin or root privileges to run. Exiting..." - super(PermissionsError, self).__init__(pe, *args, **kwargs) - else: - super(PermissionsError, self).__init__(*args, **kwargs) -class IfaceError(SystemExit): - def __init__(self, *args, **kwargs): - super(IfaceError, self).__init__(*args, **kwargs) +def getClientPlatform(platform_name=None): + for name, test in PLATFORMS.items(): + if not platform_name or platform_name.upper() == name: + if test: + return name, test + +def getPosixIface(): + from twisted.internet.test import _posixifaces + + log.msg("Attempting to discover network interfaces...") + ifaces = _posixifaces._interfaces() + ifup = tryInterfaces(ifaces) + return ifup + +def getWindowsIface(): + from twisted.internet.test import _win32ifaces + + log.msg("Attempting to discover network interfaces...") + ifaces = _win32ifaces._interfaces() + ifup = tryInterfaces(ifaces) + return ifup + +def getPlatformAndIfaces(platform_name=None): + client, test = getClientPlatform(platform_name) + if client: + if client == ('LINUX' or 'DARWIN') or client[-3:] == 'BSD': + return getPosixIface() + elif client == 'WINDOWS': + return getWindowsIface() + ## XXX fixme figure out how to get iface for Solaris + else: + return None + else: + raise UnsupportedPlatform + +def checkInterfaces(ifaces=None, timeout=1): + """ + @param ifaces: + A dictionary in the form of ifaces['if_name'] = 'if_addr'. + """ + try: + from scapy.all import IP, ICMP + from scapy.all import sr1 ## we want this check to be blocking + except: + log.msg(("Scapy required: www.secdev.org/projects/scapy")) + + ifup = {} + if not ifaces: + log.debug("net.checkInterfaces(): no interfaces specified!") + return None + + for iface in ifaces: + for ifname, ifaddr in iface: + log.debug("net.checkInterfaces(): testing iface {} by pinging" + + " local address {}".format(ifname, ifaddr)) + try: + pkt = IP(dst=ifaddr)/ICMP() + ans, unans = sr(pkt, iface=ifname, timeout=5, retry=3) + except Exception, e: + raise PermissionsError if e.find("Errno 1") else log.err(e) + else: + if ans.summary(): + log.debug("net.checkInterfaces(): got answer on interface %s" + + ":\n%s".format(ifname, ans.summary())) + ifup.update(ifname, ifaddr) + else: + log.debug("Interface test packet was unanswered:\n%s" + % unans.summary()) + if len(ifup) > 0: + log.msg("Discovered working network interfaces: %s" % ifup) + return ifup + else: + raise IfaceError + +def getNonLoopbackIfaces(platform_name=None): + try: + ifaces = getPlatformAndIfaces(platform_name) + except UnsupportedPlatform, up: + log.err(up) + + if not ifaces: + log.msg("Unable to discover network interfaces...") + return None + else: + found = [{i[0]: i[2]} for i in ifaces if i[0] != 'lo'] + log.debug("utils.net.getClientIfaces: Found non-loopback interfaces: %s" + % pprint(found)) + try: + interfaces = checkInterfaces(found) + except IfaceError, ie: + log.err(ie) + return None + else: + return interfaces + +def getNetworksFromRoutes(): + from scapy.all import conf, ltoa + from ipaddr import IPNetwork, IPAddress + + ## Hide the 'no routes' warnings + conf.verb = 0 + + networks = [] + client = conf.route + log.debug("Local Routing Table:\n{}".format(client)) + + for nw, nm, gw, iface, addr in client.routes: + n = IPNetwork( ltoa(nw) ) + (n.netmask, n.gateway, n.ipaddr) = [IPAddress(x) for x in [nm, gw, addr]] + n.iface = iface + if not n.compressed in networks: + networks.append(n) + + return networks + +def getDefaultIface(): + networks = getNetworksFromRoutes() + for net in networks: + if net.is_private: + return net + raise IfaceError + +def getLocalAddress(): + default_iface = getDefaultIface() + return default_iface.ipaddr

1 0

[ooni-probe/master] * Updated oonib requirements.
by isis＠torproject.org 16 Nov '12

16 Nov '12

commit 4dd13f70594d1a58dde6785a72de60956e9ed189 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Nov 15 23:12:36 2012 +0000 * Updated oonib requirements. --- oonib/requirements.txt | 2 -- 1 files changed, 0 insertions(+), 2 deletions(-) diff --git a/oonib/requirements.txt b/oonib/requirements.txt index f9d7541..9c0ab46 100644 --- a/oonib/requirements.txt +++ b/oonib/requirements.txt @@ -14,10 +14,8 @@ numpy==1.6.2 pyOpenSSL==0.13 pygeoip==0.2.5 repoze.sphinx.autointerface==0.7.1 -scapy==2.1.0 storm==0.19 transaction==1.3.0 -txtorcon==0.7 wsgiref==0.1.2 zope.component==4.0.0 zope.event==4.0.0

1 0

[ooni-probe/master] * Rewrite of echo.
by isis＠torproject.org 16 Nov '12

16 Nov '12

commit 9780774b3ab5ccdae79216fa47c844db1622857d Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Nov 15 23:09:46 2012 +0000 * Rewrite of echo. --- nettests/bridge_reachability/echo.py | 176 ++++++++++++++++------------------ 1 files changed, 83 insertions(+), 93 deletions(-) diff --git a/nettests/bridge_reachability/echo.py b/nettests/bridge_reachability/echo.py index 5198766..40436e7 100644 --- a/nettests/bridge_reachability/echo.py +++ b/nettests/bridge_reachability/echo.py @@ -6,59 +6,57 @@ # +---------+ # A simple ICMP-8 ping test. # -# :author: Isis Lovecruft -# :version: 0.0.1-pre-alpha -# :license: (c) 2012 Isis Lovecruft +# @authors: Isis Lovecruft, <isis(a)torproject.org> +# @version: 0.0.2-pre-alpha +# @license: copyright (c) 2012 Isis Lovecruft # see attached LICENCE file # import os import sys -from pprint import pprint - -from twisted.internet import reactor -from twisted.plugin import IPlugin from twisted.python import usage -from ooni.nettest import NetTestCase -from ooni.utils import log, Storage -from ooni.utils.net import PermissionsError, IfaceError +from twisted.internet import reactor, defer +from ooni import nettest +from ooni.utils import log, net, Storage try: - from scapy.all import sr1, IP, ICMP ## XXX v4/v6? + from scapy.all import IP, ICMP + from scapy.all import sr1 from ooni.lib import txscapy from ooni.lib.txscapy import txsr, txsend from ooni.templates.scapyt import BaseScapyTest except: log.msg("This test requires scapy, see www.secdev.org/projects/scapy") -class EchoTest(BaseScapyTest): +class UsageOptions(usage.Options): + optParameters = [ + ['dst', 'd', None, 'Host IP to ping'], + ['file', 'f', None, 'File of list of IPs to ping'], + ['interface', 'i', None, 'Network interface to use'], + ['count', 'c', 1, 'Number of packets to send', int], + ['size', 's', 56, 'Number of bytes to send in ICMP data field', int], + ['ttl', 'l', 25, 'Set the IP Time to Live', int], + ['timeout', 't', 2, 'Seconds until timeout if no response', int], + ['pcap', 'p', None, 'Save pcap to this file'], + ['receive', 'r', True, 'Receive response packets']] + +class EchoTest(nettest.NetTestCase): """ xxx fill me in """ name = 'echo' author = 'Isis Lovecruft <isis(a)torproject.org>' - description = 'A simple ICMP-8 test to see if a host is reachable.' - version = '0.0.1' - inputFile = ['file', 'f', None, 'File of list of IPs to ping'] + description = 'A simple ping test to see if a host is reachable.' + version = '0.0.2' requiresRoot = True - optParameters = [ - ['interface', 'i', None, 'Network interface to use'], - ['count', 'c', 5, 'Number of packets to send', int], - ['size', 's', 56, 'Number of bytes to send in ICMP data field', int], - ['ttl', 'l', 25, 'Set the IP Time to Live', int], - ['timeout', 't', 2, 'Seconds until timeout if no response', int], - ['pcap', 'p', None, 'Save pcap to this file'], - ['receive', 'r', True, 'Receive response packets'] - ] + usageOptions = UsageOptions + #requiredOptions = ['dst'] def setUp(self, *a, **kw): - ''' - :ivar ifaces: - Struct returned from getifaddrs(3) and turned into a tuple in the - form (*ifa_name, AF_FAMILY, *ifa_addr) - ''' + self.destinations = {} + if self.localOptions: for key, value in self.localOptions.items(): log.debug("setting self.%s = %s" % (key, value)) @@ -67,76 +65,68 @@ class EchoTest(BaseScapyTest): self.timeout *= 1000 ## convert to milliseconds if not self.interface: - log.msg("No network interface specified!") + try: + iface = net.getDefaultIface() + except Exception, e: + log.msg("No network interface specified!") + log.err(e) + else: + log.msg("Using system default interface: %s" % iface) + self.interface = iface if self.pcap: try: self.pcapfile = open(self.pcap, 'a+') except: log.msg("Unable to write to pcap file %s" % self.pcap) - self.pcapfile = None - - try: - assert os.path.isfile(self.file) - fp = open(self.file, 'r') - except Exception, e: - hosts = ['8.8.8.8', '38.229.72.14'] - log.err(e) - else: - self.inputs = self.inputProcessor(fp) - self.removePorts(hosts) - - log.debug("Initialization of %s test completed with:\n%s" - % (self.name, ''.join(self.__dict__))) - - @staticmethod - def inputParser(self, one_input): - log.debug("Removing possible ports from host addresses...") - log.debug("Initial inputs:\n%s" % pprint(inputs)) - - #host = [h.rsplit(':', 1)[0] for h in inputs] - host = h.rsplit(':', 1)[0] - log.debug("Inputs converted to:\n%s" % hosts) - - return host - - def tryInterfaces(self, ifaces): - try: - from scapy.all import sr1 ## we want this check to be blocking - except: - log.msg("This test requires scapy: www.secdev.org/projects/scapy") - raise SystemExit - - ifup = {} - while ifaces: - for ifname, ifaddr in ifaces: - log.debug("Currently testing network capabilities of interface" - + "%s by sending a packet to our address %s" - % (ifname, ifaddr)) - try: - pkt = IP(dst=ifaddr)/ICMP() - ans, unans = sr(pkt, iface=ifname, timeout=self.timeout) - except Exception, e: - raise PermissionsError if e.find("Errno 1") else log.err(e) - else: - ## xxx i think this logic might be wrong - log.debug("Interface test packet\n%s\n\n%s" - % (pkt.summary(), pkt.show2())) - if ans.summary(): - log.info("Received answer for test packet on interface" - +"%s :\n%s" % (ifname, ans.summary())) - ifup.update(ifname, ifaddr) - else: - log.info("Our interface test packet was unanswered:\n%s" - % unans.summary()) - - if len(ifup) > 0: - log.msg("Discovered the following working network interfaces: %s" - % ifup) - return ifup + else: + self.pcap = net.capturePacket(self.pcapfile) + + if not self.dst: + if self.file: + self.dstProcessor(self.file) + for key, value in self.destinations.items(): + for label, data in value.items(): + if not 'ans' in data: + self.dst = label else: - raise IfaceError("Could not find a working network interface.") + self.addDest(self.dst) + log.debug("self.dst is now: %s" % self.dst) - def test_icmp(self): - return self.sr(IP(dst=self.input)/ICMP()) + log.debug("Initialization of %s test completed." % self.name) + + def addDest(self, dest): + d = dest.strip() + self.destinations[d] = {'dst_ip': d} + + def dstProcessor(self, inputfile): + from ipaddr import IPAddress + if os.path.isfile(inputfile): + with open(inputfile) as f: + for line in f.readlines(): + if line.startswith('#'): + continue + self.addDest(line) + + def test_icmp(self): + def process_response(echo_reply, dest): + ans, unans = echo_reply + if ans: + log.msg("Recieved echo reply from %s: %s" % (dest, ans)) + else: + log.msg("No reply was received from %s. Possible censorship event." % dest) + log.debug("Unanswered packets: %s" % unans) + self.report[dest] = echo_reply + + for label, data in self.destinations.items(): + reply = sr1(IP(dst=lebal)/ICMP()) + process = process_reponse(reply, label) + + #(ans, unans) = ping + #self.destinations[self.dst].update({'ans': ans, + # 'unans': unans, + # 'response_packet': ping}) + #return ping + + #return reply

1 0

[ooni-probe/master] * Updated oonib requirements.
by isis＠torproject.org 16 Nov '12

16 Nov '12

commit cd7c33cacf8f8acfac08688318069fa03ae53195 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Nov 15 23:12:36 2012 +0000 * Updated oonib requirements. --- oonib/requirements.txt | 2 -- 1 files changed, 0 insertions(+), 2 deletions(-) diff --git a/oonib/requirements.txt b/oonib/requirements.txt index f9d7541..9c0ab46 100644 --- a/oonib/requirements.txt +++ b/oonib/requirements.txt @@ -14,10 +14,8 @@ numpy==1.6.2 pyOpenSSL==0.13 pygeoip==0.2.5 repoze.sphinx.autointerface==0.7.1 -scapy==2.1.0 storm==0.19 transaction==1.3.0 -txtorcon==0.7 wsgiref==0.1.2 zope.component==4.0.0 zope.event==4.0.0

1 0

[ooni-probe/master] Merge branch 'master' into master-copy
by isis＠torproject.org 16 Nov '12

16 Nov '12

commit 7affab1647a51f639968e5742a09ce74677e579e Merge: 4dd13f7 471312f Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Nov 16 00:33:55 2012 +0000 Merge branch 'master' into master-copy Apply fixes to a couple bugs in ooni.utils.net. ooni/utils/net.py | 94 +++++++++++++++++++++++++++++++++------------------- 1 files changed, 60 insertions(+), 34 deletions(-) diff --cc ooni/utils/net.py index 2583c74,4fdff49..c5b01a3 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@@ -15,38 -19,42 +19,59 @@@ from scapy.all import util from ooni.utils import log, txscapy - #if sys.platformm.system() == 'Windows': + #if sys.platform.system() == 'Windows': # import _winreg as winreg - userAgents = [("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6", "Firefox 2.0, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)", "Internet Explorer 7, Windows Vista"), - ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30)", "Internet Explorer 7, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 6, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 5.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 5, Windows XP"), - ("Opera/9.20 (Windows NT 6.0; U; en)", "Opera 9.2, Windows Vista"), - ("Opera/9.00 (Windows NT 5.1; U; en)", "Opera 9.0, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.50", "Opera 8.5, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.0", "Opera 8.0, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1) Opera 7.02 [en]", "Opera 7.02, Windows XP"), - ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20060127 Netscape/8.1", "Netscape 8.1, Windows XP")] + PLATFORMS = {'LINUX': sys.platform.startswith("linux"), + 'OPENBSD': sys.platform.startswith("openbsd"), + 'FREEBSD': sys.platform.startswith("freebsd"), + 'NETBSD': sys.platform.startswith("netbsd"), + 'DARWIN': sys.platform.startswith("darwin"), + 'SOLARIS': sys.platform.startswith("sunos"), + 'WINDOWS': sys.platform.startswith("win32")} + + userAgents = [ + ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6", "Firefox 2.0, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)", "Internet Explorer 7, Windows Vista"), + ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30)", "Internet Explorer 7, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 6, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 5.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 5, Windows XP"), + ("Opera/9.20 (Windows NT 6.0; U; en)", "Opera 9.2, Windows Vista"), + ("Opera/9.00 (Windows NT 5.1; U; en)", "Opera 9.0, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.50", "Opera 8.5, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.0", "Opera 8.0, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1) Opera 7.02 [en]", "Opera 7.02, Windows XP"), + ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20060127 Netscape/8.1", "Netscape 8.1, Windows XP") + ] + + + class UnsupportedPlatform(Exception): + """Support for this platform is not currently available.""" + + class IfaceError(Exception): + """Could not find default network interface.""" + + class PermissionsError(SystemExit): + """This test requires admin or root privileges to run. Exiting...""" + +PLATFORMS = {'LINUX': sys.platform.startswith("linux"), + 'OPENBSD': sys.platform.startswith("openbsd"), + 'FREEBSD': sys.platform.startswith("freebsd"), + 'NETBSD': sys.platform.startswith("netbsd"), + 'DARWIN': sys.platform.startswith("darwin"), + 'SOLARIS': sys.platform.startswith("sunos"), + 'WINDOWS': sys.platform.startswith("win32")} + +class UnsupportedPlatform(Exception): + """Support for this platform is not currently available.""" + +class IfaceError(Exception): + """Could not find default network interface.""" + +class PermissionsError(SystemExit): + """This test requires admin or root privileges to run. Exiting...""" + class StringProducer(object): implements(IBodyProducer) @@@ -105,26 -114,29 +130,29 @@@ def getClientPlatform(platform_name=Non if test: return name, test - def getPlatformAndIfaces(platform_name=None): -def getPosixIface(): ++def getPosixIfaces(): + from twisted.internet.test import _posixifaces - client, test = getClientPlatform(platform_name) + log.msg("Attempting to discover network interfaces...") + ifaces = _posixifaces._interfaces() + ifup = tryInterfaces(ifaces) + return ifup - def getPosixIfaces(): - from twisted.internet.test import _posixifaces - all_ifaces = _posixifaces._interfaces() - ifup = checkInterfaces(all_ifaces) - return all_ifaces -def getWindowsIface(): ++def getWindowsIfaces(): + from twisted.internet.test import _win32ifaces - def getWindowsIfacse(): - from twisted.internet.test import _win32ifaces - all_ifaces = _win32ifaces._interfaces() - return all_ifaces + log.msg("Attempting to discover network interfaces...") + ifaces = _win32ifaces._interfaces() + ifup = tryInterfaces(ifaces) + return ifup -def getPlatformAndIfaces(platform_name=None): ++def getIfaces(platform_name=None): + client, test = getClientPlatform(platform_name) if client: if client == ('LINUX' or 'DARWIN') or client[-3:] == 'BSD': - all_ifaces = getPosixIfaces() - return getPosixIface() ++ return getPosixIfaces() elif client == 'WINDOWS': - all_ifaces = getWindowsIfaces() - return getWindowsIface() ++ return getWindowsIfaces() ## XXX fixme figure out how to get iface for Solaris else: return None @@@ -146,12 -156,12 +172,12 @@@ def checkInterfaces(ifaces=None, timeou ifup = {} if not ifaces: -- log.debug("net.checkInterfaces(): no interfaces specified!") ++ log.debug("checkInterfaces(): no interfaces specified!") return None for iface in ifaces: for ifname, ifaddr in iface: -- log.debug("net.checkInterfaces(): testing iface {} by pinging" ++ log.debug("checkInterfaces(): testing iface {} by pinging" + " local address {}".format(ifname, ifaddr)) try: pkt = IP(dst=ifaddr)/ICMP() @@@ -160,7 -170,7 +186,7 @@@ raise PermissionsError if e.find("Errno 1") else log.err(e) else: if ans.summary(): -- log.debug("net.checkInterfaces(): got answer on interface %s" ++ log.debug("checkInterfaces(): got answer on interface %s" + ":\n%s".format(ifname, ans.summary())) ifup.update(ifname, ifaddr) else: @@@ -174,7 -184,7 +200,7 @@@ def getNonLoopbackIfaces(platform_name=None): try: -- ifaces = getPlatformAndIfaces(platform_name) ++ ifaces = getIfaces(platform_name) except UnsupportedPlatform, up: log.err(up) @@@ -183,8 -193,8 +209,8 @@@ return None else: found = [{i[0]: i[2]} for i in ifaces if i[0] != 'lo'] -- log.debug("utils.net.getClientIfaces: Found non-loopback interfaces: %s" - % pprint(found)) ++ log.debug("getNonLoopbackIfaces: Found non-loopback interfaces: %s" + % found) try: interfaces = checkInterfaces(found) except IfaceError, ie:

1 0

[ooni-probe/master] * Adding back in a few changes that got lost in an octopus battle.
by isis＠torproject.org 16 Nov '12

16 Nov '12

commit 471312f6e754a7b004761acced8ab462bad803e3 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Nov 15 23:52:04 2012 +0000 * Adding back in a few changes that got lost in an octopus battle. --- ooni/utils/net.py | 59 +++++++++++++++++++++++++++++++++++++++++++++------- 1 files changed, 51 insertions(+), 8 deletions(-) diff --git a/ooni/utils/net.py b/ooni/utils/net.py index 0cbc095..4fdff49 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -19,16 +19,30 @@ from scapy.all import utils from ooni.utils import log, txscapy -#if platformm.system() == 'Windows': +#if sys.platform.system() == 'Windows': # import _winreg as winreg -PLATFORMS = {'LINUX': platform.startswith("linux"), - 'OPENBSD': platform.startswith("openbsd"), - 'FREEBSD': platform.startswith("freebsd"), - 'NETBSD': platform.startswith("netbsd"), - 'DARWIN': platform.startswith("darwin"), - 'SOLARIS': platform.startswith("sunos"), - 'WINDOWS': platform.startswith("win32")} +PLATFORMS = {'LINUX': sys.platform.startswith("linux"), + 'OPENBSD': sys.platform.startswith("openbsd"), + 'FREEBSD': sys.platform.startswith("freebsd"), + 'NETBSD': sys.platform.startswith("netbsd"), + 'DARWIN': sys.platform.startswith("darwin"), + 'SOLARIS': sys.platform.startswith("sunos"), + 'WINDOWS': sys.platform.startswith("win32")} + +userAgents = [ + ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6", "Firefox 2.0, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)", "Internet Explorer 7, Windows Vista"), + ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30)", "Internet Explorer 7, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 6, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 5.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 5, Windows XP"), + ("Opera/9.20 (Windows NT 6.0; U; en)", "Opera 9.2, Windows Vista"), + ("Opera/9.00 (Windows NT 5.1; U; en)", "Opera 9.0, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.50", "Opera 8.5, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.0", "Opera 8.0, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1) Opera 7.02 [en]", "Opera 7.02, Windows XP"), + ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20060127 Netscape/8.1", "Netscape 8.1, Windows XP") + ] class UnsupportedPlatform(Exception): @@ -40,6 +54,35 @@ class IfaceError(Exception): class PermissionsError(SystemExit): """This test requires admin or root privileges to run. Exiting...""" + +class StringProducer(object): + implements(IBodyProducer) + + def __init__(self, body): + self.body = body + self.length = len(body) + + def startProducing(self, consumer): + consumer.write(self.body) + return defer.succeed(None) + + def pauseProducing(self): + pass + + def stopProducing(self): + pass + +class BodyReceiver(protocol.Protocol): + def __init__(self, finished): + self.finished = finished + self.data = "" + + def dataReceived(self, bytes): + self.data += bytes + + def connectionLost(self, reason): + self.finished.callback(self.data) + def capturePackets(pcap_filename): from scapy.all import sniff global stop_packet_capture

1 0

[tor/master] I think it is correct to decorate these addresses.
by nickm＠torproject.org 15 Nov '12

15 Nov '12

commit d276894772cc1ed2cfdcbeb6efd96b1abca3f2dc Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Oct 31 22:18:55 2012 -0400 I think it is correct to decorate these addresses. --- src/or/addressmap.c | 2 +- src/or/connection_edge.c | 4 ++-- src/or/relay.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/or/addressmap.c b/src/or/addressmap.c index 45e29c1..75cfca6 100644 --- a/src/or/addressmap.c +++ b/src/or/addressmap.c @@ -636,7 +636,7 @@ client_dns_set_addressmap(origin_circuit_t *on_circ, if (tor_addr_family(val) != AF_INET) return; - if (! tor_addr_to_str(valbuf, val, sizeof(valbuf), 0)) /* XXXX decorate? */ + if (! tor_addr_to_str(valbuf, val, sizeof(valbuf), 1)) return; client_dns_set_addressmap_impl(on_circ, address, valbuf, exitname, ttl); diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index c14e116..1dd1dbc 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -1344,7 +1344,7 @@ connection_ap_get_original_destination(entry_connection_t *conn, } tor_addr_from_sockaddr(&addr, (struct sockaddr*)&orig_dst, &req->port); - tor_addr_to_str(req->address, &addr, sizeof(req->address), 0); + tor_addr_to_str(req->address, &addr, sizeof(req->address), 1); return 0; #elif defined(TRANS_PF) @@ -1405,7 +1405,7 @@ connection_ap_get_original_destination(entry_connection_t *conn, return -1; } - tor_addr_to_str(req->address, &addr, sizeof(req->address), 0); + tor_addr_to_str(req->address, &addr, sizeof(req->address), 1); req->port = ntohs(pnl.rdport); return 0; diff --git a/src/or/relay.c b/src/or/relay.c index a05e955..e01d933 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -853,7 +853,7 @@ remap_event_helper(entry_connection_t *conn, const tor_addr_t *new_addr) { tor_addr_to_str(conn->socks_request->address, new_addr, sizeof(conn->socks_request->address), - 0); /* XXXXX Should decorate be 1? */ + 1); control_event_stream_status(conn, STREAM_EVENT_REMAP, REMAP_STREAM_SOURCE_EXIT); }

1 0

[tor/master] Better checking of exit policies for connections by hostname
by nickm＠torproject.org 15 Nov '12

15 Nov '12

commit 6e27282dab8f4d4999830f0310ea3f8aa75c6b63 Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Oct 31 22:04:25 2012 -0400 Better checking of exit policies for connections by hostname --- src/or/connection_edge.c | 10 +++++++--- 1 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index e1c6938..c14e116 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -2625,11 +2625,15 @@ connection_ap_can_use_exit(const entry_connection_t *conn, const node_t *exit) } if (conn->socks_request->command == SOCKS_COMMAND_CONNECT) { - struct in_addr in; tor_addr_t addr, *addrp = NULL; addr_policy_result_t r; - if (tor_inet_aton(conn->socks_request->address, &in)) { - tor_addr_from_in(&addr, &in); + if (0 == tor_addr_parse(&addr, conn->socks_request->address)) { + addrp = &addr; + } else if (!conn->ipv4_traffic_ok && conn->ipv6_traffic_ok) { + tor_addr_make_null(&addr, AF_INET6); + addrp = &addr; + } else if (conn->ipv4_traffic_ok && !conn->ipv6_traffic_ok) { + tor_addr_make_null(&addr, AF_INET); addrp = &addr; } r = compare_tor_addr_to_node_policy(addrp, conn->socks_request->port,exit);

1 0

[tor/master] Actually send BEGIN cell flags
by nickm＠torproject.org 15 Nov '12

15 Nov '12

commit 807b781a3d9028db496640541a6168be936ab424 Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Oct 31 22:41:20 2012 -0400 Actually send BEGIN cell flags This uses advertised IPv6 ports as an implicit version check. --- src/or/connection_edge.c | 48 ++++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 48 insertions(+), 0 deletions(-) diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index 2f3ed5d..ead9c49 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -1647,6 +1647,51 @@ connection_ap_supports_optimistic_data(const entry_connection_t *conn) return conn->may_use_optimistic_data; } +/** DOCDOC */ +static uint32_t +connection_ap_get_begincell_flags(entry_connection_t *ap_conn) +{ + edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(ap_conn); + const node_t *exitnode = NULL; + const crypt_path_t *cpath_layer = edge_conn->cpath_layer; + uint32_t flags = 0; + if (ap_conn->use_begindir) + return 0; + + if (edge_conn->on_circuit->purpose != CIRCUIT_PURPOSE_C_GENERAL) + return 0; + + if (ap_conn->ipv4_traffic_ok && !ap_conn->ipv6_traffic_ok) + return 0; + + if (! cpath_layer || + ! cpath_layer->extend_info) + return 0; + + if (!ap_conn->ipv4_traffic_ok) + flags |= BEGIN_FLAG_IPV4_NOT_OK; + + exitnode = node_get_by_id(cpath_layer->extend_info->identity_digest); + + if (ap_conn->ipv6_traffic_ok && exitnode) { + tor_addr_t a; + tor_addr_make_null(&a, AF_INET6); + if (compare_tor_addr_to_node_policy(&a, ap_conn->socks_request->port, + exitnode) + != ADDR_POLICY_REJECTED) { + flags |= BEGIN_FLAG_IPV6_OK; + } + } + + if (flags == BEGIN_FLAG_IPV4_NOT_OK) { + log_warn(LD_BUG, "Hey; I'm about to ask a node for a connection that I " + "am telling it to fulfil with neither IPv4 nor IPv6. That's " + "probably not going to work."); + } + + return flags; +} + /** Write a relay begin cell, using destaddr and destport from ap_conn's * socks_request field, and send it down circ. * @@ -1682,6 +1727,9 @@ connection_ap_handshake_send_begin(entry_connection_t *ap_conn) return -1; } + /* Set up begin cell flags. */ + edge_conn->begincell_flags = connection_ap_get_begincell_flags(ap_conn); + tor_snprintf(payload,RELAY_PAYLOAD_SIZE, "%s:%d", (circ->base_.purpose == CIRCUIT_PURPOSE_C_GENERAL) ? ap_conn->socks_request->address : "",

1 0