November 2012 - tor-commits - lists.torproject.org

[torbutton/master] Update unsigned rdf.
by mikeperry＠torproject.org 16 Nov '12

16 Nov '12

commit 1cbd9114e9c9dcdd428b41365e42becac0ce1464 Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Thu Nov 15 15:29:20 2012 -0800 Update unsigned rdf. --- update-unsigned.rdf | 16 +++++++++++++++- 1 files changed, 15 insertions(+), 1 deletions(-) diff --git a/update-unsigned.rdf b/update-unsigned.rdf index f7157f1..d4a013a 100644 --- a/update-unsigned.rdf +++ b/update-unsigned.rdf @@ -31,6 +31,7 @@ <li resource="urn:mozilla:extension:{e0204bd5-9d31-402b-a99d-a6aa8ffebdca}:1.4.6"/> <li resource="urn:mozilla:extension:{e0204bd5-9d31-402b-a99d-a6aa8ffebdca}:1.4.6.1"/> <li resource="urn:mozilla:extension:{e0204bd5-9d31-402b-a99d-a6aa8ffebdca}:1.4.6.2"/> + <li resource="urn:mozilla:extension:{e0204bd5-9d31-402b-a99d-a6aa8ffebdca}:1.4.6.3"/> </Seq> </em:updates> @@ -374,5 +375,18 @@ </Description> </em:targetApplication> </Description> - + + <Description about="urn:mozilla:extension:{e0204bd5-9d31-402b-a99d-a6aa8ffebdca}:1.4.6.3"> + <em:version>1.4.6.3</em:version> + <em:targetApplication> + <Description> + <em:id>{ec8030f7-c20a-464f-9b0e-13a3a9e97384}</em:id> + <em:minVersion>0.9</em:minVersion> + <em:maxVersion>10000.0</em:maxVersion> + <em:updateLink>https://www.torproject.org/dist/torbutton/torbutton-1.4.6.3.xpi</em:updateLink> + <em:updateHash>sha1:968455a5b702b3cabb6c16499e531a5a8f4c5f21</em:updateHash> + </Description> + </em:targetApplication> + </Description> + </RDF>

1 0

[torbutton/master] Bug 6431: Add menu hint to Torbutton icon
by mikeperry＠torproject.org 16 Nov '12

16 Nov '12

commit 9a096a9ce8a0f126b10753bfe436e8a84bd80e96 Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Thu Nov 15 19:44:36 2012 -0800 Bug 6431: Add menu hint to Torbutton icon Patch from Pearl Crescent. --- src/chrome/content/torbutton.xul | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/src/chrome/content/torbutton.xul b/src/chrome/content/torbutton.xul index df42e6b..f851e3c 100644 --- a/src/chrome/content/torbutton.xul +++ b/src/chrome/content/torbutton.xul @@ -31,6 +31,8 @@ <toolbarbutton id="torbutton-button" class="toolbarbutton-1 chromeclass-toolbar-additional" + type="menu" + orient="horizontal" label="Torbutton" tooltiptext="&torbutton.button.tooltip;" menu="torbutton-context-menu"

1 0

[torbutton/master] Bug 6156: Rate limit version check queries
by mikeperry＠torproject.org 16 Nov '12

16 Nov '12

commit 1581678d45093f9ac08dd9e20152493a8fdaeddb Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Thu Nov 15 19:45:41 2012 -0800 Bug 6156: Rate limit version check queries Patch from Pearl Crescent. --- src/chrome/content/torbutton.js | 17 +++++++++++++++++ 1 files changed, 17 insertions(+), 0 deletions(-) diff --git a/src/chrome/content/torbutton.js b/src/chrome/content/torbutton.js index e629102..e305e4e 100644 --- a/src/chrome/content/torbutton.js +++ b/src/chrome/content/torbutton.js @@ -967,6 +967,23 @@ function torbutton_do_async_versioncheck() { if (!m_tb_tbb || !m_tb_prefs.getBoolPref("extensions.torbutton.versioncheck_enabled")) { return; } + + // Suppress update check if done recently. + const kLastCheckPref = "extension.torbutton.lastUpdateCheck"; + const kMinSecsBetweenChecks = 15 * 60; // 15 minutes + var now = Date.now() / 1000; + var lastCheckTime; + try { + lastCheckTime = parseFloat(m_tb_prefs.getCharPref(kLastCheckPref)); + if (isNaN(lastCheckTime)) + lastCheckTime = undefined; + } catch (e) {} + + if (lastCheckTime && ((now - lastCheckTime) < kMinSecsBetweenChecks)) + return; + + m_tb_prefs.setCharPref(kLastCheckPref, now); + torbutton_log(3, "Checking version with socks port: " +m_tb_prefs.getIntPref("extensions.torbutton.socks_port")); try {

1 0

[tor/master] Allow IPv4 traffic on default and old-style-config SocksPorts.
by nickm＠torproject.org 16 Nov '12

16 Nov '12

commit 28cbe908398f24366a9056d2508ae94690d82bba Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Nov 15 22:49:43 2012 -0500 Allow IPv4 traffic on default and old-style-config SocksPorts. Looks like when i was writing the code to set the ipv4_traffic flag on port_cfg_t, I missed some cases, such as the one where the port was set from its default value. Fix for 7493. Bug not in any released Tor. --- src/or/config.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diff --git a/src/or/config.c b/src/or/config.c index e069c7c..f7a803f 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -4543,6 +4543,7 @@ parse_port_config(smartlist_t *out, tor_addr_make_unspec(&cfg->addr); /* Server ports default to 0.0.0.0 */ cfg->no_listen = 1; cfg->bind_ipv4_only = 1; + cfg->ipv4_traffic = 1; smartlist_add(out, cfg); } @@ -4562,6 +4563,7 @@ parse_port_config(smartlist_t *out, cfg->session_group = SESSION_GROUP_UNSET; cfg->isolation_flags = ISO_DEFAULT; cfg->no_advertise = 1; + cfg->ipv4_traffic = 1; smartlist_add(out, cfg); } } @@ -4585,6 +4587,7 @@ parse_port_config(smartlist_t *out, tor_addr_parse(&cfg->addr, defaultaddr); cfg->session_group = SESSION_GROUP_UNSET; cfg->isolation_flags = ISO_DEFAULT; + cfg->ipv4_traffic = 1; smartlist_add(out, cfg); } return 0;

1 0

[ooni-probe/master] Merge branch 'master' of git-rw.torproject.org:ooni-probe
by isis＠torproject.org 16 Nov '12

16 Nov '12

commit bab98e2564892dcf7d7e052700b71b0b0d75e874 Merge: 1008bd5 82f72cf Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Nov 15 10:40:07 2012 +0000 Merge branch 'master' of git-rw.torproject.org:ooni-probe before_i_commit.sh | 4 +- inputs/Makefile | 8 +++ nettests/core/daphn3.py | 12 ++-- ooni/nettest.py | 25 +++----- ooni/runner.py | 2 +- test_inputs/daph3_yaml_file.txt | 2 + to-be-ported/very-old/traceroute.py | 108 ----------------------------------- 7 files changed, 29 insertions(+), 132 deletions(-)

1 0

[ooni-probe/master] Merge branch 'master' of git-rw.torproject.org:ooni-probe
by isis＠torproject.org 16 Nov '12

16 Nov '12

commit 1008bd5c04e9a9fa1f1726fa5474725d2714f37c Merge: 870d882 2bf3da5 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Nov 15 00:26:43 2012 +0000 Merge branch 'master' of git-rw.torproject.org:ooni-probe Conflicts: ooni/utils/net.py AUTHORS | 2 +- CHANGES.yaml | 5 - HACKING | 141 ++++++------ INSTALL | 1 - README.md | 123 +++++------ TODO | 105 +-------- before_i_commit.sh | 2 +- bin/oonib | 41 +++- bin/ooniprobe | 5 +- docs/source/index.rst | 51 +---- docs/source/install.rst | 119 ---------- nettests/bridge_reachability/echo.py | 2 +- nettests/core/captiveportal.py | 23 +- nettests/core/chinatrigger.py | 9 +- nettests/core/daphn3.py | 117 +++++++++ nettests/core/http_host.py | 74 +++++- nettests/core/http_requests.py | 118 +++++++++ nettests/core/http_uk_mobile_networks.py | 85 +++++++ nettests/core/keyword_filtering.py | 52 ++++ nettests/core/squid.py | 2 +- nettests/core/tcpconnect.py | 46 ++++ nettests/core/traceroute.py | 113 +++++++++ nettests/core/url_list.py | 59 ++++- nettests/examples/example_scapyt.py | 22 ++- nettests/simpletest.py | 27 -- ooni/__init__.py | 4 + ooni/config.py | 20 ++- ooni/inputunit.py | 19 +- ooni/kit/daphn3.py | 208 ++++++++++++++++ ooni/lib/rfc3339.py | 283 ---------------------- ooni/lib/txscapy.py | 381 ------------------------------ ooni/nettest.py | 38 ++-- ooni/nodes.py | 24 +- ooni/ooni-probe.conf | 100 -------- ooni/oonicli.py | 67 ++++-- ooni/protocols/daphn3.py | 311 ------------------------ ooni/reporter.py | 340 +++++++++++++++------------ ooni/runner.py | 117 ++++++---- ooni/templates/httpt.py | 120 ++++------ ooni/templates/scapyt.py | 124 +++-------- ooni/utils/__init__.py | 102 +-------- ooni/utils/date.py | 30 --- ooni/utils/geodata.py | 31 ++- ooni/utils/hacks.py | 75 ++++--- ooni/utils/log.py | 26 ++- ooni/utils/net.py | 146 ++++++++---- ooni/utils/otime.py | 8 + ooni/utils/txscapy.py | 140 +++++++++++ oonib/__init__.py | 22 ++ oonib/config.py | 43 ++++ oonib/db/__init__.py | 30 +++ oonib/db/tables.py | 123 ++++++++++ oonib/lib/__init__.py | 8 - oonib/lib/ssl.py | 8 - oonib/models.py | 122 ++++++++++ oonib/oonibackend.conf.sample | 12 - oonib/oonibackend.py | 117 +++++----- oonib/report/api.py | 164 ++++++++++++- oonib/report/db/__init__.py | 33 --- oonib/requirements.txt | 24 ++ oonib/runner.py | 25 ++ oonib/testhelpers/__init__.py | 5 + oonib/testhelpers/daphn3.py | 47 ---- oonib/testhelpers/dns.py | 16 -- oonib/testhelpers/dns_helpers.py | 16 ++ oonib/testhelpers/http_helpers.py | 80 +++++++ oonib/testhelpers/httph.py | 92 ------- oonib/testhelpers/ssl_helpers.py | 9 + oonib/testhelpers/tcp_helpers.py | 72 ++++++ ooniprobe.conf | 6 +- requirements.txt | 45 ++-- tests/assets/ipports.txt | 2 - tests/assets/urllist.txt | 3 - tests/legacy/test_plugins.py | 40 --- tests/legacy/test_tests.py | 45 ---- tests/test_mutate.py | 15 ++ tests/test_worker.py | 29 --- to-be-ported/old-api/daphn3.py | 152 ------------ to-be-ported/old-api/httpt.py | 94 -------- to-be-ported/old-api/tcpconnect.py | 65 ----- 80 files changed, 2696 insertions(+), 2926 deletions(-) diff --cc ooni/utils/net.py index 4d8907d,3ddba61..2583c74 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@@ -2,169 -2,91 +2,221 @@@ # # net.py # -------- - # OONI utilities for network infrastructure and hardware. - # - # :authors: Isis Lovecruft, Arturo Filasto - # :version: 0.0.1-pre-alpha - # :license: (c) 2012 Isis Lovecruft, Arturo Filasto - # see attached LICENCE file + # OONI utilities for networking related operations - from pprint import pprint - from sys import platform + import sys -from zope.interface import implements - #if platformm.system() == 'Windows': - # import _winreg as winreg ++from zope.interface import implements + from twisted.internet import protocol, defer + from twisted.internet import threads, reactor + from twisted.web.iweb import IBodyProducer - from ooni.utils import log + from scapy.all import utils - PLATFORMS = {'LINUX': platform.startswith("linux"), - 'OPENBSD': platform.startswith("openbsd"), - 'FREEBSD': platform.startswith("freebsd"), - 'NETBSD': platform.startswith("netbsd"), - 'DARWIN': platform.startswith("darwin"), - 'SOLARIS': platform.startswith("sunos"), - 'WINDOWS': platform.startswith("win32")} + from ooni.utils import log, txscapy ++#if sys.platformm.system() == 'Windows': ++# import _winreg as winreg + - class PlatformNameException(Exception): - """Specified platform does not match client platform.""" + userAgents = [("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6", "Firefox 2.0, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)", "Internet Explorer 7, Windows Vista"), + ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30)", "Internet Explorer 7, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 6, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 5.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 5, Windows XP"), + ("Opera/9.20 (Windows NT 6.0; U; en)", "Opera 9.2, Windows Vista"), + ("Opera/9.00 (Windows NT 5.1; U; en)", "Opera 9.0, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.50", "Opera 8.5, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.0", "Opera 8.0, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1) Opera 7.02 [en]", "Opera 7.02, Windows XP"), + ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20060127 Netscape/8.1", "Netscape 8.1, Windows XP")] + ++PLATFORMS = {'LINUX': sys.platform.startswith("linux"), ++ 'OPENBSD': sys.platform.startswith("openbsd"), ++ 'FREEBSD': sys.platform.startswith("freebsd"), ++ 'NETBSD': sys.platform.startswith("netbsd"), ++ 'DARWIN': sys.platform.startswith("darwin"), ++ 'SOLARIS': sys.platform.startswith("sunos"), ++ 'WINDOWS': sys.platform.startswith("win32")} + +class UnsupportedPlatform(Exception): + """Support for this platform is not currently available.""" + +class IfaceError(Exception): + """Could not find default network interface.""" + +class PermissionsError(SystemExit): + """This test requires admin or root privileges to run. Exiting...""" + - def getClientAddress(): - address = {'asn': 'REPLACE_ME', - 'ip': 'REPLACE_ME'} - return address + class StringProducer(object): + implements(IBodyProducer) + + def __init__(self, body): + self.body = body + self.length = len(body) + + def startProducing(self, consumer): + consumer.write(self.body) + return defer.succeed(None) + + def pauseProducing(self): + pass + + def stopProducing(self): + pass + + class BodyReceiver(protocol.Protocol): + def __init__(self, finished): + self.finished = finished + self.data = "" + + def dataReceived(self, bytes): + self.data += bytes + + def connectionLost(self, reason): + self.finished.callback(self.data) + + def capturePackets(pcap_filename): + from scapy.all import sniff + global stop_packet_capture + stop_packet_capture = False + + def stopCapture(): + # XXX this is a bit of a hack to stop capturing packets when we close + # the reactor. Ideally we would want to be able to do this + # programmatically, but this requires some work on implementing + # properly the sniff function with deferreds. + global stop_packet_capture + stop_packet_capture = True + + def writePacketToPcap(pkt): + from scapy.all import utils + pcapwriter = txscapy.TXPcapWriter(pcap_filename, append=True) + pcapwriter.write(pkt) + if stop_packet_capture: + sys.exit(1) - + d = threads.deferToThread(sniff, lfilter=writePacketToPcap) + reactor.addSystemEventTrigger('before', 'shutdown', stopCapture) + return d + -class PermissionsError(SystemExit): - def __init__(self, *args, **kwargs): - if not args and not kwargs: - pe = "This test requires admin or root privileges to run. Exiting..." - super(PermissionsError, self).__init__(pe, *args, **kwargs) - else: - super(PermissionsError, self).__init__(*args, **kwargs) -class IfaceError(SystemExit): - def __init__(self, *args, **kwargs): - super(IfaceError, self).__init__(*args, **kwargs) +def getClientPlatform(platform_name=None): + for name, test in PLATFORMS.items(): + if not platform_name or platform_name.upper() == name: + if test: + return name, test + - def getPosixIface(): - from twisted.internet.test import _posixifaces ++def getPlatformAndIfaces(platform_name=None): + - log.msg("Attempting to discover network interfaces...") - ifaces = _posixifaces._interfaces() - ifup = tryInterfaces(ifaces) - return ifup ++ client, test = getClientPlatform(platform_name) + - def getWindowsIface(): - from twisted.internet.test import _win32ifaces ++ def getPosixIfaces(): ++ from twisted.internet.test import _posixifaces ++ all_ifaces = _posixifaces._interfaces() ++ ifup = checkInterfaces(all_ifaces) ++ return all_ifaces + - log.msg("Attempting to discover network interfaces...") - ifaces = _win32ifaces._interfaces() - ifup = tryInterfaces(ifaces) - return ifup ++ def getWindowsIfacse(): ++ from twisted.internet.test import _win32ifaces ++ all_ifaces = _win32ifaces._interfaces() ++ return all_ifaces + - def getPlatformAndIfaces(platform_name=None): - client, test = getClientPlatform(platform_name) + if client: + if client == ('LINUX' or 'DARWIN') or client[-3:] == 'BSD': - return getPosixIface() ++ all_ifaces = getPosixIfaces() + elif client == 'WINDOWS': - return getWindowsIface() ++ all_ifaces = getWindowsIfaces() + ## XXX fixme figure out how to get iface for Solaris + else: + return None ++ ifup = checkInterfaces(all_ifaces) ++ return ifup + else: + raise UnsupportedPlatform + +def checkInterfaces(ifaces=None, timeout=1): + """ + @param ifaces: + A dictionary in the form of ifaces['if_name'] = 'if_addr'. + """ + try: + from scapy.all import IP, ICMP + from scapy.all import sr1 ## we want this check to be blocking + except: + log.msg(("Scapy required: www.secdev.org/projects/scapy")) + + ifup = {} + if not ifaces: + log.debug("net.checkInterfaces(): no interfaces specified!") + return None + + for iface in ifaces: + for ifname, ifaddr in iface: + log.debug("net.checkInterfaces(): testing iface {} by pinging" + + " local address {}".format(ifname, ifaddr)) + try: + pkt = IP(dst=ifaddr)/ICMP() + ans, unans = sr(pkt, iface=ifname, timeout=5, retry=3) + except Exception, e: + raise PermissionsError if e.find("Errno 1") else log.err(e) + else: + if ans.summary(): + log.debug("net.checkInterfaces(): got answer on interface %s" + + ":\n%s".format(ifname, ans.summary())) + ifup.update(ifname, ifaddr) + else: + log.debug("Interface test packet was unanswered:\n%s" + % unans.summary()) + if len(ifup) > 0: + log.msg("Discovered working network interfaces: %s" % ifup) + return ifup + else: + raise IfaceError + +def getNonLoopbackIfaces(platform_name=None): + try: + ifaces = getPlatformAndIfaces(platform_name) + except UnsupportedPlatform, up: + log.err(up) + + if not ifaces: + log.msg("Unable to discover network interfaces...") + return None + else: + found = [{i[0]: i[2]} for i in ifaces if i[0] != 'lo'] + log.debug("utils.net.getClientIfaces: Found non-loopback interfaces: %s" - % pprint(found)) ++ % found) + try: + interfaces = checkInterfaces(found) + except IfaceError, ie: + log.err(ie) + return None + else: + return interfaces + +def getNetworksFromRoutes(): - from scapy.all import conf, ltoa ++ from scapy.all import conf, ltoa, read_routes + from ipaddr import IPNetwork, IPAddress + + ## Hide the 'no routes' warnings + conf.verb = 0 + + networks = [] - client = conf.route - log.debug("Local Routing Table:\n{}".format(client)) - - for nw, nm, gw, iface, addr in client.routes: ++ for nw, nm, gw, iface, addr in read_routes(): + n = IPNetwork( ltoa(nw) ) + (n.netmask, n.gateway, n.ipaddr) = [IPAddress(x) for x in [nm, gw, addr]] + n.iface = iface + if not n.compressed in networks: + networks.append(n) + + return networks + +def getDefaultIface(): + networks = getNetworksFromRoutes() + for net in networks: + if net.is_private: - return net ++ return net.iface + raise IfaceError + +def getLocalAddress(): + default_iface = getDefaultIface() + return default_iface.ipaddr

1 0

[ooni-probe/master] * Moved iface and ipaddr discovery function to ooni.utils.net.
by isis＠torproject.org 16 Nov '12

16 Nov '12

commit 870d8824f0d2fd8c83624a908641f7ba512bcc1f Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun Nov 11 13:56:09 2012 +0000 * Moved iface and ipaddr discovery function to ooni.utils.net. --- nettests/bridge_reachability/echo.py | 26 ------ ooni/utils/net.py | 159 ++++++++++++++++++++++++++++++++-- 2 files changed, 150 insertions(+), 35 deletions(-) diff --git a/nettests/bridge_reachability/echo.py b/nettests/bridge_reachability/echo.py index 542e017..d2aa721 100644 --- a/nettests/bridge_reachability/echo.py +++ b/nettests/bridge_reachability/echo.py @@ -59,41 +59,15 @@ class EchoTest(BaseScapyTest): Struct returned from getifaddrs(3) and turned into a tuple in the form (*ifa_name, AF_FAMILY, *ifa_addr) ''' - if self.localOptions: - log.debug("%s: local_options found" % self.name) for key, value in self.localOptions.items(): log.debug("setting self.%s = %s" % (key, value)) setattr(self, key, value) - ## xxx is this now .subOptions? - #self.inputFile = self.localOptions['file'] self.timeout *= 1000 ## convert to milliseconds if not self.interface: log.msg("No network interface specified!") - log.debug("OS detected: %s" % sys.platform) - if LINUX or OPENBSD or NETBSD or FREEBSD or DARWIN or SOLARIS: - from twisted.internet.test import _posixifaces - log.msg("Attempting to discover network interfaces...") - ifaces = _posixifaces._interfaces() - elif WINDOWS: - from twisted.internet.test import _win32ifaces - log.msg("Attempting to discover network interfaces...") - ifaces = _win32ifaces._interfaces() - else: - log.debug("Client OS %s not accounted for!" % sys.platform) - log.debug("Unable to discover network interfaces...") - ifaces = [('lo', '')] - - ## found = {'eth0': '1.1.1.1'} - found = [{i[0]: i[2]} for i in ifaces if i[0] != 'lo'] - log.info("Found interfaces:\n%s" % pprint(found)) - self.interfaces = self.tryInterfaces(found) - else: - ## xxx need a way to check that iface exists, is up, and - ## we have permissions on it - log.debug("Our interface has been set to %s" % self.interface) if self.pcap: try: diff --git a/ooni/utils/net.py b/ooni/utils/net.py index 46c29e5..4d8907d 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -9,21 +9,162 @@ # :license: (c) 2012 Isis Lovecruft, Arturo Filasto # see attached LICENCE file +from pprint import pprint +from sys import platform + +#if platformm.system() == 'Windows': +# import _winreg as winreg + +from ooni.utils import log + +PLATFORMS = {'LINUX': platform.startswith("linux"), + 'OPENBSD': platform.startswith("openbsd"), + 'FREEBSD': platform.startswith("freebsd"), + 'NETBSD': platform.startswith("netbsd"), + 'DARWIN': platform.startswith("darwin"), + 'SOLARIS': platform.startswith("sunos"), + 'WINDOWS': platform.startswith("win32")} + + +class PlatformNameException(Exception): + """Specified platform does not match client platform.""" + +class UnsupportedPlatform(Exception): + """Support for this platform is not currently available.""" + +class IfaceError(Exception): + """Could not find default network interface.""" + +class PermissionsError(SystemExit): + """This test requires admin or root privileges to run. Exiting...""" def getClientAddress(): address = {'asn': 'REPLACE_ME', 'ip': 'REPLACE_ME'} return address +def getClientPlatform(platform_name=None): + for name, test in PLATFORMS.items(): + if not platform_name or platform_name.upper() == name: + if test: + return name, test -class PermissionsError(SystemExit): - def __init__(self, *args, **kwargs): - if not args and not kwargs: - pe = "This test requires admin or root privileges to run. Exiting..." - super(PermissionsError, self).__init__(pe, *args, **kwargs) +def getPosixIface(): + from twisted.internet.test import _posixifaces + + log.msg("Attempting to discover network interfaces...") + ifaces = _posixifaces._interfaces() + ifup = tryInterfaces(ifaces) + return ifup + +def getWindowsIface(): + from twisted.internet.test import _win32ifaces + + log.msg("Attempting to discover network interfaces...") + ifaces = _win32ifaces._interfaces() + ifup = tryInterfaces(ifaces) + return ifup + +def getPlatformAndIfaces(platform_name=None): + client, test = getClientPlatform(platform_name) + if client: + if client == ('LINUX' or 'DARWIN') or client[-3:] == 'BSD': + return getPosixIface() + elif client == 'WINDOWS': + return getWindowsIface() + ## XXX fixme figure out how to get iface for Solaris + else: + return None + else: + raise UnsupportedPlatform + +def checkInterfaces(ifaces=None, timeout=1): + """ + @param ifaces: + A dictionary in the form of ifaces['if_name'] = 'if_addr'. + """ + try: + from scapy.all import IP, ICMP + from scapy.all import sr1 ## we want this check to be blocking + except: + log.msg(("Scapy required: www.secdev.org/projects/scapy")) + + ifup = {} + if not ifaces: + log.debug("net.checkInterfaces(): no interfaces specified!") + return None + + for iface in ifaces: + for ifname, ifaddr in iface: + log.debug("net.checkInterfaces(): testing iface {} by pinging" + + " local address {}".format(ifname, ifaddr)) + try: + pkt = IP(dst=ifaddr)/ICMP() + ans, unans = sr(pkt, iface=ifname, timeout=5, retry=3) + except Exception, e: + raise PermissionsError if e.find("Errno 1") else log.err(e) + else: + if ans.summary(): + log.debug("net.checkInterfaces(): got answer on interface %s" + + ":\n%s".format(ifname, ans.summary())) + ifup.update(ifname, ifaddr) + else: + log.debug("Interface test packet was unanswered:\n%s" + % unans.summary()) + if len(ifup) > 0: + log.msg("Discovered working network interfaces: %s" % ifup) + return ifup + else: + raise IfaceError + +def getNonLoopbackIfaces(platform_name=None): + try: + ifaces = getPlatformAndIfaces(platform_name) + except UnsupportedPlatform, up: + log.err(up) + + if not ifaces: + log.msg("Unable to discover network interfaces...") + return None + else: + found = [{i[0]: i[2]} for i in ifaces if i[0] != 'lo'] + log.debug("utils.net.getClientIfaces: Found non-loopback interfaces: %s" + % pprint(found)) + try: + interfaces = checkInterfaces(found) + except IfaceError, ie: + log.err(ie) + return None else: - super(PermissionsError, self).__init__(*args, **kwargs) + return interfaces + +def getNetworksFromRoutes(): + from scapy.all import conf, ltoa + from ipaddr import IPNetwork, IPAddress + + ## Hide the 'no routes' warnings + conf.verb = 0 + + networks = [] + client = conf.route + log.debug("Local Routing Table:\n{}".format(client)) + + for nw, nm, gw, iface, addr in client.routes: + n = IPNetwork( ltoa(nw) ) + (n.netmask, n.gateway, n.ipaddr) = [IPAddress(x) for x in [nm, gw, addr]] + n.iface = iface + if not n.compressed in networks: + networks.append(n) + + return networks + +def getDefaultIface(): + networks = getNetworksFromRoutes() + for net in networks: + if net.is_private: + return net + raise IfaceError -class IfaceError(SystemExit): - def __init__(self, *args, **kwargs): - super(IfaceError, self).__init__(*args, **kwargs) +def getLocalAddress(): + default_iface = getDefaultIface() + return default_iface.ipaddr

1 0

[ooni-probe/master] * Minor change to Makefile.
by isis＠torproject.org 16 Nov '12

16 Nov '12

commit e6fa66090d9a435f5e9c77ffa2e7376fc053e932 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Nov 15 23:06:57 2012 +0000 * Minor change to Makefile. --- Makefile | 28 ++++++++++++++++------------ 1 files changed, 16 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index a965c00..13d277e 100644 --- a/Makefile +++ b/Makefile @@ -158,14 +158,16 @@ libdnet: scapy-all: scapy pypcap libdnet -tags-recursive: - list='$(SUBDIRS)'; for subdir in $$list; do \ - test "$$subdir" = . || (cd $$subdir && make tags); \ - done - ETAGS = etags ETAGSFLAGS = ETAGSARGS = +## or change to ${SUBDIRS} +TAGME = ${topsrc_dir} + +tags-recursive: + list='$(TAGME)'; for subdir in $$list; do \ + test "$$subdir" = . || (cd $$subdir && make tags); \ + done tags: TAGS @@ -182,17 +184,19 @@ TAGS: tags-recursive $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \ $(TAGS_FILES) $(LISP) tags=; \ here=`pwd`; \ - list='$(SUBDIRS)'; for subdir in $$list; do \ + echo ${E_OR_V}; \ + list='$(TAGME)'; for subdir in $$list; do \ if test "$$subdir" = .; then :; else \ test -f $$subdir/TAGS && tags="$$tags -i $$here/$$subdir/TAGS"; \ fi; \ done; \ list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ unique=`for i in $$list; do \ - if test -f "$$i"; fi; \ - done | \ - $(AWK) ' { files[$$0] = 1; } \ - END { for (i in files) print i; }'`; \ + if test -f "$$i"; then echo $$i; fi; \ + done;` test -z "$(ETAGS_ARGS)$$tags$$unique" \ - || $(ETAGS) $(ETAGSFLAGS) $(ETAGS_ARGS) \ - $$tags $$unique + || $(ETAGS) $(ETAGSFLAGS) $(ETAGS_ARGS) $$tags $$unique + +awkward: + | $(AWK) ' { files[$$0] = 1; } \ + END { for (i in files) print i; }'`; \

1 0

[ooni-probe/master] * Rewrite of echo.
by isis＠torproject.org 16 Nov '12

16 Nov '12

commit 56ae275afcd3ca2517dd9e8ff8ec6b83448c34a6 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Nov 15 23:09:46 2012 +0000 * Rewrite of echo. --- nettests/bridge_reachability/echo.py | 176 ++++++++++++++++------------------ 1 files changed, 83 insertions(+), 93 deletions(-) diff --git a/nettests/bridge_reachability/echo.py b/nettests/bridge_reachability/echo.py index 5198766..40436e7 100644 --- a/nettests/bridge_reachability/echo.py +++ b/nettests/bridge_reachability/echo.py @@ -6,59 +6,57 @@ # +---------+ # A simple ICMP-8 ping test. # -# :author: Isis Lovecruft -# :version: 0.0.1-pre-alpha -# :license: (c) 2012 Isis Lovecruft +# @authors: Isis Lovecruft, <isis(a)torproject.org> +# @version: 0.0.2-pre-alpha +# @license: copyright (c) 2012 Isis Lovecruft # see attached LICENCE file # import os import sys -from pprint import pprint - -from twisted.internet import reactor -from twisted.plugin import IPlugin from twisted.python import usage -from ooni.nettest import NetTestCase -from ooni.utils import log, Storage -from ooni.utils.net import PermissionsError, IfaceError +from twisted.internet import reactor, defer +from ooni import nettest +from ooni.utils import log, net, Storage try: - from scapy.all import sr1, IP, ICMP ## XXX v4/v6? + from scapy.all import IP, ICMP + from scapy.all import sr1 from ooni.lib import txscapy from ooni.lib.txscapy import txsr, txsend from ooni.templates.scapyt import BaseScapyTest except: log.msg("This test requires scapy, see www.secdev.org/projects/scapy") -class EchoTest(BaseScapyTest): +class UsageOptions(usage.Options): + optParameters = [ + ['dst', 'd', None, 'Host IP to ping'], + ['file', 'f', None, 'File of list of IPs to ping'], + ['interface', 'i', None, 'Network interface to use'], + ['count', 'c', 1, 'Number of packets to send', int], + ['size', 's', 56, 'Number of bytes to send in ICMP data field', int], + ['ttl', 'l', 25, 'Set the IP Time to Live', int], + ['timeout', 't', 2, 'Seconds until timeout if no response', int], + ['pcap', 'p', None, 'Save pcap to this file'], + ['receive', 'r', True, 'Receive response packets']] + +class EchoTest(nettest.NetTestCase): """ xxx fill me in """ name = 'echo' author = 'Isis Lovecruft <isis(a)torproject.org>' - description = 'A simple ICMP-8 test to see if a host is reachable.' - version = '0.0.1' - inputFile = ['file', 'f', None, 'File of list of IPs to ping'] + description = 'A simple ping test to see if a host is reachable.' + version = '0.0.2' requiresRoot = True - optParameters = [ - ['interface', 'i', None, 'Network interface to use'], - ['count', 'c', 5, 'Number of packets to send', int], - ['size', 's', 56, 'Number of bytes to send in ICMP data field', int], - ['ttl', 'l', 25, 'Set the IP Time to Live', int], - ['timeout', 't', 2, 'Seconds until timeout if no response', int], - ['pcap', 'p', None, 'Save pcap to this file'], - ['receive', 'r', True, 'Receive response packets'] - ] + usageOptions = UsageOptions + #requiredOptions = ['dst'] def setUp(self, *a, **kw): - ''' - :ivar ifaces: - Struct returned from getifaddrs(3) and turned into a tuple in the - form (*ifa_name, AF_FAMILY, *ifa_addr) - ''' + self.destinations = {} + if self.localOptions: for key, value in self.localOptions.items(): log.debug("setting self.%s = %s" % (key, value)) @@ -67,76 +65,68 @@ class EchoTest(BaseScapyTest): self.timeout *= 1000 ## convert to milliseconds if not self.interface: - log.msg("No network interface specified!") + try: + iface = net.getDefaultIface() + except Exception, e: + log.msg("No network interface specified!") + log.err(e) + else: + log.msg("Using system default interface: %s" % iface) + self.interface = iface if self.pcap: try: self.pcapfile = open(self.pcap, 'a+') except: log.msg("Unable to write to pcap file %s" % self.pcap) - self.pcapfile = None - - try: - assert os.path.isfile(self.file) - fp = open(self.file, 'r') - except Exception, e: - hosts = ['8.8.8.8', '38.229.72.14'] - log.err(e) - else: - self.inputs = self.inputProcessor(fp) - self.removePorts(hosts) - - log.debug("Initialization of %s test completed with:\n%s" - % (self.name, ''.join(self.__dict__))) - - @staticmethod - def inputParser(self, one_input): - log.debug("Removing possible ports from host addresses...") - log.debug("Initial inputs:\n%s" % pprint(inputs)) - - #host = [h.rsplit(':', 1)[0] for h in inputs] - host = h.rsplit(':', 1)[0] - log.debug("Inputs converted to:\n%s" % hosts) - - return host - - def tryInterfaces(self, ifaces): - try: - from scapy.all import sr1 ## we want this check to be blocking - except: - log.msg("This test requires scapy: www.secdev.org/projects/scapy") - raise SystemExit - - ifup = {} - while ifaces: - for ifname, ifaddr in ifaces: - log.debug("Currently testing network capabilities of interface" - + "%s by sending a packet to our address %s" - % (ifname, ifaddr)) - try: - pkt = IP(dst=ifaddr)/ICMP() - ans, unans = sr(pkt, iface=ifname, timeout=self.timeout) - except Exception, e: - raise PermissionsError if e.find("Errno 1") else log.err(e) - else: - ## xxx i think this logic might be wrong - log.debug("Interface test packet\n%s\n\n%s" - % (pkt.summary(), pkt.show2())) - if ans.summary(): - log.info("Received answer for test packet on interface" - +"%s :\n%s" % (ifname, ans.summary())) - ifup.update(ifname, ifaddr) - else: - log.info("Our interface test packet was unanswered:\n%s" - % unans.summary()) - - if len(ifup) > 0: - log.msg("Discovered the following working network interfaces: %s" - % ifup) - return ifup + else: + self.pcap = net.capturePacket(self.pcapfile) + + if not self.dst: + if self.file: + self.dstProcessor(self.file) + for key, value in self.destinations.items(): + for label, data in value.items(): + if not 'ans' in data: + self.dst = label else: - raise IfaceError("Could not find a working network interface.") + self.addDest(self.dst) + log.debug("self.dst is now: %s" % self.dst) - def test_icmp(self): - return self.sr(IP(dst=self.input)/ICMP()) + log.debug("Initialization of %s test completed." % self.name) + + def addDest(self, dest): + d = dest.strip() + self.destinations[d] = {'dst_ip': d} + + def dstProcessor(self, inputfile): + from ipaddr import IPAddress + if os.path.isfile(inputfile): + with open(inputfile) as f: + for line in f.readlines(): + if line.startswith('#'): + continue + self.addDest(line) + + def test_icmp(self): + def process_response(echo_reply, dest): + ans, unans = echo_reply + if ans: + log.msg("Recieved echo reply from %s: %s" % (dest, ans)) + else: + log.msg("No reply was received from %s. Possible censorship event." % dest) + log.debug("Unanswered packets: %s" % unans) + self.report[dest] = echo_reply + + for label, data in self.destinations.items(): + reply = sr1(IP(dst=lebal)/ICMP()) + process = process_reponse(reply, label) + + #(ans, unans) = ping + #self.destinations[self.dst].update({'ans': ans, + # 'unans': unans, + # 'response_packet': ping}) + #return ping + + #return reply

1 0

[ooni-probe/master] * Minor change to Makefile.
by isis＠torproject.org 16 Nov '12

16 Nov '12

commit 64be6a7163534713ef50b527d47c90443be879fe Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Nov 15 23:06:57 2012 +0000 * Minor change to Makefile. --- Makefile | 28 ++++++++++++++++------------ 1 files changed, 16 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index a965c00..13d277e 100644 --- a/Makefile +++ b/Makefile @@ -158,14 +158,16 @@ libdnet: scapy-all: scapy pypcap libdnet -tags-recursive: - list='$(SUBDIRS)'; for subdir in $$list; do \ - test "$$subdir" = . || (cd $$subdir && make tags); \ - done - ETAGS = etags ETAGSFLAGS = ETAGSARGS = +## or change to ${SUBDIRS} +TAGME = ${topsrc_dir} + +tags-recursive: + list='$(TAGME)'; for subdir in $$list; do \ + test "$$subdir" = . || (cd $$subdir && make tags); \ + done tags: TAGS @@ -182,17 +184,19 @@ TAGS: tags-recursive $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \ $(TAGS_FILES) $(LISP) tags=; \ here=`pwd`; \ - list='$(SUBDIRS)'; for subdir in $$list; do \ + echo ${E_OR_V}; \ + list='$(TAGME)'; for subdir in $$list; do \ if test "$$subdir" = .; then :; else \ test -f $$subdir/TAGS && tags="$$tags -i $$here/$$subdir/TAGS"; \ fi; \ done; \ list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ unique=`for i in $$list; do \ - if test -f "$$i"; fi; \ - done | \ - $(AWK) ' { files[$$0] = 1; } \ - END { for (i in files) print i; }'`; \ + if test -f "$$i"; then echo $$i; fi; \ + done;` test -z "$(ETAGS_ARGS)$$tags$$unique" \ - || $(ETAGS) $(ETAGSFLAGS) $(ETAGS_ARGS) \ - $$tags $$unique + || $(ETAGS) $(ETAGSFLAGS) $(ETAGS_ARGS) $$tags $$unique + +awkward: + | $(AWK) ' { files[$$0] = 1; } \ + END { for (i in files) print i; }'`; \

1 0