commit ddb017a1011a8f2ecd293e49f6b5eb6aa7dbae77 Author: kudrom kudrom@riseup.net Date: Sun Jul 20 20:14:37 2014 +0200

PEP8 cleanup of utils --- ooni/utils/__init__.py | 11 +++++++++-- ooni/utils/hacks.py | 7 ++++--- ooni/utils/net.py | 48 ++++++++++++++++++++++++++++++--------------- ooni/utils/onion.py | 5 +++++ ooni/utils/trueheaders.py | 19 +++++++++++++----- ooni/utils/txscapy.py | 15 +++++++------- 6 files changed, 72 insertions(+), 33 deletions(-)

diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index 767e6d2..14fe667 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -5,6 +5,7 @@ import os

from ooni import errors, otime

+ class Storage(dict): """ A Storage object is like a dictionary except `obj.foo` can be used @@ -22,6 +23,7 @@ class Storage(dict): >>> o.a None """ + def __getattr__(self, key): try: return self[key] @@ -47,10 +49,12 @@ class Storage(dict): for (k, v) in value.items(): self[k] = v

+ def checkForRoot(): if os.getuid() != 0: raise errors.InsufficientPrivileges

+ def randomSTR(length, num=True): """ Returns a random all uppercase alfa-numerical (if num True) string long length @@ -60,6 +64,7 @@ def randomSTR(length, num=True): chars += string.digits return ''.join(random.choice(chars) for x in range(length))

+ def randomstr(length, num=True): """ Returns a random all lowercase alfa-numerical (if num True) string long length @@ -69,6 +74,7 @@ def randomstr(length, num=True): chars += string.digits return ''.join(random.choice(chars) for x in range(length))

+ def randomStr(length, num=True): """ Returns a random a mixed lowercase, uppercase, alfanumerical (if num True) @@ -79,6 +85,7 @@ def randomStr(length, num=True): chars += string.digits return ''.join(random.choice(chars) for x in range(length))

+ def pushFilenameStack(filename): """ Takes as input a target filename and checks to see if a file by such name @@ -90,7 +97,7 @@ def pushFilenameStack(filename): Args: filename (str): the path to filename that you wish to create. """ - stack = glob.glob(filename+".*") + stack = glob.glob(filename + ".*") stack.sort(key=lambda x: int(x.split('.')[-1])) for f in reversed(stack): c_idx = f.split(".")[-1] @@ -98,7 +105,7 @@ def pushFilenameStack(filename): new_idx = int(c_idx) + 1 new_filename = "%s.%s" % (c_filename, new_idx) os.rename(f, new_filename) - os.rename(filename, filename+".1") + os.rename(filename, filename + ".1")

def generate_filename(testDetails, prefix=None, extension=None, filename=None): diff --git a/ooni/utils/hacks.py b/ooni/utils/hacks.py index 64b5a53..2c84a23 100644 --- a/ooni/utils/hacks.py +++ b/ooni/utils/hacks.py @@ -3,6 +3,7 @@

import copy_reg

+ def patched_reduce_ex(self, proto): """ This is a hack to overcome a bug in one of pythons core functions. It is @@ -22,20 +23,20 @@ def patched_reduce_ex(self, proto):

XXX see if there is a better way. sigh... """ - _HEAPTYPE = 1<<9 + _HEAPTYPE = 1 << 9 assert proto < 2 for base in self.__class__.__mro__: if hasattr(base, '__flags__') and not base.__flags__ & _HEAPTYPE: break else: - base = object # not really reachable + base = object # not really reachable if base is object: state = None elif base is int: state = None else: if base is self.__class__: - raise TypeError, "can't pickle %s objects" % base.__name__ + raise TypeError("can't pickle %s objects" % base.__name__) state = base(self) args = (self.__class__, base, state) try: diff --git a/ooni/utils/net.py b/ooni/utils/net.py index 845cf4c..69e39d6 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -9,33 +9,39 @@ from twisted.web.iweb import IBodyProducer

from ooni.utils import log

-#if sys.platform.system() == 'Windows': -# import _winreg as winreg +# if sys.platform.system() == 'Windows': +# import _winreg as winreg

# These user agents are taken from the "How Unique Is Your Web Browser?" # (https://panopticlick.eff.org/browser-uniqueness.pdf) paper as the browser user # agents with largest anonymity set.

userAgents = ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.7) Gecko/20091221 Firefox/3.5.7", - "Mozilla/5.0 (iPhone; U; CPU iPhone OS 3 1 2 like Mac OS X; en-us) AppleWebKit/528.18 (KHTML, like Gecko) Mobile/7D11", - "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6", - "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6", - "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6", - "Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.2) Gecko/20100115 Firefox/3.6", - "Mozilla/5.0 (Windows; U; Windows NT 6.1; de; rv:1.9.2) Gecko/20100115 Firefox/3.6", - "Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.2) Gecko/20100115 Firefox/3.6", - "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.7) Gecko/20091221 Firefox/3.5.7", - "Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.1.7) Gecko/20091221 Firefox/3.5.7 (.NET CLR 3.5.30729)") + "Mozilla/5.0 (iPhone; U; CPU iPhone OS 3 1 2 like Mac OS X; en-us)" + "AppleWebKit/528.18 (KHTML, like Gecko) Mobile/7D11", + "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.2) Gecko/20100115 Firefox/3.6", + "Mozilla/5.0 (Windows; U; Windows NT 6.1; de; rv:1.9.2) Gecko/20100115 Firefox/3.6", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.2) Gecko/20100115 Firefox/3.6", + "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.7) Gecko/20091221 Firefox/3.5.7", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.1.7) " + "Gecko/20091221 Firefox/3.5.7 (.NET CLR 3.5.30729)") +

class UnsupportedPlatform(Exception): """Support for this platform is not currently available."""

+ class IfaceError(Exception): """Could not find default network interface."""

+ class PermissionsError(SystemExit): """This test requires admin or root privileges to run. Exiting..."""

+ PLATFORMS = {'LINUX': sys.platform.startswith("linux"), 'OPENBSD': sys.platform.startswith("openbsd"), 'FREEBSD': sys.platform.startswith("freebsd"), @@ -44,6 +50,7 @@ PLATFORMS = {'LINUX': sys.platform.startswith("linux"), 'SOLARIS': sys.platform.startswith("sunos"), 'WINDOWS': sys.platform.startswith("win32")}

+ class StringProducer(object): implements(IBodyProducer)

@@ -61,6 +68,7 @@ class StringProducer(object): def stopProducing(self): pass

+ class BodyReceiver(protocol.Protocol): def __init__(self, finished, content_length=None, body_processor=None): self.finished = finished @@ -84,8 +92,9 @@ class BodyReceiver(protocol.Protocol): except Exception as exc: self.finished.errback(exc)

+ class Downloader(protocol.Protocol): - def __init__(self, download_path, + def __init__(self, download_path, finished, content_length=None): self.finished = finished self.bytes_remaining = content_length @@ -104,18 +113,21 @@ class Downloader(protocol.Protocol): self.fp.close() self.finished.callback(None)

+ def getSystemResolver(): """ XXX implement a function that returns the resolver that is currently default on the system. """

+ def getClientPlatform(platform_name=None): for name, test in PLATFORMS.items(): if not platform_name or platform_name.upper() == name: if test: return name, test

+ def getPosixIfaces(): from twisted.internet.test import _posixifaces

@@ -124,6 +136,7 @@ def getPosixIfaces(): ifup = tryInterfaces(ifaces) return ifup

+ def getWindowsIfaces(): from twisted.internet.test import _win32ifaces

@@ -132,6 +145,7 @@ def getWindowsIfaces(): ifup = tryInterfaces(ifaces) return ifup

+ def getIfaces(platform_name=None): client, test = getClientPlatform(platform_name) if client: @@ -145,6 +159,7 @@ def getIfaces(platform_name=None): else: raise UnsupportedPlatform

+ def randomFreePort(addr="127.0.0.1"): """ Args: @@ -176,7 +191,7 @@ def checkInterfaces(ifaces=None, timeout=1): """ try: from scapy.all import IP, ICMP - from scapy.all import sr1 ## we want this check to be blocking + from scapy.all import sr1 ## we want this check to be blocking except: log.msg(("Scapy required: www.secdev.org/projects/scapy"))

@@ -190,24 +205,25 @@ def checkInterfaces(ifaces=None, timeout=1): log.debug("checkInterfaces(): testing iface {} by pinging" + " local address {}".format(ifname, ifaddr)) try: - pkt = IP(dst=ifaddr)/ICMP() + pkt = IP(dst=ifaddr) / ICMP() ans, unans = sr(pkt, iface=ifname, timeout=5, retry=3) except Exception, e: raise PermissionsError if e.find("Errno 1") else log.err(e) else: if ans.summary(): log.debug("checkInterfaces(): got answer on interface %s" - + ":\n%s".format(ifname, ans.summary())) + + ":\n%s".format(ifname, ans.summary())) ifup.update(ifname, ifaddr) else: log.debug("Interface test packet was unanswered:\n%s" - % unans.summary()) + % unans.summary()) if len(ifup) > 0: log.msg("Discovered working network interfaces: %s" % ifup) return ifup else: raise IfaceError

+ def getNonLoopbackIfaces(platform_name=None): try: ifaces = getIfaces(platform_name) diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py index 50e20c3..cb490b3 100644 --- a/ooni/utils/onion.py +++ b/ooni/utils/onion.py @@ -6,14 +6,17 @@ from txtorcon.util import find_tor_binary as tx_find_tor_binary

from ooni.settings import config

+ class TorVersion(LooseVersion): pass

+ def find_tor_binary(): if config.advanced.tor_binary: return config.advanced.tor_binary return tx_find_tor_binary()

+ def tor_version(): tor_binary = find_tor_binary() if not tor_binary: @@ -29,6 +32,7 @@ def tor_version(): return TorVersion(stdout.strip().split(' ')[2]) return None

+ def transport_name(address): """ If the address of the bridge starts with a valid c identifier then @@ -44,6 +48,7 @@ def transport_name(address): else: return None

+ tor_details = { 'binary': find_tor_binary(), 'version': tor_version() diff --git a/ooni/utils/trueheaders.py b/ooni/utils/trueheaders.py index 31ee179..a879687 100644 --- a/ooni/utils/trueheaders.py +++ b/ooni/utils/trueheaders.py @@ -19,25 +19,27 @@ from twisted.internet.defer import Deferred, succeed, fail, maybeDeferred

from txsocksx.http import SOCKS5Agent from txsocksx.client import SOCKS5ClientFactory + SOCKS5ClientFactory.noisy = False

from ooni.utils import log

+ class TrueHeaders(http_headers.Headers): def __init__(self, rawHeaders=None): self._rawHeaders = dict() if rawHeaders is not None: for name, values in rawHeaders.iteritems(): if type(values) is list: - self.setRawHeaders(name, values[:]) + self.setRawHeaders(name, values[:]) elif type(values) is dict: - self._rawHeaders[name.lower()] = values + self._rawHeaders[name.lower()] = values elif type(values) is str: - self.setRawHeaders(name, values) + self.setRawHeaders(name, values)

def setRawHeaders(self, name, values): if name.lower() not in self._rawHeaders: - self._rawHeaders[name.lower()] = dict() + self._rawHeaders[name.lower()] = dict() self._rawHeaders[name.lower()]['name'] = name self._rawHeaders[name.lower()]['values'] = values

@@ -71,7 +73,7 @@ class TrueHeaders(http_headers.Headers): pass

for k, v in itertools.chain(headers_a.getAllRawHeaders(), \ - headers_b.getAllRawHeaders()): + headers_b.getAllRawHeaders()): field_names.append(k)

for name in field_names: @@ -90,6 +92,7 @@ class TrueHeaders(http_headers.Headers): return self._rawHeaders[name.lower()]['values'] return default

+ class HTTPClientParser(_newclient.HTTPClientParser): def logPrefix(self): return 'HTTPClientParser' @@ -107,6 +110,7 @@ class HTTPClientParser(_newclient.HTTPClientParser): headers = self.headers headers.addRawHeader(name, value)

+ class HTTP11ClientProtocol(_newclient.HTTP11ClientProtocol): def request(self, request): if self._state != 'QUIESCENT': @@ -142,19 +146,24 @@ class HTTP11ClientProtocol(_newclient.HTTP11ClientProtocol):

return self._finishedRequest

+ class _HTTP11ClientFactory(client._HTTP11ClientFactory): noisy = False + def buildProtocol(self, addr): return HTTP11ClientProtocol(self._quiescentCallback)

+ class HTTPConnectionPool(client.HTTPConnectionPool): _factory = _HTTP11ClientFactory

+ class TrueHeadersAgent(client.Agent): def __init__(self, *args, **kw): super(TrueHeadersAgent, self).__init__(*args, **kw) self._pool = HTTPConnectionPool(reactor, False)

+ class TrueHeadersSOCKS5Agent(SOCKS5Agent): def __init__(self, *args, **kw): super(TrueHeadersSOCKS5Agent, self).__init__(*args, **kw) diff --git a/ooni/utils/txscapy.py b/ooni/utils/txscapy.py index 89a0236..340885a 100644 --- a/ooni/utils/txscapy.py +++ b/ooni/utils/txscapy.py @@ -255,8 +255,7 @@ class ScapySender(ScapyProtocol): self.stopSending() return

- if self.expected_answers and \ - self.expected_answers == len(self.answered_packets): + if self.expected_answers and self.expected_answers == len(self.answered_packets): log.debug("Got the number of expected answers") self.stopSending()

@@ -416,7 +415,8 @@ class MPTraceroute(ScapyProtocol): self.numPackets = 1

def ICMPTraceroute(self, host): - if host not in self.hosts: self.hosts.append(host) + if host not in self.hosts: + self.hosts.append(host)

d = defer.Deferred() reactor.callLater(self.timeout, d.callback, self) @@ -425,7 +425,8 @@ class MPTraceroute(ScapyProtocol): return d

def UDPTraceroute(self, host): - if host not in self.hosts: self.hosts.append(host) + if host not in self.hosts: + self.hosts.append(host)

d = defer.Deferred() reactor.callLater(self.timeout, d.callback, self) @@ -436,7 +437,8 @@ class MPTraceroute(ScapyProtocol): return d

def TCPTraceroute(self, host): - if host not in self.hosts: self.hosts.append(host) + if host not in self.hosts: + self.hosts.append(host)

d = defer.Deferred() reactor.callLater(self.timeout, d.callback, self) @@ -533,8 +535,7 @@ class MPTraceroute(ScapyProtocol): l = packet.getlayer(1) if not l: return - elif (isinstance(l, ICMP) or isinstance(l, UDP) or - isinstance(l, TCP)): + elif isinstance(l, ICMP) or isinstance(l, UDP) or isinstance(l, TCP): self._recvbuf.append(packet)

def stopListening(self):