[tor-commits] [ooni-probe/master] PEP8 cleanup of utils
art at torproject.org
art at torproject.org
Tue Aug 5 10:33:36 UTC 2014
commit ddb017a1011a8f2ecd293e49f6b5eb6aa7dbae77
Author: kudrom <kudrom at riseup.net>
Date: Sun Jul 20 20:14:37 2014 +0200
PEP8 cleanup of utils
---
ooni/utils/__init__.py | 11 +++++++++--
ooni/utils/hacks.py | 7 ++++---
ooni/utils/net.py | 48 ++++++++++++++++++++++++++++++---------------
ooni/utils/onion.py | 5 +++++
ooni/utils/trueheaders.py | 19 +++++++++++++-----
ooni/utils/txscapy.py | 15 +++++++-------
6 files changed, 72 insertions(+), 33 deletions(-)
diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py
index 767e6d2..14fe667 100644
--- a/ooni/utils/__init__.py
+++ b/ooni/utils/__init__.py
@@ -5,6 +5,7 @@ import os
from ooni import errors, otime
+
class Storage(dict):
"""
A Storage object is like a dictionary except `obj.foo` can be used
@@ -22,6 +23,7 @@ class Storage(dict):
>>> o.a
None
"""
+
def __getattr__(self, key):
try:
return self[key]
@@ -47,10 +49,12 @@ class Storage(dict):
for (k, v) in value.items():
self[k] = v
+
def checkForRoot():
if os.getuid() != 0:
raise errors.InsufficientPrivileges
+
def randomSTR(length, num=True):
"""
Returns a random all uppercase alfa-numerical (if num True) string long length
@@ -60,6 +64,7 @@ def randomSTR(length, num=True):
chars += string.digits
return ''.join(random.choice(chars) for x in range(length))
+
def randomstr(length, num=True):
"""
Returns a random all lowercase alfa-numerical (if num True) string long length
@@ -69,6 +74,7 @@ def randomstr(length, num=True):
chars += string.digits
return ''.join(random.choice(chars) for x in range(length))
+
def randomStr(length, num=True):
"""
Returns a random a mixed lowercase, uppercase, alfanumerical (if num True)
@@ -79,6 +85,7 @@ def randomStr(length, num=True):
chars += string.digits
return ''.join(random.choice(chars) for x in range(length))
+
def pushFilenameStack(filename):
"""
Takes as input a target filename and checks to see if a file by such name
@@ -90,7 +97,7 @@ def pushFilenameStack(filename):
Args:
filename (str): the path to filename that you wish to create.
"""
- stack = glob.glob(filename+".*")
+ stack = glob.glob(filename + ".*")
stack.sort(key=lambda x: int(x.split('.')[-1]))
for f in reversed(stack):
c_idx = f.split(".")[-1]
@@ -98,7 +105,7 @@ def pushFilenameStack(filename):
new_idx = int(c_idx) + 1
new_filename = "%s.%s" % (c_filename, new_idx)
os.rename(f, new_filename)
- os.rename(filename, filename+".1")
+ os.rename(filename, filename + ".1")
def generate_filename(testDetails, prefix=None, extension=None, filename=None):
diff --git a/ooni/utils/hacks.py b/ooni/utils/hacks.py
index 64b5a53..2c84a23 100644
--- a/ooni/utils/hacks.py
+++ b/ooni/utils/hacks.py
@@ -3,6 +3,7 @@
import copy_reg
+
def patched_reduce_ex(self, proto):
"""
This is a hack to overcome a bug in one of pythons core functions. It is
@@ -22,20 +23,20 @@ def patched_reduce_ex(self, proto):
XXX see if there is a better way. sigh...
"""
- _HEAPTYPE = 1<<9
+ _HEAPTYPE = 1 << 9
assert proto < 2
for base in self.__class__.__mro__:
if hasattr(base, '__flags__') and not base.__flags__ & _HEAPTYPE:
break
else:
- base = object # not really reachable
+ base = object # not really reachable
if base is object:
state = None
elif base is int:
state = None
else:
if base is self.__class__:
- raise TypeError, "can't pickle %s objects" % base.__name__
+ raise TypeError("can't pickle %s objects" % base.__name__)
state = base(self)
args = (self.__class__, base, state)
try:
diff --git a/ooni/utils/net.py b/ooni/utils/net.py
index 845cf4c..69e39d6 100644
--- a/ooni/utils/net.py
+++ b/ooni/utils/net.py
@@ -9,33 +9,39 @@ from twisted.web.iweb import IBodyProducer
from ooni.utils import log
-#if sys.platform.system() == 'Windows':
-# import _winreg as winreg
+# if sys.platform.system() == 'Windows':
+# import _winreg as winreg
# These user agents are taken from the "How Unique Is Your Web Browser?"
# (https://panopticlick.eff.org/browser-uniqueness.pdf) paper as the browser user
# agents with largest anonymity set.
userAgents = ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.7) Gecko/20091221 Firefox/3.5.7",
- "Mozilla/5.0 (iPhone; U; CPU iPhone OS 3 1 2 like Mac OS X; en-us) AppleWebKit/528.18 (KHTML, like Gecko) Mobile/7D11",
- "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6",
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6",
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6",
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.2) Gecko/20100115 Firefox/3.6",
- "Mozilla/5.0 (Windows; U; Windows NT 6.1; de; rv:1.9.2) Gecko/20100115 Firefox/3.6",
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.2) Gecko/20100115 Firefox/3.6",
- "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.7) Gecko/20091221 Firefox/3.5.7",
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.1.7) Gecko/20091221 Firefox/3.5.7 (.NET CLR 3.5.30729)")
+ "Mozilla/5.0 (iPhone; U; CPU iPhone OS 3 1 2 like Mac OS X; en-us)"
+ "AppleWebKit/528.18 (KHTML, like Gecko) Mobile/7D11",
+ "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6",
+ "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6",
+ "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6",
+ "Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.2) Gecko/20100115 Firefox/3.6",
+ "Mozilla/5.0 (Windows; U; Windows NT 6.1; de; rv:1.9.2) Gecko/20100115 Firefox/3.6",
+ "Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.2) Gecko/20100115 Firefox/3.6",
+ "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.7) Gecko/20091221 Firefox/3.5.7",
+ "Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.1.7) "
+ "Gecko/20091221 Firefox/3.5.7 (.NET CLR 3.5.30729)")
+
class UnsupportedPlatform(Exception):
"""Support for this platform is not currently available."""
+
class IfaceError(Exception):
"""Could not find default network interface."""
+
class PermissionsError(SystemExit):
"""This test requires admin or root privileges to run. Exiting..."""
+
PLATFORMS = {'LINUX': sys.platform.startswith("linux"),
'OPENBSD': sys.platform.startswith("openbsd"),
'FREEBSD': sys.platform.startswith("freebsd"),
@@ -44,6 +50,7 @@ PLATFORMS = {'LINUX': sys.platform.startswith("linux"),
'SOLARIS': sys.platform.startswith("sunos"),
'WINDOWS': sys.platform.startswith("win32")}
+
class StringProducer(object):
implements(IBodyProducer)
@@ -61,6 +68,7 @@ class StringProducer(object):
def stopProducing(self):
pass
+
class BodyReceiver(protocol.Protocol):
def __init__(self, finished, content_length=None, body_processor=None):
self.finished = finished
@@ -84,8 +92,9 @@ class BodyReceiver(protocol.Protocol):
except Exception as exc:
self.finished.errback(exc)
+
class Downloader(protocol.Protocol):
- def __init__(self, download_path,
+ def __init__(self, download_path,
finished, content_length=None):
self.finished = finished
self.bytes_remaining = content_length
@@ -104,18 +113,21 @@ class Downloader(protocol.Protocol):
self.fp.close()
self.finished.callback(None)
+
def getSystemResolver():
"""
XXX implement a function that returns the resolver that is currently
default on the system.
"""
+
def getClientPlatform(platform_name=None):
for name, test in PLATFORMS.items():
if not platform_name or platform_name.upper() == name:
if test:
return name, test
+
def getPosixIfaces():
from twisted.internet.test import _posixifaces
@@ -124,6 +136,7 @@ def getPosixIfaces():
ifup = tryInterfaces(ifaces)
return ifup
+
def getWindowsIfaces():
from twisted.internet.test import _win32ifaces
@@ -132,6 +145,7 @@ def getWindowsIfaces():
ifup = tryInterfaces(ifaces)
return ifup
+
def getIfaces(platform_name=None):
client, test = getClientPlatform(platform_name)
if client:
@@ -145,6 +159,7 @@ def getIfaces(platform_name=None):
else:
raise UnsupportedPlatform
+
def randomFreePort(addr="127.0.0.1"):
"""
Args:
@@ -176,7 +191,7 @@ def checkInterfaces(ifaces=None, timeout=1):
"""
try:
from scapy.all import IP, ICMP
- from scapy.all import sr1 ## we want this check to be blocking
+ from scapy.all import sr1 ## we want this check to be blocking
except:
log.msg(("Scapy required: www.secdev.org/projects/scapy"))
@@ -190,24 +205,25 @@ def checkInterfaces(ifaces=None, timeout=1):
log.debug("checkInterfaces(): testing iface {} by pinging"
+ " local address {}".format(ifname, ifaddr))
try:
- pkt = IP(dst=ifaddr)/ICMP()
+ pkt = IP(dst=ifaddr) / ICMP()
ans, unans = sr(pkt, iface=ifname, timeout=5, retry=3)
except Exception, e:
raise PermissionsError if e.find("Errno 1") else log.err(e)
else:
if ans.summary():
log.debug("checkInterfaces(): got answer on interface %s"
- + ":\n%s".format(ifname, ans.summary()))
+ + ":\n%s".format(ifname, ans.summary()))
ifup.update(ifname, ifaddr)
else:
log.debug("Interface test packet was unanswered:\n%s"
- % unans.summary())
+ % unans.summary())
if len(ifup) > 0:
log.msg("Discovered working network interfaces: %s" % ifup)
return ifup
else:
raise IfaceError
+
def getNonLoopbackIfaces(platform_name=None):
try:
ifaces = getIfaces(platform_name)
diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py
index 50e20c3..cb490b3 100644
--- a/ooni/utils/onion.py
+++ b/ooni/utils/onion.py
@@ -6,14 +6,17 @@ from txtorcon.util import find_tor_binary as tx_find_tor_binary
from ooni.settings import config
+
class TorVersion(LooseVersion):
pass
+
def find_tor_binary():
if config.advanced.tor_binary:
return config.advanced.tor_binary
return tx_find_tor_binary()
+
def tor_version():
tor_binary = find_tor_binary()
if not tor_binary:
@@ -29,6 +32,7 @@ def tor_version():
return TorVersion(stdout.strip().split(' ')[2])
return None
+
def transport_name(address):
"""
If the address of the bridge starts with a valid c identifier then
@@ -44,6 +48,7 @@ def transport_name(address):
else:
return None
+
tor_details = {
'binary': find_tor_binary(),
'version': tor_version()
diff --git a/ooni/utils/trueheaders.py b/ooni/utils/trueheaders.py
index 31ee179..a879687 100644
--- a/ooni/utils/trueheaders.py
+++ b/ooni/utils/trueheaders.py
@@ -19,25 +19,27 @@ from twisted.internet.defer import Deferred, succeed, fail, maybeDeferred
from txsocksx.http import SOCKS5Agent
from txsocksx.client import SOCKS5ClientFactory
+
SOCKS5ClientFactory.noisy = False
from ooni.utils import log
+
class TrueHeaders(http_headers.Headers):
def __init__(self, rawHeaders=None):
self._rawHeaders = dict()
if rawHeaders is not None:
for name, values in rawHeaders.iteritems():
if type(values) is list:
- self.setRawHeaders(name, values[:])
+ self.setRawHeaders(name, values[:])
elif type(values) is dict:
- self._rawHeaders[name.lower()] = values
+ self._rawHeaders[name.lower()] = values
elif type(values) is str:
- self.setRawHeaders(name, values)
+ self.setRawHeaders(name, values)
def setRawHeaders(self, name, values):
if name.lower() not in self._rawHeaders:
- self._rawHeaders[name.lower()] = dict()
+ self._rawHeaders[name.lower()] = dict()
self._rawHeaders[name.lower()]['name'] = name
self._rawHeaders[name.lower()]['values'] = values
@@ -71,7 +73,7 @@ class TrueHeaders(http_headers.Headers):
pass
for k, v in itertools.chain(headers_a.getAllRawHeaders(), \
- headers_b.getAllRawHeaders()):
+ headers_b.getAllRawHeaders()):
field_names.append(k)
for name in field_names:
@@ -90,6 +92,7 @@ class TrueHeaders(http_headers.Headers):
return self._rawHeaders[name.lower()]['values']
return default
+
class HTTPClientParser(_newclient.HTTPClientParser):
def logPrefix(self):
return 'HTTPClientParser'
@@ -107,6 +110,7 @@ class HTTPClientParser(_newclient.HTTPClientParser):
headers = self.headers
headers.addRawHeader(name, value)
+
class HTTP11ClientProtocol(_newclient.HTTP11ClientProtocol):
def request(self, request):
if self._state != 'QUIESCENT':
@@ -142,19 +146,24 @@ class HTTP11ClientProtocol(_newclient.HTTP11ClientProtocol):
return self._finishedRequest
+
class _HTTP11ClientFactory(client._HTTP11ClientFactory):
noisy = False
+
def buildProtocol(self, addr):
return HTTP11ClientProtocol(self._quiescentCallback)
+
class HTTPConnectionPool(client.HTTPConnectionPool):
_factory = _HTTP11ClientFactory
+
class TrueHeadersAgent(client.Agent):
def __init__(self, *args, **kw):
super(TrueHeadersAgent, self).__init__(*args, **kw)
self._pool = HTTPConnectionPool(reactor, False)
+
class TrueHeadersSOCKS5Agent(SOCKS5Agent):
def __init__(self, *args, **kw):
super(TrueHeadersSOCKS5Agent, self).__init__(*args, **kw)
diff --git a/ooni/utils/txscapy.py b/ooni/utils/txscapy.py
index 89a0236..340885a 100644
--- a/ooni/utils/txscapy.py
+++ b/ooni/utils/txscapy.py
@@ -255,8 +255,7 @@ class ScapySender(ScapyProtocol):
self.stopSending()
return
- if self.expected_answers and \
- self.expected_answers == len(self.answered_packets):
+ if self.expected_answers and self.expected_answers == len(self.answered_packets):
log.debug("Got the number of expected answers")
self.stopSending()
@@ -416,7 +415,8 @@ class MPTraceroute(ScapyProtocol):
self.numPackets = 1
def ICMPTraceroute(self, host):
- if host not in self.hosts: self.hosts.append(host)
+ if host not in self.hosts:
+ self.hosts.append(host)
d = defer.Deferred()
reactor.callLater(self.timeout, d.callback, self)
@@ -425,7 +425,8 @@ class MPTraceroute(ScapyProtocol):
return d
def UDPTraceroute(self, host):
- if host not in self.hosts: self.hosts.append(host)
+ if host not in self.hosts:
+ self.hosts.append(host)
d = defer.Deferred()
reactor.callLater(self.timeout, d.callback, self)
@@ -436,7 +437,8 @@ class MPTraceroute(ScapyProtocol):
return d
def TCPTraceroute(self, host):
- if host not in self.hosts: self.hosts.append(host)
+ if host not in self.hosts:
+ self.hosts.append(host)
d = defer.Deferred()
reactor.callLater(self.timeout, d.callback, self)
@@ -533,8 +535,7 @@ class MPTraceroute(ScapyProtocol):
l = packet.getlayer(1)
if not l:
return
- elif (isinstance(l, ICMP) or isinstance(l, UDP) or
- isinstance(l, TCP)):
+ elif isinstance(l, ICMP) or isinstance(l, UDP) or isinstance(l, TCP):
self._recvbuf.append(packet)
def stopListening(self):
More information about the tor-commits
mailing list