commit fb4bfce377026b9341ce819d2c8d6e1a345d315c Author: kudrom kudrom@riseup.net Date: Wed Nov 5 20:30:28 2014 +0100

refactoring of net and txscapy utils --- ooni/errors.py | 16 +++++ ooni/geoip.py | 3 +- ooni/nettest.py | 2 +- .../experimental/bridge_reachability/echo.py | 62 ++++++++--------- ooni/oonicli.py | 2 +- ooni/templates/scapyt.py | 2 +- ooni/tests/test_oonicli.py | 2 +- ooni/tests/test_txscapy.py | 13 ---- ooni/tests/test_utils.py | 6 +- ooni/utils/net.py | 34 +++++++++ ooni/utils/txscapy.py | 73 ++------------------ 11 files changed, 96 insertions(+), 119 deletions(-)

diff --git a/ooni/errors.py b/ooni/errors.py index 081c4ac..ada7ce0 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -331,3 +331,19 @@ def get_error(error_key): return Error("%d" % error_key) else: return OONIBError + + +class IfaceError(Exception): + pass + + +class ProtocolNotRegistered(Exception): + pass + + +class ProtocolAlreadyRegistered(Exception): + pass + + +class LibraryNotInstalledError(Exception): + pass \ No newline at end of file diff --git a/ooni/geoip.py b/ooni/geoip.py index 9cb5da3..48e1134 100644 --- a/ooni/geoip.py +++ b/ooni/geoip.py @@ -5,6 +5,8 @@ import random from hashlib import sha256

from twisted.web import client, http_headers +from ooni.utils.net import hasRawSocketPermission + client._HTTP11ClientFactory.noisy = False

from twisted.internet import reactor, defer @@ -243,7 +245,6 @@ class ProbeIP(object): """ Perform a UDP traceroute to determine the probes IP address. """ - from ooni.utils.txscapy import hasRawSocketPermission if not hasRawSocketPermission(): raise errors.InsufficientPrivileges raise NotImplemented diff --git a/ooni/nettest.py b/ooni/nettest.py index 12fb2fb..71cb94f 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -11,7 +11,7 @@ from twisted.python import usage, reflect from ooni import otime from ooni.tasks import Measurement from ooni.utils import log, sanitize_options -from ooni.utils.txscapy import hasRawSocketPermission +from ooni.utils.net import hasRawSocketPermission from ooni.settings import config

from ooni import errors as e diff --git a/ooni/nettests/experimental/bridge_reachability/echo.py b/ooni/nettests/experimental/bridge_reachability/echo.py index d4033dd..e64a604 100644 --- a/ooni/nettests/experimental/bridge_reachability/echo.py +++ b/ooni/nettests/experimental/bridge_reachability/echo.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # -# +---------+ +# +---------+ # | echo.py | # +---------+ # A simple ICMP-8 ping test. @@ -13,22 +13,21 @@ #

import os -import sys

-from twisted.python import usage -from twisted.internet import reactor, defer -from ooni import nettest -from ooni.utils import log, net, Storage, txscapy +from twisted.python import usage +from ooni import nettest +from ooni.utils import log, net

try: - from scapy.all import IP, ICMP - from scapy.all import sr1 - from ooni.lib import txscapy - from ooni.lib.txscapy import txsr, txsend + from scapy.all import IP, ICMP + from scapy.all import sr1 + from ooni.lib import txscapy + from ooni.lib.txscapy import txsr, txsend from ooni.templates.scapyt import BaseScapyTest except: log.msg("This test requires scapy, see www.secdev.org/projects/scapy")

+ class UsageOptions(usage.Options): optParameters = [ ['dst', 'd', None, 'Host IP to ping'], @@ -41,17 +40,18 @@ class UsageOptions(usage.Options): ['pcap', 'p', None, 'Save pcap to this file'], ['receive', 'r', True, 'Receive response packets']]

+ class EchoTest(nettest.NetTestCase): """ xxx fill me in """ - name = 'echo' - author = 'Isis Lovecruft isis@torproject.org' - description = 'A simple ping test to see if a host is reachable.' - version = '0.0.2' + name = 'echo' + author = 'Isis Lovecruft isis@torproject.org' + description = 'A simple ping test to see if a host is reachable.' + version = '0.0.2' requiresRoot = True

- usageOptions = UsageOptions + usageOptions = UsageOptions #requiredOptions = ['dst']

def setUp(self, *a, **kw): @@ -62,11 +62,11 @@ class EchoTest(nettest.NetTestCase): log.debug("setting self.%s = %s" % (key, value)) setattr(self, key, value)

- self.timeout *= 1000 ## convert to milliseconds + self.timeout *= 1000 ## convert to milliseconds

if not self.interface: try: - iface = txscapy.getDefaultIface() + iface = net.getDefaultIface() except Exception, e: log.msg("No network interface specified!") log.err(e) @@ -111,22 +111,22 @@ class EchoTest(nettest.NetTestCase):

def test_icmp(self): def process_response(echo_reply, dest): - ans, unans = echo_reply - if ans: - log.msg("Recieved echo reply from %s: %s" % (dest, ans)) - else: - log.msg("No reply was received from %s. Possible censorship event." % dest) - log.debug("Unanswered packets: %s" % unans) - self.report[dest] = echo_reply + ans, unans = echo_reply + if ans: + log.msg("Recieved echo reply from %s: %s" % (dest, ans)) + else: + log.msg("No reply was received from %s. Possible censorship event." % dest) + log.debug("Unanswered packets: %s" % unans) + self.report[dest] = echo_reply

for label, data in self.destinations.items(): - reply = sr1(IP(dst=lebal)/ICMP()) + reply = sr1(IP(dst=lebal) / ICMP()) process = process_reponse(reply, label)

- #(ans, unans) = ping - #self.destinations[self.dst].update({'ans': ans, - # 'unans': unans, - # 'response_packet': ping}) - #return ping + #(ans, unans) = ping + #self.destinations[self.dst].update({'ans': ans, + # 'unans': unans, + # 'response_packet': ping}) + #return ping

- #return reply + #return reply diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 279d87c..6020501 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -14,7 +14,7 @@ from ooni.deck import Deck, nettest_to_path from ooni.nettest import NetTestLoader

from ooni.utils import log -from ooni.utils.txscapy import hasRawSocketPermission +from ooni.utils.net import hasRawSocketPermission

class Options(usage.Options): diff --git a/ooni/templates/scapyt.py b/ooni/templates/scapyt.py index 925a9d6..35dbf5b 100644 --- a/ooni/templates/scapyt.py +++ b/ooni/templates/scapyt.py @@ -1,9 +1,9 @@ from ooni.nettest import NetTestCase from ooni.utils import log from ooni.settings import config +from ooni.utils.net import hasRawSocketPermission

from ooni.utils.txscapy import ScapySender, ScapyFactory -from ooni.utils.txscapy import hasRawSocketPermission

class BaseScapyTest(NetTestCase): diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py index 0bd3180..a01aade 100644 --- a/ooni/tests/test_oonicli.py +++ b/ooni/tests/test_oonicli.py @@ -9,7 +9,7 @@ from ooni.tests.bases import ConfigTestCase from ooni.settings import config from ooni.oonicli import runWithDirector from ooni.errors import InsufficientPrivileges -from ooni.utils.txscapy import hasRawSocketPermission +from ooni.utils.net import hasRawSocketPermission

def verify_header(header): diff --git a/ooni/tests/test_txscapy.py b/ooni/tests/test_txscapy.py index 1dbb8cc..0332fcf 100644 --- a/ooni/tests/test_txscapy.py +++ b/ooni/tests/test_txscapy.py @@ -52,16 +52,3 @@ class TestTxScapy(unittest.TestCase): assert result[0][0][0] == packet_sent assert result[0][0][1] == packet_received

- def test_get_addresses(self): - addresses = txscapy.getAddresses() - assert isinstance(addresses, list) - - # @defer.inlineCallbacks - # def test_multi_traceroute(self): - # traceroute = txscapy.MPTraceroute() - # traceroute.timeout = 3 - # self.scapy_factory.registerProtocol(traceroute) - # yield traceroute.ICMPTraceroute('8.8.8.8') - # yield traceroute.TCPTraceroute('8.8.8.8') - # yield traceroute.UDPTraceroute('8.8.8.8') - # self.scapy_factory.super_socket.send.assert_called() diff --git a/ooni/tests/test_utils.py b/ooni/tests/test_utils.py index be5d578..e2e448a 100644 --- a/ooni/tests/test_utils.py +++ b/ooni/tests/test_utils.py @@ -1,7 +1,7 @@ import os from twisted.trial import unittest

-from ooni.utils import pushFilenameStack, log, generate_filename +from ooni.utils import pushFilenameStack, log, generate_filename, net

class TestUtils(unittest.TestCase): @@ -71,3 +71,7 @@ class TestUtils(unittest.TestCase): def test_generate_filename_with_extension_and_basename(self): filename = generate_filename(self.test_details, extension=self.extension, filename=self.basename) self.assertEqual(filename, 'filename.ext') + + def test_get_addresses(self): + addresses = net.getAddresses() + assert isinstance(addresses, list) diff --git a/ooni/utils/net.py b/ooni/utils/net.py index 2f1a2c6..014020c 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -6,6 +6,9 @@ from zope.interface import implements from twisted.internet import protocol, defer from twisted.web.iweb import IBodyProducer

+from scapy.config import conf + +from ooni.errors import IfaceError

try: from twisted.internet.endpoints import connectProtocol @@ -136,3 +139,34 @@ def randomFreePort(addr="127.0.0.1"): pass s.close() return port + + +def getDefaultIface(): + """ Return the default interface or raise IfaceError """ + iface = conf.route.route('0.0.0.0', verbose=0)[0] + if len(iface) > 0: + return iface + raise IfaceError + + +def getAddresses(): + from scapy.all import get_if_addr, get_if_list + from ipaddr import IPAddress + + addresses = set() + for i in get_if_list(): + try: + addresses.add(get_if_addr(i)) + except: + pass + if '0.0.0.0' in addresses: + addresses.remove('0.0.0.0') + return [IPAddress(addr) for addr in addresses] + + +def hasRawSocketPermission(): + try: + socket.socket(socket.AF_INET, socket.SOCK_RAW, socket.IPPROTO_RAW) + return True + except socket.error: + return False \ No newline at end of file diff --git a/ooni/utils/txscapy.py b/ooni/utils/txscapy.py index d0f58ba..59ac5b1 100644 --- a/ooni/utils/txscapy.py +++ b/ooni/utils/txscapy.py @@ -1,21 +1,18 @@ -import socket import sys import time import random - from twisted.internet import fdesc from twisted.internet import reactor from twisted.internet import defer, abstract - from scapy.config import conf from scapy.all import RandShort, IP, IPerror, ICMP, ICMPerror, TCP, TCPerror, UDP, UDPerror

-from ooni.utils import log -from ooni.settings import config +from ooni.errors import ProtocolNotRegistered, ProtocolAlreadyRegistered, LibraryNotInstalledError

+from ooni.utils import log

-class LibraryNotInstalledError(Exception): - pass +from ooni.utils.net import getDefaultIface, getAddresses +from ooni.settings import config

def pcapdnet_installed(): @@ -83,68 +80,6 @@ else: from scapy.all import Gen, SetGen, MTU

-def getNetworksFromRoutes(): - """ Return a list of networks from the routing table """ - from scapy.all import conf, ltoa, read_routes - from ipaddr import IPNetwork, IPAddress - - # # Hide the 'no routes' warnings - conf.verb = 0 - - networks = [] - for nw, nm, gw, iface, addr in read_routes(): - n = IPNetwork(ltoa(nw)) - (n.netmask, n.gateway, n.ipaddr) = [IPAddress(x) for x in [nm, gw, addr]] - n.iface = iface - if not n.compressed in networks: - networks.append(n) - - return networks - - -class IfaceError(Exception): - pass - - -def getAddresses(): - from scapy.all import get_if_addr, get_if_list - from ipaddr import IPAddress - - addresses = set() - for i in get_if_list(): - try: - addresses.add(get_if_addr(i)) - except: - pass - if '0.0.0.0' in addresses: - addresses.remove('0.0.0.0') - return [IPAddress(addr) for addr in addresses] - - -def getDefaultIface(): - """ Return the default interface or raise IfaceError """ - iface = conf.route.route('0.0.0.0', verbose=0)[0] - if len(iface) > 0: - return iface - raise IfaceError - - -def hasRawSocketPermission(): - try: - socket.socket(socket.AF_INET, socket.SOCK_RAW, socket.IPPROTO_RAW) - return True - except socket.error: - return False - - -class ProtocolNotRegistered(Exception): - pass - - -class ProtocolAlreadyRegistered(Exception): - pass - - class ScapyFactory(abstract.FileDescriptor): """ Inspired by muxTCP scapyLink: