June 2013 - tor-commits - lists.torproject.org

[ooni-probe/develop] Move geoip related classes into ooni.geoip
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 51fe88e01de3d559522dc2cf3ea518f65993635b Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Mar 15 09:22:13 2013 +0100 Move geoip related classes into ooni.geoip --- HACKING | 2 +- ooni/director.py | 133 +-------------------------------------- ooni/geoip.py | 168 +++++++++++++++++++++++++++++++++++++++++++++++++ ooni/nettest.py | 5 +- ooni/reporter.py | 2 +- ooni/utils/geodata.py | 40 ------------ tests/test_nettest.py | 2 +- 7 files changed, 176 insertions(+), 176 deletions(-) diff --git a/HACKING b/HACKING index b92ddbe..aad7ebb 100644 --- a/HACKING +++ b/HACKING @@ -87,7 +87,7 @@ Code Structure - ooni/utils/ In here go internal utilities that are useful to ooniprobe -- ooni/utils/geodata.py +- ooni/geoip.py In here go functions related to the understanding of geographical information of the probe diff --git a/ooni/director.py b/ooni/director.py index 07510eb..d2f5a7c 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -5,6 +5,7 @@ import os import re from ooni import config +from ooni import geoip from ooni.managers import ReportEntryManager, MeasurementManager from ooni.reporter import Report from ooni.utils import log, checkForRoot @@ -16,136 +17,6 @@ from txtorcon import TorConfig from txtorcon import TorState, launch_tor from twisted.internet import defer, reactor -from twisted.web import client, http_headers -from ooni.utils.net import userAgents, BodyReceiver - -class HTTPGeoIPLookupper(object): - url = None - - def _response(self, response): - content_length = response.headers.getRawHeaders('content-length') - - finished = defer.Deferred() - response.deliverBody(BodyReceiver(finished, content_length)) - finished.addCallback(self.parseResponse) - return finished - - def parseResponse(self, response_body): - """ - Override this with the logic for parsing the response. - - Should return the IP address of the probe. - """ - pass - - def failed(self, failure): - log.err("Failed to lookup via %s" % url) - log.exception(failure) - return failure - - def lookup(self): - agent = client.Agent(reactor) - headers = {} - headers['User-Agent'] = [random.choice(userAgents)] - - d = agent.request("GET", self.url, http_headers.Headers(headers)) - d.addCallback(self._response) - d.addErrback(self.failed) - return d - -class UbuntuGeoIP(HTTPGeoIPLookupper): - url = "http://geoip.ubuntu.com/lookup" - - def parseResponse(self, response_body): - response = ET.fromstring(response_body) - probe_ip = response.find('Ip').text - return probe_ip - -class TorProjectGeoIP(HTTPGeoIPLookupper): - url = "https://check.torproject.org/" - - def parseResponse(self, response_body): - regexp = "Your IP address appears to be: ((\d+\.)+(\d+))" - probe_ip = re.search(regexp, response_body).group(1) - return probe_ip - -class MaxMindGeoIP(HTTPGeoIPLookupper): - url = "https://www.maxmind.com/en/locate_my_ip" - - def parseResponse(self, response_body): - regexp = '((\d+\.)+(\d+))' - probe_ip = re.search(regexp, response_body).group(1) - return probe_ip - -class ProbeIP(object): - strategy = None - geoIPServices = {'ubuntu': UbuntuGeoIP, - 'torproject': TorProjectGeoIP, - 'maximind': MaxMindGeoIP - } - address = None - - @defer.inlineCallbacks - def lookup(self): - try: - yield self.askTor() - defer.returnValue(self.address) - except errors.TorStateNotFound: - log.debug("Tor is not running. Skipping IP lookup via Tor.") - except: - log.msg("Unable to lookup the probe IP via Tor.") - - try: - yield self.askTraceroute() - defer.returnValue(self.address) - except errors.InsufficientPrivileges: - log.debug("Cannot determine the probe IP address with a traceroute, becase of insufficient priviledges") - except: - log.msg("Unable to lookup the probe IP via traceroute") - - try: - yield self.askGeoIPService() - defer.returnValue(self.address) - except Exception, e: - print e - log.msg("Unable to lookup the probe IP via GeoIPService") - - @defer.inlineCallbacks - def askGeoIPService(self): - for service_name, service in self.geoIPServices.items(): - s = TorProjectGeoIP() - log.msg("Looking up your IP address via %s" % service_name) - try: - self.address = yield s.lookup() - self.strategy = 'geo_ip_service-' + service_name - break - except: - log.msg("Failed to lookup your IP via %s" % service_name) - - def askTraceroute(self): - """ - Perform a UDP traceroute to determine the probes IP address. - """ - checkForRoot() - raise NotImplemented - - def askTor(self): - """ - Obtain the probes IP address by asking the Tor Control port via GET INFO - address. - - XXX this lookup method is currently broken when there are cached descriptors or consensus documents - see: https://trac.torproject.org/projects/tor/ticket/8214 - """ - if config.tor_state: - d = config.tor_state.protocol.get_info("address") - @d.addCallback - def cb(result): - self.strategy = 'tor_get_info_address' - self.address = result.values()[0] - return d - else: - raise errors.TorStateNotFound class Director(object): """ @@ -225,7 +96,7 @@ class Director(object): log.msg("Starting Tor...") yield self.startTor() - config.probe_ip = ProbeIP() + config.probe_ip = geoip.ProbeIP() yield config.probe_ip.lookup() @property diff --git a/ooni/geoip.py b/ooni/geoip.py new file mode 100644 index 0000000..0fe8a6b --- /dev/null +++ b/ooni/geoip.py @@ -0,0 +1,168 @@ +import re +import os + +from twisted.web import client, http_headers +from ooni.utils.net import userAgents, BodyReceiver +from twisted.internet import reactor, defer, protocol + +from ooni.utils import log, net +from ooni import config + +try: + import pygeoip +except ImportError: + log.err("Unable to import pygeoip. We will not be able to run geo IP related measurements") + +class GeoIPDataFilesNotFound(Exception): + pass + +def IPToLocation(ipaddr): + city_file = os.path.join(config.advanced.geoip_data_dir, 'GeoLiteCity.dat') + country_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIP.dat') + asn_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIPASNum.dat') + + location = {'city': None, 'countrycode': None, 'asn': None} + try: + city_dat = pygeoip.GeoIP(city_file) + location['city'] = city_dat.record_by_addr(ipaddr)['city'] + + country_dat = pygeoip.GeoIP(country_file) + location['countrycode'] = country_dat.country_code_by_addr(ipaddr) + + asn_dat = pygeoip.GeoIP(asn_file) + location['asn'] = asn_dat.org_by_addr(ipaddr) + + except IOError: + log.err("Could not find GeoIP data files. Go into data/ " + "and run make geoip") + raise GeoIPDataFilesNotFound + + return location + +class HTTPGeoIPLookupper(object): + url = None + + def _response(self, response): + content_length = response.headers.getRawHeaders('content-length') + + finished = defer.Deferred() + response.deliverBody(BodyReceiver(finished, content_length)) + finished.addCallback(self.parseResponse) + return finished + + def parseResponse(self, response_body): + """ + Override this with the logic for parsing the response. + + Should return the IP address of the probe. + """ + pass + + def failed(self, failure): + log.err("Failed to lookup via %s" % url) + log.exception(failure) + return failure + + def lookup(self): + agent = client.Agent(reactor) + headers = {} + headers['User-Agent'] = [random.choice(userAgents)] + + d = agent.request("GET", self.url, http_headers.Headers(headers)) + d.addCallback(self._response) + d.addErrback(self.failed) + return d + +class UbuntuGeoIP(HTTPGeoIPLookupper): + url = "http://geoip.ubuntu.com/lookup" + + def parseResponse(self, response_body): + response = ET.fromstring(response_body) + probe_ip = response.find('Ip').text + return probe_ip + +class TorProjectGeoIP(HTTPGeoIPLookupper): + url = "https://check.torproject.org/" + + def parseResponse(self, response_body): + regexp = "Your IP address appears to be: ((\d+\.)+(\d+))" + probe_ip = re.search(regexp, response_body).group(1) + return probe_ip + +class MaxMindGeoIP(HTTPGeoIPLookupper): + url = "https://www.maxmind.com/en/locate_my_ip" + + def parseResponse(self, response_body): + regexp = '((\d+\.)+(\d+))' + probe_ip = re.search(regexp, response_body).group(1) + return probe_ip + +class ProbeIP(object): + strategy = None + geoIPServices = {'ubuntu': UbuntuGeoIP, + 'torproject': TorProjectGeoIP, + 'maximind': MaxMindGeoIP + } + address = None + + @defer.inlineCallbacks + def lookup(self): + try: + yield self.askTor() + defer.returnValue(self.address) + except errors.TorStateNotFound: + log.debug("Tor is not running. Skipping IP lookup via Tor.") + except: + log.msg("Unable to lookup the probe IP via Tor.") + + try: + yield self.askTraceroute() + defer.returnValue(self.address) + except errors.InsufficientPrivileges: + log.debug("Cannot determine the probe IP address with a traceroute, becase of insufficient priviledges") + except: + log.msg("Unable to lookup the probe IP via traceroute") + + try: + yield self.askGeoIPService() + defer.returnValue(self.address) + except Exception, e: + print e + log.msg("Unable to lookup the probe IP via GeoIPService") + + @defer.inlineCallbacks + def askGeoIPService(self): + for service_name, service in self.geoIPServices.items(): + s = TorProjectGeoIP() + log.msg("Looking up your IP address via %s" % service_name) + try: + self.address = yield s.lookup() + self.strategy = 'geo_ip_service-' + service_name + break + except: + log.msg("Failed to lookup your IP via %s" % service_name) + + def askTraceroute(self): + """ + Perform a UDP traceroute to determine the probes IP address. + """ + checkForRoot() + raise NotImplemented + + def askTor(self): + """ + Obtain the probes IP address by asking the Tor Control port via GET INFO + address. + + XXX this lookup method is currently broken when there are cached descriptors or consensus documents + see: https://trac.torproject.org/projects/tor/ticket/8214 + """ + if config.tor_state: + d = config.tor_state.protocol.get_info("address") + @d.addCallback + def cb(result): + self.strategy = 'tor_get_info_address' + self.address = result.values()[0] + return d + else: + raise errors.TorStateNotFound diff --git a/ooni/nettest.py b/ooni/nettest.py index 67dee9d..bd6bd41 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -5,8 +5,9 @@ from twisted.internet import defer, reactor from twisted.trial.runner import filenameToModule from twisted.python import usage, reflect +from ooni import geoip from ooni.tasks import Measurement -from ooni.utils import log, checkForRoot, geodata +from ooni.utils import log, checkForRoot from ooni import config from ooni import otime @@ -35,7 +36,7 @@ class NetTestLoader(object): config.privacy.includecountry or \ config.privacy.includecity): log.msg("We will include some geo data in the report") - client_geodata = geodata.IPToLocation(config.probe_ip.address) + client_geodata = geoip.IPToLocation(config.probe_ip.address) if config.privacy.includeip: client_geodata['ip'] = config.probe_ip.address diff --git a/ooni/reporter.py b/ooni/reporter.py index 20193c9..fc73b67 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -30,7 +30,7 @@ from ooni.errors import InvalidOONIBCollectorAddress from ooni.errors import ReportNotCreated, ReportAlreadyClosed from ooni import otime -from ooni.utils import geodata, pushFilenameStack +from ooni.utils import pushFilenameStack from ooni.utils.net import BodyReceiver, StringProducer, userAgents from ooni import config diff --git a/ooni/utils/geodata.py b/ooni/utils/geodata.py deleted file mode 100644 index d9883ba..0000000 --- a/ooni/utils/geodata.py +++ /dev/null @@ -1,40 +0,0 @@ -import re -import os - -from twisted.web.client import Agent -from twisted.internet import reactor, defer, protocol - -from ooni.utils import log, net -from ooni import config - -try: - import pygeoip -except ImportError: - log.err("Unable to import pygeoip. We will not be able to run geo IP related measurements") - -class GeoIPDataFilesNotFound(Exception): - pass - -def IPToLocation(ipaddr): - city_file = os.path.join(config.advanced.geoip_data_dir, 'GeoLiteCity.dat') - country_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIP.dat') - asn_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIPASNum.dat') - - location = {'city': None, 'countrycode': None, 'asn': None} - try: - city_dat = pygeoip.GeoIP(city_file) - location['city'] = city_dat.record_by_addr(ipaddr)['city'] - - country_dat = pygeoip.GeoIP(country_file) - location['countrycode'] = country_dat.country_code_by_addr(ipaddr) - - asn_dat = pygeoip.GeoIP(asn_file) - location['asn'] = asn_dat.org_by_addr(ipaddr) - - except IOError: - log.err("Could not find GeoIP data files. Go into data/ " - "and run make geoip") - raise GeoIPDataFilesNotFound - - return location - diff --git a/tests/test_nettest.py b/tests/test_nettest.py index 976757f..2d070c3 100644 --- a/tests/test_nettest.py +++ b/tests/test_nettest.py @@ -241,7 +241,7 @@ class TestNetTest(unittest.TestCase): # try: # net_test = NetTestLoader(StringIO(net_test_root_required), # dummyArgs) - # except NotRootError: + # except errors.InsufficientPrivileges: # pass #def test_create_report_succeed(self):

1 0

[ooni-probe/develop] Add test for checking the failure of the traceroute method
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit e63048bbe3e45ce597b80b8f7f7ac5de24557a54 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Mar 15 16:01:27 2013 +0100 Add test for checking the failure of the traceroute method Should raise an error when run as non root. --- ooni/errors.py | 3 ++- ooni/geoip.py | 4 ++-- ooni/utils/__init__.py | 2 ++ tests/test_geoip.py | 6 +++--- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/ooni/errors.py b/ooni/errors.py index 6b23727..62fcf93 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -14,7 +14,6 @@ from txsocksx.errors import HostUnreachable, ConnectionRefused from txsocksx.errors import TTLExpired, CommandNotSupported from socket import gaierror -from ooni.utils import log def handleAllFailures(failure): """ Here we make sure to trap all the failures that are supported by the @@ -44,6 +43,8 @@ def failureToString(failure): A string representing the HTTP response error message. """ + from ooni.utils import log + string = None if isinstance(failure.value, ConnectionRefusedError): log.err("Connection refused. The backend may be down") diff --git a/ooni/geoip.py b/ooni/geoip.py index 55c4fa4..e6a221b 100644 --- a/ooni/geoip.py +++ b/ooni/geoip.py @@ -7,8 +7,8 @@ from twisted.web import client, http_headers from ooni.utils.net import userAgents, BodyReceiver from twisted.internet import reactor, defer, protocol -from ooni.utils import log, net -from ooni import config +from ooni.utils import log, net, checkForRoot +from ooni import config, errors try: import pygeoip diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index 340693b..b039c8a 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -6,6 +6,8 @@ import yaml import imp import os +from ooni import errors + class Storage(dict): """ A Storage object is like a dictionary except `obj.foo` can be used diff --git a/tests/test_geoip.py b/tests/test_geoip.py index a4ad767..443d262 100644 --- a/tests/test_geoip.py +++ b/tests/test_geoip.py @@ -4,6 +4,7 @@ from twisted.trial import unittest from twisted.python.filepath import FilePath from twisted.protocols.policies import WrappingFactory +from ooni import errors from ooni.geoip import ProbeIP, MaxMindGeoIP, TorProjectGeoIP from ooni.geoip import UbuntuGeoIP, HTTPGeoIPLookupper, IPToLocation @@ -24,7 +25,6 @@ class MaxMindGeoIPResource(resource.Resource): class TorProjectGeoIPResource(resource.Resource): def render(self, request): - return """ Your IP address appears to be: 127.0.0.1 """ @@ -98,8 +98,8 @@ class TestProbeIP(GeoIPBaseTest): self.assertEqual(self.probe_ip.address, '127.0.0.1') return d - def test_ask_traceroute_service(self): - pass + def test_fail_traceroute_service(self): + self.assertRaises(errors.InsufficientPrivileges, self.probe_ip.askTraceroute) def test_ask_tor(self): pass

1 0

[ooni-probe/develop] Write unittests for geoip web services.
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit e851611afd663f4c58d482d39960b2dae6ac7de6 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Mar 15 15:02:04 2013 +0100 Write unittests for geoip web services. Fix some bugs found in unittesting. --- ooni/geoip.py | 26 +++++++----- tests/test_geoip.py | 109 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+), 9 deletions(-) diff --git a/ooni/geoip.py b/ooni/geoip.py index 0fe8a6b..55c4fa4 100644 --- a/ooni/geoip.py +++ b/ooni/geoip.py @@ -1,5 +1,7 @@ +import xml.etree.ElementTree as ET import re import os +import random from twisted.web import client, http_headers from ooni.utils.net import userAgents, BodyReceiver @@ -42,6 +44,11 @@ def IPToLocation(ipaddr): class HTTPGeoIPLookupper(object): url = None + _agent = client.Agent + + def __init__(self): + self.agent = self._agent(reactor) + def _response(self, response): content_length = response.headers.getRawHeaders('content-length') @@ -59,16 +66,15 @@ class HTTPGeoIPLookupper(object): pass def failed(self, failure): - log.err("Failed to lookup via %s" % url) + log.err("Failed to lookup via %s" % self.url) log.exception(failure) return failure def lookup(self): - agent = client.Agent(reactor) headers = {} headers['User-Agent'] = [random.choice(userAgents)] - d = agent.request("GET", self.url, http_headers.Headers(headers)) + d = self.agent.request("GET", self.url, http_headers.Headers(headers)) d.addCallback(self._response) d.addErrback(self.failed) return d @@ -101,7 +107,7 @@ class ProbeIP(object): strategy = None geoIPServices = {'ubuntu': UbuntuGeoIP, 'torproject': TorProjectGeoIP, - 'maximind': MaxMindGeoIP + 'maxmind': MaxMindGeoIP } address = None @@ -126,20 +132,22 @@ class ProbeIP(object): try: yield self.askGeoIPService() defer.returnValue(self.address) - except Exception, e: - print e + except: log.msg("Unable to lookup the probe IP via GeoIPService") @defer.inlineCallbacks def askGeoIPService(self): - for service_name, service in self.geoIPServices.items(): - s = TorProjectGeoIP() + # Shuffle the order in which we test the geoip services. + services = self.geoIPServices.items() + random.shuffle(services) + for service_name, service in services: + s = service() log.msg("Looking up your IP address via %s" % service_name) try: self.address = yield s.lookup() self.strategy = 'geo_ip_service-' + service_name break - except: + except Exception, e: log.msg("Failed to lookup your IP via %s" % service_name) def askTraceroute(self): diff --git a/tests/test_geoip.py b/tests/test_geoip.py new file mode 100644 index 0000000..a4ad767 --- /dev/null +++ b/tests/test_geoip.py @@ -0,0 +1,109 @@ +from twisted.web import server, static, resource +from twisted.internet import reactor +from twisted.trial import unittest +from twisted.python.filepath import FilePath +from twisted.protocols.policies import WrappingFactory + +from ooni.geoip import ProbeIP, MaxMindGeoIP, TorProjectGeoIP +from ooni.geoip import UbuntuGeoIP, HTTPGeoIPLookupper, IPToLocation + +class UbuntuGeoIPResource(resource.Resource): + def render(self, request): + return """ + <Response> + <Ip>127.0.0.1</Ip> + <Spam>Ham</Spam> + </Response> + """ + +class MaxMindGeoIPResource(resource.Resource): + def render(self, request): + return """ + 127.0.0.1 + """ + +class TorProjectGeoIPResource(resource.Resource): + def render(self, request): + + return """ + Your IP address appears to be: 127.0.0.1 + """ + +class GeoIPBaseTest(unittest.TestCase): + services = {'ubuntu': UbuntuGeoIPResource, + 'maxmind': MaxMindGeoIPResource, + 'torproject': TorProjectGeoIPResource + } + def _listen(self, site): + return reactor.listenTCP(0, site, interface="127.0.0.1") + + def setUp(self): + r = resource.Resource() + for name, service in self.services.items(): + r.putChild(name, service()) + self.site = server.Site(r, timeout=None) + + self.wrapper = WrappingFactory(self.site) + self.port = self._listen(self.wrapper) + self.portno = self.port.getHost().port + + def getUrl(self, service_name): + return "http://%s:%s/%s" % ('127.0.0.1', self.portno, service_name) + + def tearDown(self): + return self.port.stopListening() + +class TestGeoIPServices(GeoIPBaseTest): + def test_torproject_geoip(self): + gip = TorProjectGeoIP() + gip.url = self.getUrl('torproject') + d = gip.lookup() + @d.addCallback + def cb(res): + self.assertEqual(res, '127.0.0.1') + return d + + def test_ubuntu_geoip(self): + gip = UbuntuGeoIP() + gip.url = self.getUrl('ubuntu') + d = gip.lookup() + @d.addCallback + def cb(res): + self.assertEqual(res, '127.0.0.1') + return d + + def test_maxmind_geoip(self): + gip = MaxMindGeoIP() + gip.url = self.getUrl('maxmind') + d = gip.lookup() + @d.addCallback + def cb(res): + self.assertEqual(res, '127.0.0.1') + return d + +class TestProbeIP(GeoIPBaseTest): + def setUp(self): + GeoIPBaseTest.setUp(self) + + # Override the service addresses with those of the fake localhost + # resource. + self.probe_ip = ProbeIP() + for name in self.probe_ip.geoIPServices.keys(): + self.probe_ip.geoIPServices[name].url = self.getUrl(name) + + def test_ask_geoip_service(self): + d = self.probe_ip.askGeoIPService() + @d.addCallback + def cb(res): + self.assertEqual(self.probe_ip.address, '127.0.0.1') + return d + + def test_ask_traceroute_service(self): + pass + + def test_ask_tor(self): + pass + +class TestIPToLocation(unittest.TestCase): + pass +

1 0

[ooni-probe/develop] Support CPython with patch from lunar https://trac.torproject.org/projects/tor/ticket/8507
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit f5ccc326fa9715422bf0029a3f334643e247a9e8 Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Mar 19 13:39:18 2013 -0400 Support CPython with patch from lunar https://trac.torproject.org/projects/tor/ticket/8507 --- ooni/utils/geodata.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/ooni/utils/geodata.py b/ooni/utils/geodata.py index d9883ba..5bc55a7 100644 --- a/ooni/utils/geodata.py +++ b/ooni/utils/geodata.py @@ -8,9 +8,14 @@ from ooni.utils import log, net from ooni import config try: - import pygeoip + from pygeoip import GeoIP except ImportError: - log.err("Unable to import pygeoip. We will not be able to run geo IP related measurements") + try: + import GeoIP as CGeoIP + def GeoIP(database_path, *args, **kwargs): + return CGeoIP.open(database_path) + except ImportError: + log.err("Unable to import pygeoip or GeoIP. We will not be able to run geo IP related measurements") class GeoIPDataFilesNotFound(Exception): pass @@ -22,13 +27,13 @@ def IPToLocation(ipaddr): location = {'city': None, 'countrycode': None, 'asn': None} try: - city_dat = pygeoip.GeoIP(city_file) + city_dat = GeoIP(city_file) location['city'] = city_dat.record_by_addr(ipaddr)['city'] - country_dat = pygeoip.GeoIP(country_file) + country_dat = GeoIP(country_file) location['countrycode'] = country_dat.country_code_by_addr(ipaddr) - asn_dat = pygeoip.GeoIP(asn_file) + asn_dat = GeoIP(asn_file) location['asn'] = asn_dat.org_by_addr(ipaddr) except IOError:

1 0

[ooni-probe/develop] Merge branch 'master' into feature/probe_ip_lookup
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit c597157a554b47f59b1ee85fae2019a0dbc8834e Merge: 5f55b18 0b84394 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Mar 20 13:57:01 2013 -0400 Merge branch 'master' into feature/probe_ip_lookup * master: Change None to correct YAML "Null" syntax in ooniprobe.conf.sample. Support CPython with patch from lunar https://trac.torproject.org/projects/tor/ticket/8507 Add the 'reports' group to the sample config file. Change ooni.director.start() to set config.reports.pcap option. Move config processing options to new function config.processConfigFile(). Rewrite config.generatePcapFilename(). Conflicts: ooni/utils/geodata.py ooni/config.py | 61 ++++++++++++++++++++++++++++++++++--------------- ooni/director.py | 2 +- ooni/geoip.py | 15 ++++++++---- ooniprobe.conf.sample | 4 +++- 4 files changed, 57 insertions(+), 25 deletions(-) diff --cc ooni/geoip.py index efac5e9,0000000..0f8335f mode 100644,000000..100644 --- a/ooni/geoip.py +++ b/ooni/geoip.py @@@ -1,183 -1,0 +1,188 @@@ +import re +import os +import random + +from twisted.web import client, http_headers +from ooni.utils.net import userAgents, BodyReceiver +from twisted.internet import reactor, defer, protocol + +from ooni.utils import log, net, checkForRoot +from ooni import config, errors + +try: - import pygeoip ++ from pygeoip import GeoIP +except ImportError: - log.err("Unable to import pygeoip. We will not be able to run geo IP related measurements") ++ try: ++ import GeoIP as CGeoIP ++ def GeoIP(database_path, *args, **kwargs): ++ return CGeoIP.open(database_path) ++ except ImportError: ++ log.err("Unable to import pygeoip. We will not be able to run geo IP related measurements") + +class GeoIPDataFilesNotFound(Exception): + pass + +def IPToLocation(ipaddr): + city_file = os.path.join(config.advanced.geoip_data_dir, 'GeoLiteCity.dat') + country_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIP.dat') + asn_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIPASNum.dat') + + location = {'city': None, 'countrycode': None, 'asn': None} + try: - city_dat = pygeoip.GeoIP(city_file) ++ city_dat = GeoIP(city_file) + location['city'] = city_dat.record_by_addr(ipaddr)['city'] + - country_dat = pygeoip.GeoIP(country_file) ++ country_dat = GeoIP(country_file) + location['countrycode'] = country_dat.country_code_by_addr(ipaddr) + - asn_dat = pygeoip.GeoIP(asn_file) ++ asn_dat = GeoIP(asn_file) + location['asn'] = asn_dat.org_by_addr(ipaddr) + + except IOError: + log.err("Could not find GeoIP data files. Go into data/ " + "and run make geoip") + raise GeoIPDataFilesNotFound + + return location + +class HTTPGeoIPLookupper(object): + url = None + + _agent = client.Agent + + def __init__(self): + self.agent = self._agent(reactor) + + def _response(self, response): + content_length = response.headers.getRawHeaders('content-length') + + finished = defer.Deferred() + response.deliverBody(BodyReceiver(finished, content_length)) + finished.addCallback(self.parseResponse) + return finished + + def parseResponse(self, response_body): + """ + Override this with the logic for parsing the response. + + Should return the IP address of the probe. + """ + pass + + def failed(self, failure): + log.err("Failed to lookup via %s" % self.url) + log.exception(failure) + return failure + + def lookup(self): + headers = {} + headers['User-Agent'] = [random.choice(userAgents)] + + d = self.agent.request("GET", self.url, http_headers.Headers(headers)) + d.addCallback(self._response) + d.addErrback(self.failed) + return d + +class UbuntuGeoIP(HTTPGeoIPLookupper): + url = "http://geoip.ubuntu.com/lookup" + + def parseResponse(self, response_body): + m = re.match(".*<Ip>(.*)</Ip>.*", response_body) + probe_ip = m.group(1) + return probe_ip + +class TorProjectGeoIP(HTTPGeoIPLookupper): + url = "https://check.torproject.org/" + + def parseResponse(self, response_body): + regexp = "Your IP address appears to be: ((\d+\.)+(\d+))" + probe_ip = re.search(regexp, response_body).group(1) + return probe_ip + +class MaxMindGeoIP(HTTPGeoIPLookupper): + url = "https://www.maxmind.com/en/locate_my_ip" + + def parseResponse(self, response_body): + regexp = '((\d+\.)+(\d+))' + probe_ip = re.search(regexp, response_body).group(1) + return probe_ip + +class ProbeIP(object): + strategy = None + geoIPServices = {'ubuntu': UbuntuGeoIP, + 'torproject': TorProjectGeoIP, + 'maxmind': MaxMindGeoIP + } + address = None + tor_state = config.tor_state + + @defer.inlineCallbacks + def lookup(self): + try: + yield self.askTor() + log.msg("Found your IP via Tor %s" % self.address) + defer.returnValue(self.address) + except errors.TorStateNotFound: + log.debug("Tor is not running. Skipping IP lookup via Tor.") + except Exception: + log.msg("Unable to lookup the probe IP via Tor.") + + try: + yield self.askTraceroute() + log.msg("Found your IP via Traceroute %s" % self.address) + defer.returnValue(self.address) + except errors.InsufficientPrivileges: + log.debug("Cannot determine the probe IP address with a traceroute, becase of insufficient priviledges") + except: + log.msg("Unable to lookup the probe IP via traceroute") + + try: + yield self.askGeoIPService() + log.msg("Found your IP via a GeoIP service: %s" % self.address) + defer.returnValue(self.address) + except Exception, e: + log.msg("Unable to lookup the probe IP via GeoIPService") + raise e + + @defer.inlineCallbacks + def askGeoIPService(self): + # Shuffle the order in which we test the geoip services. + services = self.geoIPServices.items() + random.shuffle(services) + for service_name, service in services: + s = service() + log.msg("Looking up your IP address via %s" % service_name) + try: + self.address = yield s.lookup() + self.strategy = 'geo_ip_service-' + service_name + break + except Exception, e: + log.msg("Failed to lookup your IP via %s" % service_name) + + if not self.address: + raise errors.ProbeIPUnknown + + def askTraceroute(self): + """ + Perform a UDP traceroute to determine the probes IP address. + """ + checkForRoot() + raise NotImplemented + + def askTor(self): + """ + Obtain the probes IP address by asking the Tor Control port via GET INFO + address. + + XXX this lookup method is currently broken when there are cached descriptors or consensus documents + see: https://trac.torproject.org/projects/tor/ticket/8214 + """ + if self.tor_state: + d = self.tor_state.protocol.get_info("address") + @d.addCallback + def cb(result): + self.strategy = 'tor_get_info_address' + self.address = result.values()[0] + return d + else: + raise errors.TorStateNotFound

1 0

[ooni-probe/develop] Use regexp instead of parsing XML.
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 5f55b1822b1dd34b50b4f4f1a8dc0e9f0f64a6e4 Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Mar 19 13:52:31 2013 -0400 Use regexp instead of parsing XML. Fix unittest of the ubuntu check --- ooni/director.py | 1 - ooni/geoip.py | 5 ++--- tests/test_geoip.py | 7 +------ 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/ooni/director.py b/ooni/director.py index d2f5a7c..7c8c75b 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -1,4 +1,3 @@ -import xml.etree.ElementTree as ET import random import sys import os diff --git a/ooni/geoip.py b/ooni/geoip.py index ae06608..efac5e9 100644 --- a/ooni/geoip.py +++ b/ooni/geoip.py @@ -1,4 +1,3 @@ -import xml.etree.ElementTree as ET import re import os import random @@ -83,8 +82,8 @@ class UbuntuGeoIP(HTTPGeoIPLookupper): url = "http://geoip.ubuntu.com/lookup" def parseResponse(self, response_body): - response = ET.fromstring(response_body) - probe_ip = response.find('Ip').text + m = re.match(".*<Ip>(.*)</Ip>.*", response_body) + probe_ip = m.group(1) return probe_ip class TorProjectGeoIP(HTTPGeoIPLookupper): diff --git a/tests/test_geoip.py b/tests/test_geoip.py index 0d7165f..88f3dcb 100644 --- a/tests/test_geoip.py +++ b/tests/test_geoip.py @@ -12,12 +12,7 @@ from ooni.geoip import UbuntuGeoIP, HTTPGeoIPLookupper, IPToLocation class UbuntuGeoIPResource(resource.Resource): def render(self, request): - return """ - <Response> - <Ip>127.0.0.1</Ip> - <Spam>Ham</Spam> - </Response> - """ + return """<?xml version="1.0" encoding="UTF-8"?><Response><Ip>127.0.0.1</Ip><Status>OK</Status></Response>""" class MaxMindGeoIPResource(resource.Resource): def render(self, request):

1 0

[ooni-probe/develop] Merge branch 'feature/pygeoip'
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 0b843947b56b09d13346c221081ce63f32cdea5f Merge: 091e45e f5ccc32 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Mar 20 13:50:38 2013 -0400 Merge branch 'feature/pygeoip' * feature/pygeoip: Support CPython with patch from lunar https://trac.torproject.org/projects/tor/ticket/8507 ooni/utils/geodata.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-)

1 0

[ooni-probe/develop] Merge pull request #51 from hellais/experimental-tests
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 230da42ae4c5caf6ee39df83dc437c4647794d98 Merge: 2ee75d8 0c49af0 Author: aagbsn <aagbsn(a)extc.org> Date: Wed Mar 20 11:11:19 2013 -0700 Merge pull request #51 from hellais/experimental-tests Update HTTP Host test and Experimental tests nettests/experimental/http_filtering_bypassing.py | 84 ++++++++++++++++++++ nettests/experimental/http_trix.py | 47 +++++++++++ nettests/manipulation/http_host.py | 64 +++++++++++---- nettests/manipulation/http_invalid_request_line.py | 5 +- 4 files changed, 180 insertions(+), 20 deletions(-)

1 0

[ooni-probe/develop] Merge pull request #52 from hellais/bug7999
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 1d4d38555b8d37006044f2f59a0e263bf0e4742c Merge: 230da42 0c99be6 Author: aagbsn <aagbsn(a)extc.org> Date: Wed Mar 20 11:22:25 2013 -0700 Merge pull request #52 from hellais/bug7999 Add support for multiple inheritance in tests nettests/examples/example_dns_http.py | 11 +++++++++++ ooni/templates/dnst.py | 10 ++++++---- ooni/templates/httpt.py | 8 +++++--- ooni/templates/scapyt.py | 2 ++ ooni/templates/tcpt.py | 2 ++ 5 files changed, 26 insertions(+), 7 deletions(-)

1 0

[ooni-probe/develop] Merge pull request #71 from hellais/feature/probe_ip_lookup
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 2ee75d8d1fcb97b96b462d8b5ffbbc8a2c5c8a96 Merge: 0b84394 c597157 Author: aagbsn <aagbsn(a)extc.org> Date: Wed Mar 20 11:05:42 2013 -0700 Merge pull request #71 from hellais/feature/probe_ip_lookup Feature/probe ip lookup HACKING | 2 +- ooni/director.py | 23 +++--- ooni/errors.py | 11 ++- ooni/geoip.py | 188 ++++++++++++++++++++++++++++++++++++++++++++++++ ooni/nettest.py | 9 +-- ooni/reporter.py | 14 ++-- ooni/runner.py | 2 +- ooni/utils/__init__.py | 7 +- ooni/utils/geodata.py | 45 ------------ tests/test_geoip.py | 137 +++++++++++++++++++++++++++++++++++ tests/test_nettest.py | 3 +- 11 files changed, 366 insertions(+), 75 deletions(-)

1 0