March 2015 - tor-commits - lists.torproject.org

[bridgedb/master] Fixed some defects isis found in the previous patch
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit 4c5c35dca9ec768d8b5db031a9253eb50b3544f7 Author: Alden S. Page <pagea(a)allegheny.edu> Date: Wed Jan 7 16:07:29 2015 -0500 Fixed some defects isis found in the previous patch Changes in geo.py: Made sure we loaded GeoIP databases without a hitch before looking up any IPs. Load compressed IP from IPAddr instead of exploded. Removed unused import. Changes in HTTPServer.py: Restored accidentally removed whitespace; validated ip before calling getCountryCode. Changes in Bridges.py: Fixed documentation whitespace issue, validated ip before calling getCountryCode. Signed-off-by: Isis Lovecruft <isis(a)torproject.org> Again, I removed the changes to bridgedb.Bridges.Bridge, since the class has been rewritten for #9380 and now lives in bridgedb.bridges.Bridge. --- lib/bridgedb/HTTPServer.py | 9 +++++++-- lib/bridgedb/geo.py | 21 ++++++++++++++------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/lib/bridgedb/HTTPServer.py b/lib/bridgedb/HTTPServer.py index 3a51f7c..258861e 100644 --- a/lib/bridgedb/HTTPServer.py +++ b/lib/bridgedb/HTTPServer.py @@ -675,8 +675,12 @@ class WebResourceBridges(resource.Resource): else: ip = request.getClientIP() - # Record what country the client is in. - countryCode = bridgedb.geo.getCountryCode(IPAddress(ip)) + # Look up the country code of the input IP + if isIPAddress(ip): + countryCode = bridgedb.geo.getCountryCode(IPAddress(ip)) + else: + logging.warn("Invalid IP detected; skipping country lookup.") + countryCode = None # XXX separate function again format = request.args.get("format", None) @@ -786,6 +790,7 @@ class WebResourceBridges(resource.Resource): return rendered + class WebRoot(resource.Resource): """The parent resource of all other documents hosted by the webserver.""" diff --git a/lib/bridgedb/geo.py b/lib/bridgedb/geo.py index b2be72c..1e2e9e7 100644 --- a/lib/bridgedb/geo.py +++ b/lib/bridgedb/geo.py @@ -16,7 +16,6 @@ and geoip-database packages. import logging from os.path import isfile -from bridgedb.safelog import logSafely from ipaddr import IPv4Address, IPv6Address # IPv4 database @@ -37,16 +36,18 @@ try: except Exception as err: # pragma: no cover logging.warn("Error while loading geoip module: %r" % err) geoip = None + geoipv6 = None def getCountryCode(IPAddr): - """Returns the two-letter country code of a given IP address. + """Return the two-letter country code of a given IP address. :param IPAddr: (:class:`ipaddr.IPAddress`) An IPv4 OR IPv6 address. """ + ip = None version = None try: - ip = IPAddr.exploded + ip = IPAddr.compressed version = IPAddr.version except AttributeError as err: logging.warn("Wrong type passed to getCountryCode. Offending call:" @@ -55,11 +56,17 @@ def getCountryCode(IPAddr): # GeoIP has two databases: one for IPv4 addresses, and one for IPv6 # addresses. This will ensure we use the correct one. - db = None - if version == 4: - db = geoip + db = None + # First, make sure we loaded GeoIP properly. + if None in (geoip, geoipv6): + logging.warn("GeoIP databases failed to load; could not look up"\ + " country code.") + return None else: - db = geoipv6 + if version == 4: + db = geoip + else: + db = geoipv6 # Look up the country code of the address. countryCode = db.country_code_by_addr(ip)

1 0

[bridgedb/master] Add GeoIP CCs as bridgedb.bridges.BridgeAddressBase.country @property.
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit 38c5f2db3a02fc49584e0d0a166d11efc8314d4a Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Feb 18 23:39:07 2015 +0000 Add GeoIP CCs as bridgedb.bridges.BridgeAddressBase.country @property. This adds a ``country`` @property to Bridges and PluggableTransports, which, when accessed, determines the two-letter GeoIP country code of the bridge/PT's ``address`` @property. (Therefore, this doesn't work on additional ORAddresses currently, but it does work on all other bridge IP addresses.) These changes add back in the functionality that Alden S. Page contributed in the patches for #12872 to the bridgedb.Bridges.Bridge class, but which had to be removed due to conflicts with the changes for * ADD a BridgeAddressBase.country @property. * FIXES part of #12872. --- lib/bridgedb/bridges.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/lib/bridgedb/bridges.py b/lib/bridgedb/bridges.py index 10e1ffc..b791730 100644 --- a/lib/bridgedb/bridges.py +++ b/lib/bridgedb/bridges.py @@ -25,6 +25,7 @@ from Crypto.Util.number import long_to_bytes import bridgedb.Storage +from bridgedb import geo from bridgedb import safelog from bridgedb import bridgerequest from bridgedb.crypto import removePKCS1Padding @@ -141,11 +142,15 @@ class BridgeAddressBase(object): :type address: ``ipaddr.IPv4Address`` or ``ipaddr.IPv6Address`` :ivar address: The IP address of :class:`Bridge` or one of its :class:`PluggableTransport`s. + + :type country: str + :ivar country: The two-letter GeoIP country code of the :ivar:`address`. """ def __init__(self): self._fingerprint = None self._address = None + self._country = None @property def fingerprint(self): @@ -199,6 +204,18 @@ class BridgeAddressBase(object): """Reset this Bridge's address to ``None``.""" self._address = None + @property + def country(self): + """Get the two-letter GeoIP country code for the :ivar:`address`. + + :rtype: str or ``None`` + :returns: If :ivar:`address` is set, this returns a two-letter country + code for the geolocated region that :ivar:`address` is within; + otherwise, returns ``None``. + """ + if self.address: + return geo.getCountryCode(self.address) + class PluggableTransport(BridgeAddressBase): """A single instance of a Pluggable Transport (PT) offered by a

1 0

[bridgedb/master] Add unittest for bridgedb.bridges.BridgeAddressBase.country @property.
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit fdf4afad33f8759f4154797a4bd37d678ff40daa Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 19 00:42:18 2015 +0000 Add unittest for bridgedb.bridges.BridgeAddressBase.country @property. --- lib/bridgedb/test/test_bridges.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/bridgedb/test/test_bridges.py b/lib/bridgedb/test/test_bridges.py index f63b25e..d42fb24 100644 --- a/lib/bridgedb/test/test_bridges.py +++ b/lib/bridgedb/test/test_bridges.py @@ -406,6 +406,18 @@ class BridgeAddressBaseTests(unittest.TestCase): self.assertIsNone(self.bab.address) self.assertIsNone(self.bab._address) + def test_BridgeAddressBase_country(self): + """The getter method for the country property should get the + address's geoIP country code. + """ + self.bab.address = '11.12.13.14' + self.assertEqual(self.bab.address, ipaddr.IPv4Address('11.12.13.14')) + + cc = self.bab.country + self.assertIsNotNone(cc) + self.assertIsInstance(cc, basestring) + self.assertEqual(len(cc), 2) + class PluggableTransportTests(unittest.TestCase): """Tests for :class:`bridgedb.bridges.PluggableTransport."""

1 0

[bridgedb/master] Add unittests for bridgedb.geo module.
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit feeb440961f3c8db4cc4e2476ee0373c4f49da94 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 19 00:41:22 2015 +0000 Add unittests for bridgedb.geo module. --- lib/bridgedb/test/test_geo.py | 96 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/lib/bridgedb/test/test_geo.py b/lib/bridgedb/test/test_geo.py new file mode 100644 index 0000000..5e5547b --- /dev/null +++ b/lib/bridgedb/test/test_geo.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft <isis(a)torproject.org> +# please also see AUTHORS file +# :copyright: (c) 2007-2015, The Tor Project, Inc. +# (c) 2007-2015, Isis Lovecruft +# :license: see LICENSE for licensing information + +"""Unittests for the :mod:`bridgedb.geo` module.""" + +import ipaddr + +from twisted.trial import unittest + +from bridgedb import geo + + +class GeoTests(unittest.TestCase): + """Unittests for the :mod:`bridgedb.geo` module.""" + + def setUp(self): + self._orig_geoip = geo.geoip + self._orig_geoipv6 = geo.geoipv6 + + self.ipv4 = ipaddr.IPAddress('38.229.72.16') + self.ipv6 = ipaddr.IPAddress('2620:0:6b0:b:1a1a:0:26e5:4810') + + # WARNING: this is going to fail if the torproject.org A and AAAA + # records above are reassigned to another host, or are randomly + # geolocated to somewhere other than where they are currently + # geolocated (in the US): + self.expectedCC = 'US' + + def tearDown(self): + geo.geoip = self._orig_geoip + geo.geoipv6 = self._orig_geoipv6 + + def test_geo_getCountryCode_ipv4_str(self): + """Should return None since the IP isn't an ``ipaddr.IPAddress``.""" + self.assertIsNone(geo.getCountryCode(str(self.ipv4))) + + def test_geo_getCountryCode_ipv4_no_geoip_loopback(self): + """Should return None since this IP isn't geolocatable (hopefully ever).""" + ipv4 = ipaddr.IPAddress('127.0.0.1') + self.assertIsNone(geo.getCountryCode(ipv4)) + + def test_geo_getCountryCode_ipv4_class(self): + """Should return the CC since the IP is an ``ipaddr.IPAddress``.""" + cc = geo.getCountryCode(self.ipv4) + self.assertIsNotNone(cc) + self.assertIsInstance(cc, basestring) + self.assertEqual(len(cc), 2) + self.assertEqual(cc, self.expectedCC) + + def test_geo_getCountryCode_ipv6_str(self): + """Should return None since the IP isn't an ``ipaddr.IPAddress``.""" + self.assertIsNone(geo.getCountryCode(str(self.ipv6))) + + def test_geo_getCountryCode_ipv6_no_geoip_record(self): + """Should return None since this IP isn't geolocatable (yet).""" + ipv6 = ipaddr.IPAddress('20::72a:e224:44d8:a606:4115') + self.assertIsNone(geo.getCountryCode(ipv6)) + + def test_geo_getCountryCode_ipv6_no_geoip_link_local(self): + """Should return None since this IP isn't geolocatable (hopefully ever).""" + ipv6 = ipaddr.IPAddress('ff02::') + self.assertIsNone(geo.getCountryCode(ipv6)) + + def test_geo_getCountryCode_ipv6_no_geoip_loopback(self): + """Should return None since this IP isn't geolocatable (hopefully ever).""" + ipv6 = ipaddr.IPAddress('::1') + self.assertIsNone(geo.getCountryCode(ipv6)) + + def test_geo_getCountryCode_ipv6_class(self): + """Should return the CC since the IP is an ``ipaddr.IPAddress``.""" + cc = geo.getCountryCode(self.ipv6) + self.assertIsNotNone(cc) + self.assertIsInstance(cc, basestring) + self.assertEqual(len(cc), 2) + self.assertEqual(cc, self.expectedCC) + + def test_geo_getCountryCode_no_geoip(self): + """When missing the geo.geoip database, getCountryCode() should return + None. + """ + geo.geoip = None + self.assertIsNone(geo.getCountryCode(self.ipv4)) + + def test_geo_getCountryCode_no_geoipv6(self): + """When missing the geo.geoipv6 database, getCountryCode() should + return None. + """ + geo.geoipv6 = None + self.assertIsNone(geo.getCountryCode(self.ipv4))

1 0

[bridgedb/master] And coverage skip pragma to GeoIP DB file check in bridgedb.geo.
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit 877ea4f353120526ff06dfb4977bf73dfde88318 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 19 00:46:16 2015 +0000 And coverage skip pragma to GeoIP DB file check in bridgedb.geo. --- lib/bridgedb/geo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bridgedb/geo.py b/lib/bridgedb/geo.py index 1e2e9e7..2a574ae 100644 --- a/lib/bridgedb/geo.py +++ b/lib/bridgedb/geo.py @@ -24,7 +24,7 @@ GEOIP_DBFILE = '/usr/share/GeoIP/GeoIP.dat' GEOIPv6_DBFILE = '/usr/share/GeoIP/GeoIPv6.dat' try: # Make sure we have the database before trying to import the module: - if not (isfile(GEOIP_DBFILE) and isfile(GEOIPv6_DBFILE)): + if not (isfile(GEOIP_DBFILE) and isfile(GEOIPv6_DBFILE)): # pragma: no cover raise EnvironmentError("Could not find %r. On Debian-based systems, "\ "please install the geoip-database package." % GEOIP_DBFILE)

1 0

[bridgedb/master] Add CHANGELOG entries for BridgeDB-0.3.0.
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit dd2398c1b303bf0e7deaa4f03e3e20000d6fb906 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 19 04:14:23 2015 +0000 Add CHANGELOG entries for BridgeDB-0.3.0. --- CHANGELOG | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index 8c21489..27aeed0 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,56 @@ +Changes in version 0.3.0 - XXX + + * FIXES #2895 https://bugs.torproject.org/2895 + BridgeDB no longer assumes that any extrainfo descriptor files are + in chronological order. + + * FIXES #4405 https://bugs.torproject.org/4405 + BridgeDB now has a built-in timer mechanism for scheduling + cronjobesque events. This is now used to routinely download and + parse the list of Tor exit relays in a completely asynchronous + manner. + + * FIXES #9380 https://bugs.torproject.org/9380 + BridgeDB now uses Stem (https://stem.torproject.org) for its + parsers, and has better classes for parsing and storing + information on Bridges and their Pluggable Transports. + Additionally, all of BridgeDB's parses and the new + Bridge/PluggableTransport classes all have 100% unittest and + integration test coverage. + + * FIXES #11216 https://bugs.torproject.org/11216 + BridgeDB no longer parses any extrainfo descriptor files + cumulatively. Before, a Bridge which had a descriptor in + cached-extrainfo and in cached-extrainfo.new and supported obfs3, + obfs4, and scramblesuit transports would be parsed twice, + resulting in the Bridge having six transports. This is no longer + the case. + + * HOTFIXES an issue with non-deterministic unittest failures in + the Mechanize-based integrations tests in + lib/bridgedb/test/test_https.py. hotfix/0.2.4-mechanize-tags + + * FIXES #12872 https://bugs.torproject.org/12872 + BridgeDB now has geolocational information for Bridges, telling it + which country each Bridge's primary ORAddress is within, as well + as geolocational information for each PluggableTransport address. + Thanks to Alden S. Page for the patches. + +And includes the following general changes: + * CHANGES BridgeDB's continuous integration infrastructure to run + tests for: + - Twisted-13.2.0 (Debian Wheezy version), + - Twisted-14.0.2 (Debian Jessie version), and + - Twisted-15.0.0 (latest and greatest) + As well as testing both: + - pyOpenSSL-0.13.1 (Debian Wheezy version), and + - pyOpenSSL-0.14 (Debian Jessie version). + See https://travis-ci.org/isislovecruft/bridgedb/builds + + * FIXES an issue with the $PYTHON_EGG_CACHE directory being group + writable on Travis-CI build machines. + + Changes in version 0.2.4 - 2015-02-03 * HOTFIXES a UnicodeDecodeError resulting from patches for #12627.

1 0

[bridgedb/master] Remove an unnecessary backslash from a multiline string in bridgedb.geo.
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit e38e1374acb3bf6e21ffafe8b4edfe0003b80d1a Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 19 00:47:01 2015 +0000 Remove an unnecessary backslash from a multiline string in bridgedb.geo. --- lib/bridgedb/geo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bridgedb/geo.py b/lib/bridgedb/geo.py index 2a574ae..3b48016 100644 --- a/lib/bridgedb/geo.py +++ b/lib/bridgedb/geo.py @@ -25,7 +25,7 @@ GEOIPv6_DBFILE = '/usr/share/GeoIP/GeoIPv6.dat' try: # Make sure we have the database before trying to import the module: if not (isfile(GEOIP_DBFILE) and isfile(GEOIPv6_DBFILE)): # pragma: no cover - raise EnvironmentError("Could not find %r. On Debian-based systems, "\ + raise EnvironmentError("Could not find %r. On Debian-based systems, " "please install the geoip-database package." % GEOIP_DBFILE)

1 0

[bridgedb/master] Install libgeoip-dev and geoip-database packages for CI builds.
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit 21b9a3916c273e483a58d1101eb3991f65afc00b Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 19 01:01:13 2015 +0000 Install libgeoip-dev and geoip-database packages for CI builds. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2765078..a0f4814 100644 --- a/.travis.yml +++ b/.travis.yml @@ -59,7 +59,7 @@ before_install: - chmod -R og-w $PYTHON_EGG_CACHE install: - - sudo apt-get install -qq --no-install-suggests --no-install-recommends build-essential openssl sqlite3 libgpgme11 libgpgme11-dev python-dev python-setuptools + - sudo apt-get install -qq --no-install-suggests --no-install-recommends build-essential openssl sqlite3 libgpgme11 libgpgme11-dev python-dev python-setuptools libgeoip-dev geoip-database - pip install -q --no-use-wheel -r .travis.requirements.txt - pip install -q --no-use-wheel Twisted==$TWISTED_VERSION pyOpenSSL==$PYOPENSSL_VERSION - make install

1 0

[bridgedb/master] Remove unused import of bridgedb.geo in bridgedb.Bridges module.
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit 45b092770e776e19189f66a5aa081331a367f7fa Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Feb 19 01:04:38 2015 +0000 Remove unused import of bridgedb.geo in bridgedb.Bridges module. Fix on dec1d60; it was my mistake that the hunk got added while rebasing the patches on top of the #9380 changes. --- lib/bridgedb/Bridges.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index 0904d67..bdb911a 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -21,7 +21,6 @@ import random import bridgedb.Storage import bridgedb.Bucket -import bridgedb.geo from bridgedb.bridges import Bridge from bridgedb.crypto import getHMACFunc

1 0

[bridgedb/master] Fix missing import of addOrUpdateBridgeHistory() in bridgedb.Main.
by isis＠torproject.org 21 Mar '15

21 Mar '15

commit 70a90f3c65cb63a6839b75dadd6999b422bd56ec Author: Isis Lovecruft <isis(a)torproject.org> Date: Sat Feb 21 22:43:11 2015 +0000 Fix missing import of addOrUpdateBridgeHistory() in bridgedb.Main. --- lib/bridgedb/Main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 7bc5cff..440784b 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -46,6 +46,8 @@ import bridgedb.Bridges as Bridges import bridgedb.Dist as Dist import bridgedb.Storage +from bridgedb.Stability import addOrUpdateBridgeHistory + def updateBridgeHistory(bridges, timestamps): """Process all the timestamps and update the bridge stability statistics in @@ -76,7 +78,7 @@ def updateBridgeHistory(bridges, timestamps): ("Adding/updating timestamps in BridgeHistory for %s in " "database: %s") % (fingerprint, timestamp)) timestamp = toUnixSeconds(timestamp.timetuple()) - bridgedb.Stability.addOrUpdateBridgeHistory(bridge, timestamp) + addOrUpdateBridgeHistory(bridge, timestamp) # Replace the timestamps so the next sort is (hopefully) less # expensive: sortedTimestamps[fingerprint] = stamps

1 0