commit dec1d60e13087cd16cc7dd9feec1cd86828d275c Author: Alden S. Page <pagea@allegheny.edu> Date: Thu Jan 1 21:24:09 2015 -0500 Added countryCode attribute, refactored GeoIP logic into its own module. Instances of Bridge now keep track of their country codes. This should help us hand out bridges more intelligently. In particular, we will soon be able to withold bridges from .ir and .sy as per issue #12843. Since multiple modules make use of geoip (HTTPServer and Bridges), I moved the geoip logic out of HTTPServer and into its own module called geo.py. Signed-off-by: Isis Lovecruft <isis@torproject.org> I removed the bridgedb.Bridges.Bridge.coutryCode attribute because it'll need to be ported to bridgedb.bridges.Bridge now that #9380 is merged. --- lib/bridgedb/Bridges.py | 1 + lib/bridgedb/HTTPServer.py | 29 +++--------------- lib/bridgedb/geo.py | 71 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 25 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index bdb911a..0904d67 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -21,6 +21,7 @@ import random import bridgedb.Storage import bridgedb.Bucket +import bridgedb.geo from bridgedb.bridges import Bridge from bridgedb.crypto import getHMACFunc diff --git a/lib/bridgedb/HTTPServer.py b/lib/bridgedb/HTTPServer.py index 389d5a4..3a51f7c 100644 --- a/lib/bridgedb/HTTPServer.py +++ b/lib/bridgedb/HTTPServer.py @@ -24,6 +24,7 @@ from functools import partial from ipaddr import IPv4Address from ipaddr import IPv6Address +from ipaddr import IPAddress import mako.exceptions from mako.template import Template @@ -38,6 +39,7 @@ from twisted.web import static from twisted.web.util import redirectTo import bridgedb.Dist +import bridgedb.geo from bridgedb import captcha from bridgedb import crypto @@ -55,7 +57,6 @@ from bridgedb.safelog import logSafely TEMPLATE_DIR = os.path.join(os.path.dirname(__file__), 'templates') -GEOIP_DBFILE = '/usr/share/GeoIP/GeoIP.dat' rtl_langs = ('ar', 'he', 'fa', 'gu_IN', 'ku') # Setting `filesystem_checks` to False is recommended for production servers, @@ -71,23 +72,6 @@ lookup = TemplateLookup(directories=[TEMPLATE_DIR], collection_size=500) logging.debug("Set template root to %s" % TEMPLATE_DIR) -try: - # Make sure we have the database before trying to import the module: - if not os.path.isfile(GEOIP_DBFILE): # pragma: no cover - raise EnvironmentError("Could not find %r. On Debian-based systems, "\ - "please install the geoip-database package." - % GEOIP_DBFILE) - # This is a "pure" python version which interacts with the Maxmind GeoIP - # API (version 1). It requires, in Debian, the libgeoip-dev and - # geoip-database packages. - import pygeoip - geoip = pygeoip.GeoIP(GEOIP_DBFILE, flags=pygeoip.MEMORY_CACHE) - logging.info("GeoIP database loaded") -except Exception as err: # pragma: no cover - logging.warn("Error while loading geoip module: %r" % err) - geoip = None - - def replaceErrorPage(error, template_name=None): """Create a general error page for displaying in place of tracebacks. @@ -691,12 +675,8 @@ class WebResourceBridges(resource.Resource): else: ip = request.getClientIP() - # XXX This can also be a separate function - # XXX if the ip is None, this throws an exception - if geoip: - countryCode = geoip.country_code_by_addr(ip) - if countryCode: - logging.debug("Client request from GeoIP CC: %s" % countryCode) + # Record what country the client is in. + countryCode = bridgedb.geo.getCountryCode(IPAddress(ip)) # XXX separate function again format = request.args.get("format", None) @@ -806,7 +786,6 @@ class WebResourceBridges(resource.Resource): return rendered - class WebRoot(resource.Resource): """The parent resource of all other documents hosted by the webserver.""" diff --git a/lib/bridgedb/geo.py b/lib/bridgedb/geo.py new file mode 100644 index 0000000..b2be72c --- /dev/null +++ b/lib/bridgedb/geo.py @@ -0,0 +1,71 @@ +# +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: see AUTHORS file +# :copyright: (c) 2007-2015, The Tor Project, Inc. +# :license: 3-Clause BSD, see LICENSE for licensing information + +""" +Boilerplate setup for GeoIP. GeoIP allows us to look up the country code +associated with an IP address. This is a "pure" python version which interacts +with the Maxmind GeoIP API (version 1). It requires, in Debian, the libgeoip-dev +and geoip-database packages. +""" + +import logging +from os.path import isfile + +from bridgedb.safelog import logSafely +from ipaddr import IPv4Address, IPv6Address + +# IPv4 database +GEOIP_DBFILE = '/usr/share/GeoIP/GeoIP.dat' +# IPv6 database +GEOIPv6_DBFILE = '/usr/share/GeoIP/GeoIPv6.dat' +try: + # Make sure we have the database before trying to import the module: + if not (isfile(GEOIP_DBFILE) and isfile(GEOIPv6_DBFILE)): + raise EnvironmentError("Could not find %r. On Debian-based systems, "\ + "please install the geoip-database package." + % GEOIP_DBFILE) + + import pygeoip + geoip = pygeoip.GeoIP(GEOIP_DBFILE, flags=pygeoip.MEMORY_CACHE) + geoipv6 = pygeoip.GeoIP(GEOIPv6_DBFILE, flags=pygeoip.MEMORY_CACHE) + logging.info("GeoIP databases loaded") +except Exception as err: # pragma: no cover + logging.warn("Error while loading geoip module: %r" % err) + geoip = None + +def getCountryCode(IPAddr): + """Returns the two-letter country code of a given IP address. + + :param IPAddr: (:class:`ipaddr.IPAddress`) An IPv4 OR IPv6 address. + """ + ip = None + version = None + try: + ip = IPAddr.exploded + version = IPAddr.version + except AttributeError as err: + logging.warn("Wrong type passed to getCountryCode. Offending call:" + " %r" % err) + return None + + # GeoIP has two databases: one for IPv4 addresses, and one for IPv6 + # addresses. This will ensure we use the correct one. + db = None + if version == 4: + db = geoip + else: + db = geoipv6 + + # Look up the country code of the address. + countryCode = db.country_code_by_addr(ip) + if countryCode: + logging.debug("Looked up country code: %s" % countryCode) + return countryCode + else: + logging.debug("Country code was not detected. IP: %s" % ip) + return None