[tor-commits] [bridgedb/main] Update BridgeDB to use tor-geoipdb for IP-country mapping
meskio at torproject.org
meskio at torproject.org
Thu Nov 25 12:17:13 UTC 2021
commit 58afbd9284f07440533ecab3ec709f113c0443e3
Author: lorik55 <abhilashmhaisne at gmail.com>
Date: Fri Nov 12 16:50:26 2021 +0530
Update BridgeDB to use tor-geoipdb for IP-country mapping
Update missing copyright notice
Update from Maxmind to torgeoip DB: Fixes and Improvements
Update from Maxmind to torgeoip DB: Improve performance for IPv6 lookup
Update from Maxmind to torgeoipdb: Add tor-geoip as requirement for CI
Update install-dependencies
Update install-dependencies
Update install-dependencies
Skip commented lines while loading geoIP DB. Update install dependencies.
geo.py minor fix
Update geo.py: Fix issue for out of range addresses
Pycodestyle fixes and comment modifications
Update geo.py to use tor-geoip DB
Update geo.py to use tor-geoip DB
---
bridgedb/geo.py | 102 ++++++++++++++++++++++++++++++++++++++-----
scripts/install-dependencies | 2 +-
2 files changed, 91 insertions(+), 13 deletions(-)
diff --git a/bridgedb/geo.py b/bridgedb/geo.py
index b42ef4f..67ff374 100644
--- a/bridgedb/geo.py
+++ b/bridgedb/geo.py
@@ -9,8 +9,7 @@
"""
Boilerplate setup for GeoIP. GeoIP allows us to look up the country code
associated with an IP address. This is a "pure" python version which interacts
-with the Maxmind GeoIP API (version 1). It requires, in Debian, the libgeoip-dev
-and geoip-database packages.
+with Tor GeoIP DB. It requires, in Debian, the tor-geoipdb package.
"""
import logging
@@ -19,26 +18,105 @@ from os.path import isfile
from ipaddr import IPv4Address, IPv6Address
# IPv4 database
-GEOIP_DBFILE = '/usr/share/GeoIP/GeoIP.dat'
+GEOIP_DBFILE = '/usr/share/tor/geoip'
# IPv6 database
-GEOIPv6_DBFILE = '/usr/share/GeoIP/GeoIPv6.dat'
+GEOIPv6_DBFILE = '/usr/share/tor/geoip6'
+
+
+def loadFromGeoIPDB(filepath):
+ """Load entries from IPV4 or IPV6 Tor Geo IP DB Files.
+
+ :param str filepath: Path to Tor GeoIP DB file.
+ :rtype: ``None`` or list
+
+ :returns: Returns a table containing all entries from Tor Geo IP DB file.
+ """
+ parsedTable = []
+ with open(filepath) as fd:
+ rawData = fd.read()
+
+ splitLines = rawData.split('\n')
+ for line in splitLines[:-1]:
+ if line.startswith('#'):
+ continue
+ singleRecord = line.split(',')
+ singleRecord[0] = int(singleRecord[0])
+ singleRecord[1] = int(singleRecord[1])
+ parsedTable.append(singleRecord)
+
+ return parsedTable
+
+
+def loadFromGeoIPDB6(filepath):
+ """Load entries from IPV4 or IPV6 Tor Geo IP DB Files.
+
+ :param str filepath: Path to Tor GeoIP DB file.
+ :rtype: ``None`` or list
+
+ :returns: Returns a table containing all entries from Tor Geo IP DB file.
+ """
+ parsedTable = []
+ with open(filepath) as fd:
+ rawData = fd.read()
+
+ splitLines = rawData.split('\n')
+ for line in splitLines[:-1]:
+ if line.startswith('#'):
+ continue
+ singleRecord = line.split(',')
+ singleRecord[0] = int(IPv6Address(singleRecord[0]))
+ singleRecord[1] = int(IPv6Address(singleRecord[1]))
+ parsedTable.append(singleRecord)
+
+ return parsedTable
+
+
try:
# Make sure we have the database before trying to import the module:
- if not (isfile(GEOIP_DBFILE) and isfile(GEOIPv6_DBFILE)): # pragma: no cover
+ if not (isfile(GEOIP_DBFILE) and isfile(GEOIPv6_DBFILE)):
+ # pragma: no cover
raise EnvironmentError("Could not find %r. On Debian-based systems, "
- "please install the geoip-database package."
+ "please install the tor-geoipdb package."
% GEOIP_DBFILE)
- import pygeoip
- geoip = pygeoip.GeoIP(GEOIP_DBFILE, flags=pygeoip.MEMORY_CACHE)
- geoipv6 = pygeoip.GeoIP(GEOIPv6_DBFILE, flags=pygeoip.MEMORY_CACHE)
+ geoip = loadFromGeoIPDB(GEOIP_DBFILE)
+ geoipv6 = loadFromGeoIPDB6(GEOIPv6_DBFILE)
logging.info("GeoIP databases loaded")
except Exception as err: # pragma: no cover
- logging.warn("Error while loading geoip module: %r" % err)
+ logging.warn("Error while loading data from GeoIP Database: %r" % err)
geoip = None
geoipv6 = None
+def countryCodeByAddress(table, addr):
+ """Lookup Country Code in Geo IP tables.
+
+ :param list table: Contains list of IP Address ranges and mapped countries
+ :type addr: :class:`ipaddr.IPAddress`
+ :param addr: An IPv4 OR IPv6 address.
+ :rtype: ``None`` or str
+
+ :returns: If the GeoIP databases are loaded, and the **ip** lookup is
+ successful, then this returns a two-letter country code. Otherwise,
+ this returns ``None``.
+ """
+ if addr.is_loopback:
+ return None
+
+ addrLong = int(addr)
+
+ # First find the range_end which is greater than input.
+ # Then if input lies in that range we return the country.
+ for item in table:
+ if item[1] >= addrLong:
+ if item[0] <= addrLong:
+ return item[2]
+ else:
+ return None
+
+ return None
+
+
def getCountryCode(ip):
"""Return the two-letter country code of a given IP address.
@@ -64,7 +142,7 @@ def getCountryCode(ip):
db = None
# First, make sure we loaded GeoIP properly.
if None in (geoip, geoipv6):
- logging.warn("GeoIP databases aren't loaded; can't look up country code")
+ logging.warn("GeoIP databases not loaded; can't look up country code")
return None
else:
if version == 4:
@@ -73,7 +151,7 @@ def getCountryCode(ip):
db = geoipv6
# Look up the country code of the address.
- countryCode = db.country_code_by_addr(addr)
+ countryCode = countryCodeByAddress(db, ip)
if countryCode:
logging.debug("Looked up country code: %s" % countryCode)
return countryCode
diff --git a/scripts/install-dependencies b/scripts/install-dependencies
index 4b5e435..f4a7ab0 100755
--- a/scripts/install-dependencies
+++ b/scripts/install-dependencies
@@ -9,7 +9,7 @@ PIP=$(which pip)
PIP_FLAGS='--no-binary :all'
DEPENDS="build-essential openssl sqlite3 python3-dev python3-setuptools"
-DEPENDS="${DEPENDS} libgeoip-dev geoip-database libjpeg-dev"
+DEPENDS="${DEPENDS} libgeoip-dev tor-geoipdb libjpeg-dev"
HERE=$(dirname $0)
if [ "$EUID" != "0" ] ; then SUDO=$(which sudo); fi
More information about the tor-commits
mailing list