- tor-commits - lists.torproject.org

[translation/vidalia_alpha] Update translations for vidalia_alpha
by translation＠torproject.org 06 Nov '12

06 Nov '12

commit 5fa580f95e16b8f5b5340e8c485b210c0f8d6377 Author: Translation commit bot <translation(a)torproject.org> Date: Tue Nov 6 16:15:22 2012 +0000 Update translations for vidalia_alpha --- es/vidalia_es.po | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/es/vidalia_es.po b/es/vidalia_es.po index ea54535..ee281c1 100644 --- a/es/vidalia_es.po +++ b/es/vidalia_es.po @@ -9,7 +9,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: https://trac.torproject.org/projects/tor\n" "POT-Creation-Date: 2012-03-21 17:46+0000\n" -"PO-Revision-Date: 2012-11-06 14:38+0000\n" +"PO-Revision-Date: 2012-11-06 15:55+0000\n" "Last-Translator: strel <strelnic(a)gmail.com>\n" "Language-Team: Spanish (http://www.transifex.com/projects/p/torproject/language/es/)\n" "MIME-Version: 1.0\n" @@ -166,7 +166,7 @@ msgstr "NOTA: esto será editado mientras torrc se carga" msgctxt "AdvancedPage" msgid "ControlSocket path doesn't exist." -msgstr "La ruta de ControlSocket de no existe." +msgstr "La ruta de ControlSocket no existe." msgctxt "AdvancedPage" msgid ""

1 0

[ooni-probe/master] Add script that verifys proper functionality of basic tests
by art＠torproject.org 06 Nov '12

06 Nov '12

commit 0b9e1629186a081dac4a72046caa8f1244c20154 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Nov 6 16:34:34 2012 +0100 Add script that verifys proper functionality of basic tests * As suggested by Jake * There is a subset of inputs that will run with the tests to avoid having to wait loads of time for the test running. * This should not require more than 30 seconds on a decent network --- before_i_commit.sh | 27 +++++++++ ooni/oonicli.py | 7 +-- ooni/utils/log.py | 14 +++-- oonib/oonibackend.py | 12 +++- oonib/report/api.py | 63 ++++------------------ oonib/report/db/models.py | 83 ----------------------------- test_inputs/README | 48 +++++++++++++++++ test_inputs/dns_tamper_file.txt | 3 + test_inputs/dns_tamper_test_resolvers.txt | 2 + test_inputs/http_host_file.txt | 2 + test_inputs/keyword_filtering_file.txt | 2 + test_inputs/url_lists_file.txt | 2 + 12 files changed, 118 insertions(+), 147 deletions(-) diff --git a/before_i_commit.sh b/before_i_commit.sh new file mode 100755 index 0000000..2816ea5 --- /dev/null +++ b/before_i_commit.sh @@ -0,0 +1,27 @@ +#!/bin/sh +# This script should be run before you commit to verify that the basic tests +# are working as they should +# Once you have run it you can inspect the log file via +# +# $ less before_i_commit.log +# To clean up everything that is left by the running of this tool, do as +# following: +# +# rm *.yamloo; rm before_i_commit.log +# + +find . -type f -name "*.py[co]" -delete + +./bin/ooniprobe -l before_i_commit.log -o url_lists.yamloo nettests/core/url_list.py -f test_inputs/url_lists_file.txt + +./bin/ooniprobe -l before_i_commit.log -o dns_tamper_test.yamloo nettests/core/dnstamper.py -t test_inputs/dns_tamper_test_resolvers.txt -f test_inputs/dns_tamper_file.txt + +./bin/ooniprobe -l before_i_commit.log -o captive_portal_test.yamloo nettests/core/captiveportal.py + +./bin/ooniprobe -l before_i_commit.log -o http_host.yamloo nettests/core/http_host.py -b http://ooni.nu/test -f test_inputs/http_host_file.txt + +./bin/ooniprobe -l before_i_commit.log -o keyword_filtering.yamloo nettests/core/keyword_filtering.py -b http://ooni.nu/test/ -f test_inputs/keyword_filtering_file.txt + +./bin/ooniprobe -l before_i_commit.log -o url_lists.yamloo nettests/core/url_list.py -f test_inputs/url_lists_file.txt + + diff --git a/ooni/oonicli.py b/ooni/oonicli.py index aa3cdc0..1988652 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -46,7 +46,7 @@ class Options(usage.Options, app.ReactorSelectionMixin): optParameters = [ ["reportfile", "o", None, "report file name"], - ["logfile", "l", "test.log", "log file name"], + ["logfile", "l", None, "log file name"], ['temp-directory', None, '_ooni_temp', 'Path to use as working directory for tests.'] ] @@ -87,9 +87,6 @@ class Options(usage.Options, app.ReactorSelectionMixin): def run(): - log.start() - log.debug("Started logging") - if len(sys.argv) == 1: sys.argv.append("--help") config = Options() @@ -101,6 +98,8 @@ def run(): if config['debug-stacktraces']: defer.setDebugging(True) + log.start(config['logfile']) + classes = runner.findTestClassesFromConfig(config) casesList, options = runner.loadTestsAndOptions(classes, config) diff --git a/ooni/utils/log.py b/ooni/utils/log.py index e93295a..771bad8 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -13,18 +13,22 @@ from twisted.python.logfile import DailyLogFile from ooni.utils import otime from ooni import oconfig -# XXX make this a config option -log_file = oconfig.basic.logfile +def start(logfile=None): + daily_logfile = None + + if not logfile: + logfile = oconfig.basic.logfile + + log_folder = os.path.dirname(logfile) + log_filename = os.path.basename(logfile) -def start(log_file=log_file): - log_folder = os.path.join('/', *log_file.split('/')[:-1]) - log_filename = log_file.split('/')[-1] daily_logfile = DailyLogFile(log_filename, log_folder) txlog.msg("Starting OONI on %s (%s UTC)" % (otime.prettyDateNow(), otime.utcPrettyDateNow())) logging.basicConfig() python_logging = txlog.PythonLoggingObserver() + if oconfig.advanced.debug: python_logging.logger.setLevel(logging.DEBUG) else: diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index 29f3f58..e2e800f 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -9,20 +9,26 @@ import json import random import string -from twisted.application import internet, service from twisted.internet import protocol, reactor, defer -from twisted.protocols import basic +from twisted.application import internet, service from twisted.web import resource, server, static from twisted.web.microdom import escape +from twisted.protocols import basic from twisted.names import dns +from ooni.utils import log + from oonib.report.api import reportingBackend -from oonib.lib import config from oonib.lib.ssl import SSLContext +from oonib.lib import config + from oonib.testhelpers.httph import HTTPBackend, DebugHTTPServer from oonib.testhelpers.dns import ProxyDNSServer from oonib.testhelpers.daphn3 import Daphn3Server + +log.start('/tmp/oonib.log') + # This tells twisted to set the server.version = config.main.server_version diff --git a/oonib/report/api.py b/oonib/report/api.py index ad0f62c..ea6085d 100644 --- a/oonib/report/api.py +++ b/oonib/report/api.py @@ -8,46 +8,15 @@ This is the async pcap reporting system. It requires the client to have created """ import random import string +import json + from twisted.internet import reactor, defer + from cyclone import web from oonib.report.db import models backend_version = '0.0.1' -def generateReportID(): - size = 100 - report_id = ''.join(random.choice(string.ascii_letters) for x in range(size)) - return report_id - -(a)defer.inlineCallbacks -def newReport(software_name, software_version, test_name, test_version, - progress, content): - - report_id = generateReportID() - - new_report = models.Report() - - new_report.report_id = unicode(report_id) - - new_report.software_name = unicode(software_name) - new_report.software_version = unicode(software_version) - new_report.test_name = unicode(test_name) - new_report.test_version = unicode(test_version) - new_report.progress = unicode(progress) - new_report.content = unicode(content) - - print "Software Name: %s" % software_name - print "Software Version: %s" % software_version - print "Test Name: %s" % test_name - print "Test Version: %s" % test_version - print "Progress: %s" % progress - print "Content: %s" % content - - yield new_report.save() - - defer.returnValue({'backend_version': backend_version, 'report_id': - report_id}) - def updateReport(report_id, content): print "Report ID: %s" % report_id print "Content: %s" % content @@ -61,7 +30,7 @@ class NewReportHandler(web.RequestHandler): @web.asynchronous @defer.inlineCallbacks - def get(self): + def post(self): """ Creates a new report with the input @@ -84,27 +53,18 @@ class NewReportHandler(web.RequestHandler): {'backend_version': 'XXX', 'report_id': 'XXX'} """ - # This is the list of supported arguments - arguments = ['software_name', 'software_version', - 'test_name','test_version', - 'progress', 'content'] - report = {} - for arg in arguments: - if len(self.get_arguments(arg)) == 0: - raise web.HTTPError(400, "%s not specified as argument of POST" - % arg) - report[arg] = self.get_argument(arg) + parsed_request = json.loads(self.request.body) - try: - test_helper = self.get_argument('test_helper') + # XXX here we should validate and sanitize the request + report_data = parsed_request - except web.HTTPError: - pass + new_report = models.Report() - new_report = yield newReport(**report) + print "Got %s as request" % parsed_request + result = yield new_report.new(report_data) - self.write(new_report) + self.write(result) self.finish() def put(self): @@ -128,4 +88,3 @@ spec = [(r"/report/new", NewReportHandler), (r"/report/pcap", PCAPReportHandler)] reportingBackend = web.Application(spec) - diff --git a/oonib/report/db/models.py b/oonib/report/db/models.py deleted file mode 100644 index 21e60eb..0000000 --- a/oonib/report/db/models.py +++ /dev/null @@ -1,83 +0,0 @@ -__all__ = ['Report', 'TestHelperTMP'] -from storm.twisted.transact import transact -from storm.locals import * - -from oonib.report.db import getStore, transactor - -class OModel(object): - - transactor = transactor - - @transact - def create(query): - store = Store(database) - store.execute(query) - store.commit() - - @transact - def save(self): - store = getStore() - store.add(self) - store.commit() - -class Report(OModel): - """ - This represents an OONI Report as stored in the database. - - report_id: this is generated by the backend and is used by the client to - reference a previous report and append to it. It should be - treated as a shared secret between the probe and backend. - - software_name: this indicates the name of the software performing the test - (this will default to ooniprobe) - - software_version: this is the version number of the software running the - test. - - test_name: the name of the test on which the report is being created. - - test_version: indicates the version of the test - - progress: what is the current progress of the report. This allows clients - to report event partial reports up to a certain percentage of - progress. Once the report is complete progress will be 100. - - content: what is the content of the report. If the current progress is less - than 100 we should append to the YAML data structure that is - currently stored in such field. - """ - __storm_table__ = 'reports' - - createQuery = "CREATE TABLE " + __storm_table__ +\ - "(id INTEGER PRIMARY KEY, report_id VARCHAR, software_name VARCHAR,"\ - "software_version VARCHAR, test_name VARCHAR, test_version VARCHAR,"\ - "progress VARCHAR, content VARCHAR)" - - - id = Int(primary=True) - - report_id = Unicode() - - software_name = Unicode() - software_version = Unicode() - test_name = Unicode() - test_version = Unicode() - progress = Unicode() - - content = Unicode() - -class TestHelperTMP(OModel): - __storm_table__ = 'testhelpertmp' - - createQuery = "CREATE TABLE " + __storm_table__ +\ - "(id INTEGER PRIMARY KEY, report_id VARCHAR, test_helper VARCHAR,"\ - " client_ip VARCHAR, creation_time VARCHAR)" - - id = Int(primary=True) - - report_id = Unicode() - - test_helper = Unicode() - client_ip = Unicode() - - creation_time = Date() diff --git a/test_inputs/README b/test_inputs/README new file mode 100644 index 0000000..1eff781 --- /dev/null +++ b/test_inputs/README @@ -0,0 +1,48 @@ +In here you will find some very simple input lists that are useful for testing +the correct functionality of the various OONIProbe tests. + +# DNS Tamper + +./bin/ooniprobe -o dns_tamper_test.yamloo nettests/core/dnstamper.py -t test_inputs/dns_tamper_test_resolvers.txt -f test_inputs/dns_tamper_file.txt + +less dns_tamper_test.yamloo + +# Captive Portal + +./bin/ooniprobe -o captive_portal_test.yamloo nettests/core/captiveportal.py + +less captive_portal_test.yamloo + +# HTTP Host + +./bin/ooniprobe -o http_host.yamloo nettests/core/http_host.py -b http://ooni.nu/test -f test_inputs/http_host_file.txt + +less http_host.yamloo + +# Keyword filtering + +./bin/ooniprobe -o keyword_filtering.yamloo nettests/core/keyword_filtering.py -b http://ooni.nu/test/ -f test_inputs/keyword_filtering_file.txt + +less keyword_filtering.yamloo + +# URL List + +./bin/ooniprobe -o url_lists.yamloo nettests/core/url_list.py -f test_inputs/url_lists_file.txt + +less url_lists.yamloo + +# Squid transparent proxy + +./bin/ooniprobe -o squid.yamloo nettests/core/squid.py + +less squid.yamloo + +# HTTP Requests + +Not Implemented + +# Traceroute + +Not Implemented + + diff --git a/test_inputs/dns_tamper_file.txt b/test_inputs/dns_tamper_file.txt new file mode 100644 index 0000000..25f365c --- /dev/null +++ b/test_inputs/dns_tamper_file.txt @@ -0,0 +1,3 @@ +torproject.org +google.com +measurementlab.net diff --git a/test_inputs/dns_tamper_test_resolvers.txt b/test_inputs/dns_tamper_test_resolvers.txt new file mode 100644 index 0000000..14c77e0 --- /dev/null +++ b/test_inputs/dns_tamper_test_resolvers.txt @@ -0,0 +1,2 @@ +8.8.8.8 +8.8.4.4 diff --git a/test_inputs/http_host_file.txt b/test_inputs/http_host_file.txt new file mode 100644 index 0000000..12afb18 --- /dev/null +++ b/test_inputs/http_host_file.txt @@ -0,0 +1,2 @@ +torproject.org +ooni.nu diff --git a/test_inputs/keyword_filtering_file.txt b/test_inputs/keyword_filtering_file.txt new file mode 100644 index 0000000..4583bae --- /dev/null +++ b/test_inputs/keyword_filtering_file.txt @@ -0,0 +1,2 @@ +antani +sblinda diff --git a/test_inputs/url_lists_file.txt b/test_inputs/url_lists_file.txt new file mode 100644 index 0000000..16a4f58 --- /dev/null +++ b/test_inputs/url_lists_file.txt @@ -0,0 +1,2 @@ +http://ooni.nu/test +http://torproject.org/

1 0

[metrics-tasks/master] Add simple Python database lookup code (#6471).
by karsten＠torproject.org 06 Nov '12

06 Nov '12

commit 5a202017333cb64f5b9ca7d862e7ef9677761221 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Nov 6 10:19:40 2012 -0500 Add simple Python database lookup code (#6471). --- task-6471/python/.gitignore | 3 + task-6471/python/pygeodate.py | 140 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+), 0 deletions(-) diff --git a/task-6471/python/.gitignore b/task-6471/python/.gitignore new file mode 100644 index 0000000..4d9663e --- /dev/null +++ b/task-6471/python/.gitignore @@ -0,0 +1,3 @@ +*.csv +*.csv.bz2 + diff --git a/task-6471/python/pygeodate.py b/task-6471/python/pygeodate.py new file mode 100644 index 0000000..6480f20 --- /dev/null +++ b/task-6471/python/pygeodate.py @@ -0,0 +1,140 @@ +import bisect +import socket +import struct +import datetime + +class Range: + def __init__(self, line): + parts = line.split(',') + # TODO Extend to IPv6. The cool thing in Python is that ints have + # variable size, so we can use 48 bit keys for IPv4 (32 bit for the + # IPv4 address and 16 bit for the database date) and 144 bit keys for + # IPv6. + self.start_address = Database.address_ston(parts[0]) + self.end_address = Database.address_ston(parts[1]) + self.code = parts[2] + self.start_date = Database.date_ston(parts[3]) + self.end_date = Database.date_ston(parts[4]) + self.key = Database.create_key(self.start_address, self.start_date) + + def __str__(self): + return "%s,%s,%s,%s,%s" % \ + (Database.address_ntos(self.start_address), + Database.address_ntos(self.end_address), + self.code, + Database.date_ntos(self.start_date), + Database.date_ntos(self.end_date)) + +class Database: + def __init__(self): + # TODO Replace with crit-bit tree if performance becomes a problem + self.data = [] + self.dates = [] + self.keys = [] + + @staticmethod + def address_ston(address_string): + try: + address_struct = socket.inet_pton(socket.AF_INET, address_string) + except socket.error: + raise ValueError + return struct.unpack('!I', address_struct)[0] + + @staticmethod + def address_ntos(address): + return socket.inet_ntop(socket.AF_INET, struct.pack('!I', address)) + + @staticmethod + def date_ston(date_string): + date_datetime = datetime.datetime.strptime(date_string, '%Y%m%d') + return int(date_datetime.strftime('%s')) / 86400 + + @staticmethod + def date_ntos(date): + return datetime.datetime.fromtimestamp(date * 86400).strftime('%Y%m%d') + + @staticmethod + def address_kton(key): + return key >> 16 + + @staticmethod + def date_kton(key): + return key & 0xffff + + @staticmethod + def address_ktos(key): + return Database.address_ntos(Database.address_kton(key)) + + @staticmethod + def date_ktos(key): + return Database.date_ntos(Database.date_kton(key)) + + @staticmethod + def create_key(address, date): + return (address << 16) + date + + def load_combined_databases(self, path): + with open(path) as input_file: + for line in input_file.readlines(): + line = line.strip() + if line.startswith('!'): + self.add_date(line) + continue + else: + self.add_range(line) + self.data.sort() + self.keys = [r[0] for r in self.data] + + def add_date(self, line): + date = line.split("!")[1] + if date not in self.dates: + bisect.insort(self.dates, date) + + def add_range(self, line): + r = Range(line) + self.data.append((r.key, r)) + + def lookup_address_and_date(self, address_string, date_string): + if len(self.data) == 0: + return '??' + dates_pos = max(0, bisect.bisect(self.dates, date_string) - 1) + address = Database.address_ston(address_string) + date = Database.date_ston(self.dates[dates_pos]) + key = Database.create_key(address, date) + pos = bisect.bisect(self.keys, key + 1) + # Look up address and date by iterating backwards over possibly + # matching ranges. + while pos: + pos = pos - 1 + r = self.data[pos][1] + # If either the end address or end date of the range we're looking + # at is smaller than the values we're looking for, we can be sure + # not to find it anymore. + if r.end_address < address or r.end_date < date: + return '??' + # If the range starts at a later date, skip it and look at the next + # one. + if r.start_date > date: + continue + # Both address and date ranges match, so return the assigned + # code. + return r.code + # No ranges (left) to look at. We don't have what we were looking + # for. */ + return '??'; + +if __name__ == "__main__": + db = Database() + db.load_combined_databases('geoip-2007-10-2012-09.csv') + with open('test-cases-2007-10-2012-09.csv') as input_file: + for line in input_file.readlines(): + line = line.strip() + parts = line.split(',') + address_string = parts[0] + date_string = parts[1] + expected = parts[2] + result = db.lookup_address_and_date(address_string, date_string) + if (expected != result): + print "! %s -> %s" % (line, result) + break +

1 0

[metrics-tasks/master] Take database dates from directory names (#6471).
by karsten＠torproject.org 06 Nov '12

06 Nov '12

commit d200e5911e850748c87c5f160519b9c61b95adbd Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Nov 5 23:10:53 2012 -0500 Take database dates from directory names (#6471). Last modified times of .csv files do not always match publication dates. --- .../org/torproject/task6471/ConvertExample.java | 4 +- .../org/torproject/task6471/DatabaseImporter.java | 8 +++--- .../torproject/task6471/DatabaseImporterImpl.java | 21 ++++++++++++++----- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/task-6471/java/src/org/torproject/task6471/ConvertExample.java b/task-6471/java/src/org/torproject/task6471/ConvertExample.java index c96047c..c4ef4c6 100644 --- a/task-6471/java/src/org/torproject/task6471/ConvertExample.java +++ b/task-6471/java/src/org/torproject/task6471/ConvertExample.java @@ -11,7 +11,7 @@ public class ConvertExample { System.out.print("Saving combined ASN database to disk... "); startMillis = endMillis; - combinedDatabase.saveCombinedDatabases("asn-2012-07-2012-10.csv"); + combinedDatabase.saveCombinedDatabases("asn-2005-09-2012-11.csv"); endMillis = System.currentTimeMillis(); System.out.println((endMillis - startMillis) + " millis."); startMillis = endMillis; @@ -25,7 +25,7 @@ public class ConvertExample { System.out.print("Saving combined city database to disk... "); startMillis = endMillis; - combinedDatabase.saveCombinedDatabases("city-2012-07-2012-10.csv"); + combinedDatabase.saveCombinedDatabases("city-2009-06-2012-10.csv"); endMillis = System.currentTimeMillis(); System.out.println((endMillis - startMillis) + " millis."); startMillis = endMillis; diff --git a/task-6471/java/src/org/torproject/task6471/DatabaseImporter.java b/task-6471/java/src/org/torproject/task6471/DatabaseImporter.java index 330ecea..0d4ac92 100644 --- a/task-6471/java/src/org/torproject/task6471/DatabaseImporter.java +++ b/task-6471/java/src/org/torproject/task6471/DatabaseImporter.java @@ -32,8 +32,8 @@ public interface DatabaseImporter extends Database { * address ranges and block numbers, and GeoLiteCity-Location.csv * contains country codes for block numbers, among other things. Only * the range start and end addresses and the country code are imported. - * The database date is taken from the file modification time of the - * GeoLiteCity-Blocks.csv file. + * The database date is taken from the directory name containing blocks + * and location file. * * A typical entry from the GeoLiteCity-Blocks.csv file is: * ""3758093312","3758094335","108612"" @@ -46,8 +46,8 @@ public interface DatabaseImporter extends Database { /** * Import the contents of one or more Maxmind GeoIPASNum2.csv databases. * Only the range start and end addresses and the AS number are - * imported. The database date is taken from the file modification - * time. + * imported. The database date is taken from the directory name which + * is expected to be yyyy-mm/, e.g., 2012-11/GeoIPASNum2.csv. * * A typical entry from such a database file is: * "3758063616,3758079999,"AS9381 Wharf T&T Ltd."" diff --git a/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java b/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java index 407d8ca..4d15827 100644 --- a/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java +++ b/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java @@ -170,10 +170,12 @@ public class DatabaseImporterImpl extends DatabaseImpl boolean importGeoLiteCityBlocksAndLocationFiles(File blocksFile, File locationFile) { - long lastModifiedMillis = blocksFile.lastModified(); String databaseFileName = blocksFile.getName() + "+" + locationFile.getName(); - int databaseDate = (int) (lastModifiedMillis / 86400000); + String databaseDateString = + blocksFile.getParentFile().getName().substring( + "GeoLiteCity_".length()); + int databaseDate = convertDateStringToNumber(databaseDateString); this.addDatabase(databaseFileName, databaseDate); try { /* Parse location file first and remember country codes for given @@ -262,17 +264,24 @@ public class DatabaseImporterImpl extends DatabaseImpl } private boolean importGeoIPASNum2File(File file) { - long lastModifiedMillis = file.lastModified(); String databaseFileName = file.getName(); - int databaseDate = (int) (lastModifiedMillis / 86400000); + String databaseDateString = + file.getParentFile().getName().replaceAll("-", "") + "01"; + int databaseDate = convertDateStringToNumber(databaseDateString); this.addDatabase(databaseFileName, databaseDate); try { BufferedReader br = new BufferedReader(new FileReader(file)); String line; while ((line = br.readLine()) != null) { String[] parts = line.split(","); - long startAddress = Long.parseLong(parts[0]), - endAddress = Long.parseLong(parts[1]); + try { + Long.parseLong(parts[0].trim()); + Long.parseLong(parts[1].trim()); + } catch (NumberFormatException e) { + System.err.println(file.getAbsolutePath() + " '" + line + "'"); + } + long startAddress = Long.parseLong(parts[0].trim()), + endAddress = Long.parseLong(parts[1].trim()); String code = parts[2].split(" ")[0].replaceAll("\"", ""); if (!code.startsWith("AS")) { /* Don't import illegal range. */

1 0

[metrics-tasks/master] Support parsing of Maxmind's formats (#6471).
by karsten＠torproject.org 06 Nov '12

06 Nov '12

commit 2d1bd61d767f07f430106b35f67068ccb873c252 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sun Nov 4 14:55:50 2012 -0500 Support parsing of Maxmind's formats (#6471). --- task-6471/java/build.xml | 7 + .../org/torproject/task6471/ConvertExample.java | 34 ++++ .../org/torproject/task6471/DatabaseImporter.java | 37 ++++- .../torproject/task6471/DatabaseImporterImpl.java | 193 ++++++++++++++++++-- .../src/org/torproject/task6471/DatabaseTest.java | 48 ++++-- 5 files changed, 289 insertions(+), 30 deletions(-) diff --git a/task-6471/java/build.xml b/task-6471/java/build.xml index bda8d25..40969ab 100644 --- a/task-6471/java/build.xml +++ b/task-6471/java/build.xml @@ -45,5 +45,12 @@ <classpath refid="classpath"/> </java> </target> + <target name="convert" depends="compile"> + <java fork="true" + maxmemory="2048m" + classname="org.torproject.task6471.ConvertExample"> + <classpath refid="classpath"/> + </java> + </target> </project> diff --git a/task-6471/java/src/org/torproject/task6471/ConvertExample.java b/task-6471/java/src/org/torproject/task6471/ConvertExample.java new file mode 100644 index 0000000..c96047c --- /dev/null +++ b/task-6471/java/src/org/torproject/task6471/ConvertExample.java @@ -0,0 +1,34 @@ +package org.torproject.task6471; + +public class ConvertExample { + public static void main(String[] args) { + System.out.print("Importing ASN database files... "); + long startMillis = System.currentTimeMillis(); + DatabaseImporter combinedDatabase = new DatabaseImporterImpl(); + combinedDatabase.importGeoIPASNum2FileOrDirectory("../asn"); + long endMillis = System.currentTimeMillis(); + System.out.println((endMillis - startMillis) + " millis."); + + System.out.print("Saving combined ASN database to disk... "); + startMillis = endMillis; + combinedDatabase.saveCombinedDatabases("asn-2012-07-2012-10.csv"); + endMillis = System.currentTimeMillis(); + System.out.println((endMillis - startMillis) + " millis."); + startMillis = endMillis; + + System.out.print("Importing city database files... "); + startMillis = System.currentTimeMillis(); + combinedDatabase = new DatabaseImporterImpl(); + combinedDatabase.importGeoLiteCityFileOrDirectory("../city"); + endMillis = System.currentTimeMillis(); + System.out.println((endMillis - startMillis) + " millis."); + + System.out.print("Saving combined city database to disk... "); + startMillis = endMillis; + combinedDatabase.saveCombinedDatabases("city-2012-07-2012-10.csv"); + endMillis = System.currentTimeMillis(); + System.out.println((endMillis - startMillis) + " millis."); + startMillis = endMillis; + + } +} diff --git a/task-6471/java/src/org/torproject/task6471/DatabaseImporter.java b/task-6471/java/src/org/torproject/task6471/DatabaseImporter.java index 641def4..330ecea 100644 --- a/task-6471/java/src/org/torproject/task6471/DatabaseImporter.java +++ b/task-6471/java/src/org/torproject/task6471/DatabaseImporter.java @@ -8,8 +8,7 @@ public interface DatabaseImporter extends Database { * are expected to conform to the RIR Statistics Exchange Format. * Only IPv4 address ranges are imported, whereas ASN and IPv6 lines are * ignored. Only the country code, start address, and address range - * length fields are imported. (TODO Extend to IPv6 and find similar - * data source for ASN.) + * length fields are imported. * * A typical entry from a RIR file is: * "ripencc|FR|ipv4|2.0.0.0|1048576|20100712|allocated". @@ -28,6 +27,40 @@ public interface DatabaseImporter extends Database { public boolean importRegionalRegistryStatsFileOrDirectory(String path); /** + * Import the contents of one or more Maxmind GeoLiteCity databases, + * each of them consisting of two files: GeoLiteCity-Blocks.csv contains + * address ranges and block numbers, and GeoLiteCity-Location.csv + * contains country codes for block numbers, among other things. Only + * the range start and end addresses and the country code are imported. + * The database date is taken from the file modification time of the + * GeoLiteCity-Blocks.csv file. + * + * A typical entry from the GeoLiteCity-Blocks.csv file is: + * ""3758093312","3758094335","108612"" + * + * A typical entry from the GeoLiteCity-Location.csv file is: + * "108612,"IN","09","Rajkot","",22.3000,70.7833,," + */ + public boolean importGeoLiteCityFileOrDirectory(String path); + + /** + * Import the contents of one or more Maxmind GeoIPASNum2.csv databases. + * Only the range start and end addresses and the AS number are + * imported. The database date is taken from the file modification + * time. + * + * A typical entry from such a database file is: + * "3758063616,3758079999,"AS9381 Wharf T&T Ltd."" + * + * @param path Path to a stats file or directory. + * @return True if importing the file or directory was successful, + * false otherwise. + */ + public boolean importGeoIPASNum2FileOrDirectory(String path); + + /* TODO Extend all imported formats to IPv6 equivalents. */ + + /** * Save the combined databases in a format that can later be loaded much * more efficiently than importing the original RIR files again. * diff --git a/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java b/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java index 4795883..2a6c203 100644 --- a/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java +++ b/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java @@ -6,13 +6,18 @@ import java.io.File; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.SortedMap; import java.util.Stack; +import java.util.TimeZone; import java.util.TreeMap; public class DatabaseImporterImpl extends DatabaseImpl @@ -74,7 +79,7 @@ public class DatabaseImporterImpl extends DatabaseImpl String databaseFileName = file.getName(); while ((line = br.readLine()) != null) { if (line.startsWith("#") || line.length() == 0) { - /* Skip comment line. */ + /* Skip comment or empty line. */ continue; } String[] parts = line.split("\\|"); @@ -101,8 +106,8 @@ public class DatabaseImporterImpl extends DatabaseImpl } String startAddressString = parts[3]; long addresses = Long.parseLong(parts[4]); - this.addRange(databaseFileName, code, startAddressString, - addresses); + this.addRegionalRegistryStatsFileRange(databaseFileName, code, + startAddressString, addresses); } br.close(); this.repairTree(); @@ -112,6 +117,174 @@ public class DatabaseImporterImpl extends DatabaseImpl return true; } + void addRegionalRegistryStatsFileRange(String databaseFileName, + String code, String startAddressString, long addresses) { + String databaseDateString = + databaseFileName.substring(databaseFileName.length() - 8); + int databaseDate = convertDateStringToNumber(databaseDateString); + long startAddress = convertAddressStringToNumber(startAddressString); + long endAddress = startAddress + addresses - 1L; + this.addRange(databaseFileName, databaseDate, startAddress, + endAddress, code); + } + + public boolean importGeoLiteCityFileOrDirectory(String path) { + boolean allImportsSuccessful = true; + Stack<File> stackedFiles = new Stack<File>(); + stackedFiles.add(new File(path)); + SortedMap<File, Set<File>> filesByDirectory = + new TreeMap<File, Set<File>>(); + while (!stackedFiles.isEmpty()) { + File file = stackedFiles.pop(); + if (file.isDirectory()) { + stackedFiles.addAll(Arrays.asList(file.listFiles())); + } else if (!file.getName().endsWith(".csv")) { + System.err.println("Parsing other files than .csv is not " + + "supported: '" + file.getAbsolutePath() + "'. Skipping."); + } else { + if (!filesByDirectory.containsKey(file.getParentFile())) { + filesByDirectory.put(file.getParentFile(), new HashSet<File>()); + } + filesByDirectory.get(file.getParentFile()).add(file); + } + } + for (Set<File> files : filesByDirectory.values()) { + File blocksFile = null, locationFile = null; + for (File file : files) { + if (file.getName().equals("GeoLiteCity-Blocks.csv")) { + blocksFile = file; + } else if (file.getName().equals("GeoLiteCity-Location.csv")) { + locationFile = file; + } + } + if (blocksFile != null && locationFile != null) { + if (!this.importGeoLiteCityBlocksAndLocationFiles(blocksFile, + locationFile)) { + allImportsSuccessful = false; + } + files.remove(blocksFile); + files.remove(locationFile); + if (!files.isEmpty()) { + System.err.println("Did not recognize the following files, or " + + "did not find the blocks/location equivalent:"); + for (File file : files) { + System.err.println(file.getAbsolutePath()); + } + } + } + } + return allImportsSuccessful; + } + + boolean importGeoLiteCityBlocksAndLocationFiles(File blocksFile, + File locationFile) { + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + long lastModifiedMillis = blocksFile.lastModified(); + String databaseFileName = blocksFile.getName() + " " + + locationFile.getName() + " " + + dateFormat.format(lastModifiedMillis); + int databaseDate = (int) (lastModifiedMillis / 86400000); + try { + /* Parse location file first and remember country codes for given + * locations. */ + Map<Integer, String> locations = new HashMap<Integer, String>(); + BufferedReader br = new BufferedReader(new FileReader( + locationFile)); + String line; + while ((line = br.readLine()) != null) { + if (line.startsWith("Copyright") || line.startsWith("locId")) { + /* Skip copyright notice and column headers. */ + continue; + } + String[] parts = line.split(","); + int location = Integer.parseInt(parts[0]); + String code = parts[1].replaceAll("\"", "").toLowerCase(); + locations.put(location, code); + } + br.close(); + /* Parse blocks file and add ranges to the database. */ + br = new BufferedReader(new FileReader(blocksFile)); + while ((line = br.readLine()) != null) { + if (!line.startsWith("\"")) { + /* Skip copyright notice and column headers. */ + continue; + } + String[] parts = line.replaceAll("\"", "").split(","); + long startAddress = Long.parseLong(parts[0]), + endAddress = Long.parseLong(parts[1]); + int location = Integer.parseInt(parts[2]); + if (!locations.containsKey(location)) { + System.err.println(blocksFile.getAbsolutePath() + " contains " + + "line '" + line + "' that doesn't match any line in " + + locationFile.getAbsolutePath() + ". Aborting."); + break; + } + String code = locations.get(location); + this.addRange(databaseFileName, databaseDate, startAddress, + endAddress, code); + } + br.close(); + } catch (IOException e) { + return false; + } + return true; + } + + public boolean importGeoIPASNum2FileOrDirectory(String path) { + boolean allImportsSuccessful = true; + Stack<File> stackedFiles = new Stack<File>(); + stackedFiles.add(new File(path)); + List<File> allFiles = new ArrayList<File>(); + while (!stackedFiles.isEmpty()) { + File file = stackedFiles.pop(); + if (file.isDirectory()) { + stackedFiles.addAll(Arrays.asList(file.listFiles())); + } else if (!file.getName().endsWith(".csv")) { + System.err.println("Parsing other files than .csv is not " + + "supported: '" + file.getAbsolutePath() + "'. Skipping."); + } else { + allFiles.add(file); + } + } + Collections.sort(allFiles, Collections.reverseOrder()); + for (File file : allFiles) { + if (!this.importGeoIPASNum2File(file)) { + allImportsSuccessful = false; + } + } + return allImportsSuccessful; + } + + private boolean importGeoIPASNum2File(File file) { + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + long lastModifiedMillis = file.lastModified(); + String databaseFileName = file.getName() + " " + + dateFormat.format(lastModifiedMillis); + int databaseDate = (int) (lastModifiedMillis / 86400000); + try { + BufferedReader br = new BufferedReader(new FileReader(file)); + String line; + while ((line = br.readLine()) != null) { + String[] parts = line.split(","); + long startAddress = Long.parseLong(parts[0]), + endAddress = Long.parseLong(parts[1]); + String code = parts[2].split(" ")[0].replaceAll("\"", ""); + if (!code.startsWith("AS")) { + /* Don't import illegal range. */ + continue; + } + this.addRange(databaseFileName, databaseDate, startAddress, + endAddress, code); + } + br.close(); + this.repairTree(); + } catch (IOException e) { + return false; + } + return true; + } /** * Internal counters for import statistics. @@ -127,15 +300,9 @@ public class DatabaseImporterImpl extends DatabaseImpl * is called prior to any lookupAddress() calls. No further checks are * performed that the tree is repaired before looking up an address. */ - void addRange(String databaseFileName, String code, - String startAddressString, long addresses) { - + void addRange(String databaseFileName, int databaseDate, + long startAddress, long endAddress, String code) { this.rangeImports++; - String databaseDateString = - databaseFileName.substring(databaseFileName.length() - 8); - int databaseDate = convertDateStringToNumber(databaseDateString); - long startAddress = convertAddressStringToNumber(startAddressString); - long endAddress = startAddress + addresses - 1L; /* Add new database date and file name if we didn't know them yet, * and note that we need to repair the tree after importing. */ @@ -337,6 +504,9 @@ public class DatabaseImporterImpl extends DatabaseImpl * interface, because the caller needs to make sure that repairTree() * is called prior to any lookupAddress() calls. No further checks are * performed that the tree is repaired before look up an address. + * + * TODO While repairing the tree, we might also optimize it by merging + * adjacent address ranges with the same database date ranges. */ void repairTree() { if (this.addedDatabaseDate < 0) { @@ -429,7 +599,6 @@ public class DatabaseImporterImpl extends DatabaseImpl } return true; } - /* Return a nicely formatted string summarizing database contents and * usage statistics. */ diff --git a/task-6471/java/src/org/torproject/task6471/DatabaseTest.java b/task-6471/java/src/org/torproject/task6471/DatabaseTest.java index fb0f19d..236810b 100644 --- a/task-6471/java/src/org/torproject/task6471/DatabaseTest.java +++ b/task-6471/java/src/org/torproject/task6471/DatabaseTest.java @@ -12,7 +12,8 @@ public class DatabaseTest { @Test() public void testSingleIpRangeSingleDatebase() { DatabaseImporterImpl database = new DatabaseImporterImpl(); - database.addRange("20120901", "us", "3.0.0.0", 16777216); + database.addRegionalRegistryStatsFileRange("20120901", "us", + "3.0.0.0", 16777216); database.repairTree(); assertEquals(1, database.getNumberOfElements()); assertEquals(null, database.lookupIpv4AddressAndDate( @@ -60,8 +61,10 @@ public class DatabaseTest { @Test() public void testTwoAdjacentIpRangesSingleDatabase() { DatabaseImporterImpl database = new DatabaseImporterImpl(); - database.addRange("20120901", "us", "3.0.0.0", 16777216); - database.addRange("20120901", "ca", "4.0.0.0", 16777216); + database.addRegionalRegistryStatsFileRange("20120901", "us", + "3.0.0.0", 16777216); + database.addRegionalRegistryStatsFileRange("20120901", "ca", + "4.0.0.0", 16777216); database.repairTree(); assertEquals(2, database.getNumberOfElements()); assertEquals(null, database.lookupIpv4AddressAndDate( @@ -85,8 +88,10 @@ public class DatabaseTest { @Test() public void testTwoNonAdjacentIpDateRangesSingleDatabase() { DatabaseImporterImpl database = new DatabaseImporterImpl(); - database.addRange("20120901", "us", "3.0.0.0", 16777216); - database.addRange("20120901", "ca", "6.0.0.0", 16777216); + database.addRegionalRegistryStatsFileRange("20120901", "us", + "3.0.0.0", 16777216); + database.addRegionalRegistryStatsFileRange("20120901", "ca", + "6.0.0.0", 16777216); database.repairTree(); assertEquals(2, database.getNumberOfElements()); assertEquals(null, database.lookupIpv4AddressAndDate( @@ -104,8 +109,10 @@ public class DatabaseTest { @Test() public void testDuplicateImport() { DatabaseImporterImpl database = new DatabaseImporterImpl(); - database.addRange("20120901", "us", "3.0.0.0", 16777216); - database.addRange("20120901", "us", "3.0.0.0", 16777216); + database.addRegionalRegistryStatsFileRange("20120901", "us", + "3.0.0.0", 16777216); + database.addRegionalRegistryStatsFileRange("20120901", "us", + "3.0.0.0", 16777216); database.repairTree(); assertEquals(1, database.getNumberOfElements()); assertEquals(null, database.lookupIpv4AddressAndDate( @@ -119,8 +126,10 @@ public class DatabaseTest { @Test() public void testDuplicateImportDifferentCode() { DatabaseImporterImpl database = new DatabaseImporterImpl(); - database.addRange("20120901", "us", "3.0.0.0", 16777216); - database.addRange("20120901", "ca", "3.0.0.0", 16777216); + database.addRegionalRegistryStatsFileRange("20120901", "us", + "3.0.0.0", 16777216); + database.addRegionalRegistryStatsFileRange("20120901", "ca", + "3.0.0.0", 16777216); database.repairTree(); assertEquals(1, database.getNumberOfElements()); assertEquals("us", database.lookupIpv4AddressAndDate( @@ -130,9 +139,11 @@ public class DatabaseTest { @Test() public void testLeaveIpChangeUnchanged() { DatabaseImporterImpl database = new DatabaseImporterImpl(); - database.addRange("20120901", "us", "3.0.0.0", 16777216); + database.addRegionalRegistryStatsFileRange("20120901", "us", + "3.0.0.0", 16777216); database.repairTree(); - database.addRange("20121001", "us", "3.0.0.0", 16777216); + database.addRegionalRegistryStatsFileRange("20121001", "us", + "3.0.0.0", 16777216); database.repairTree(); assertEquals(1, database.getNumberOfElements()); assertEquals("us", database.lookupIpv4AddressAndDate( @@ -148,9 +159,11 @@ public class DatabaseTest { @Test() public void testLeaveIpChangeUnchangedReverseOrder() { DatabaseImporterImpl database = new DatabaseImporterImpl(); - database.addRange("20121001", "us", "3.0.0.0", 16777216); + database.addRegionalRegistryStatsFileRange("20121001", "us", + "3.0.0.0", 16777216); database.repairTree(); - database.addRange("20120901", "us", "3.0.0.0", 16777216); + database.addRegionalRegistryStatsFileRange("20120901", "us", + "3.0.0.0", 16777216); database.repairTree(); assertEquals(1, database.getNumberOfElements()); assertEquals("us", database.lookupIpv4AddressAndDate( @@ -166,11 +179,14 @@ public class DatabaseTest { @Test() public void testMissingIpRange() { DatabaseImporterImpl database = new DatabaseImporterImpl(); - database.addRange("20120901", "us", "3.0.0.0", 16777216); + database.addRegionalRegistryStatsFileRange("20120901", "us", + "3.0.0.0", 16777216); database.repairTree(); - database.addRange("20121101", "us", "3.0.0.0", 16777216); + database.addRegionalRegistryStatsFileRange("20121101", "us", + "3.0.0.0", 16777216); database.repairTree(); - database.addRange("20121001", "us", "6.0.0.0", 16777216); + database.addRegionalRegistryStatsFileRange("20121001", "us", + "6.0.0.0", 16777216); database.repairTree(); assertEquals(3, database.getNumberOfElements()); assertEquals("us", database.lookupIpv4AddressAndDate(

1 0

[metrics-tasks/master] Generalize database to AS numbers (#6471).
by karsten＠torproject.org 06 Nov '12

06 Nov '12

commit 5ffcbdc3713b4e3e698b7fee147e5301d242ef80 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sat Nov 3 21:39:13 2012 -0400 Generalize database to AS numbers (#6471). --- .../src/org/torproject/task6471/Countries.java | 284 -------------------- .../java/src/org/torproject/task6471/Database.java | 49 +--- .../src/org/torproject/task6471/DatabaseImpl.java | 57 +--- .../torproject/task6471/DatabaseImporterImpl.java | 59 ++--- .../task6471/DatabasePerformanceExample.java | 17 +- .../src/org/torproject/task6471/DatabaseTest.java | 103 ++++---- 6 files changed, 114 insertions(+), 455 deletions(-) diff --git a/task-6471/java/src/org/torproject/task6471/Countries.java b/task-6471/java/src/org/torproject/task6471/Countries.java deleted file mode 100644 index a70c4bf..0000000 --- a/task-6471/java/src/org/torproject/task6471/Countries.java +++ /dev/null @@ -1,284 +0,0 @@ -/* Copyright 2011, 2012 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.task6471; - -import java.util.ArrayList; -import java.util.List; - -public class Countries { - - private static Countries instance = new Countries(); - - public static Countries getInstance() { - return Countries.instance; - } - - /* List of arrays of length 2, containing country codes at [0] and - * country names at [1], alphabetically ordered by country names. */ - private List<String[]> knownCountries; - - private Countries() { - this.knownCountries = new ArrayList<String[]>(); - this.knownCountries.add("af;Afghanistan".split(";")); - this.knownCountries.add("ax;Aland Islands".split(";")); - this.knownCountries.add("al;Albania".split(";")); - this.knownCountries.add("dz;Algeria".split(";")); - this.knownCountries.add("as;American Samoa".split(";")); - this.knownCountries.add("ad;Andorra".split(";")); - this.knownCountries.add("ao;Angola".split(";")); - this.knownCountries.add("ai;Anguilla".split(";")); - this.knownCountries.add("aq;Antarctica".split(";")); - this.knownCountries.add("ag;Antigua and Barbuda".split(";")); - this.knownCountries.add("ar;Argentina".split(";")); - this.knownCountries.add("am;Armenia".split(";")); - this.knownCountries.add("aw;Aruba".split(";")); - this.knownCountries.add("au;Australia".split(";")); - this.knownCountries.add("at;Austria".split(";")); - this.knownCountries.add("az;Azerbaijan".split(";")); - this.knownCountries.add("bs;Bahamas".split(";")); - this.knownCountries.add("bh;Bahrain".split(";")); - this.knownCountries.add("bd;Bangladesh".split(";")); - this.knownCountries.add("bb;Barbados".split(";")); - this.knownCountries.add("by;Belarus".split(";")); - this.knownCountries.add("be;Belgium".split(";")); - this.knownCountries.add("bz;Belize".split(";")); - this.knownCountries.add("bj;Benin".split(";")); - this.knownCountries.add("bm;Bermuda".split(";")); - this.knownCountries.add("bt;Bhutan".split(";")); - this.knownCountries.add("bo;Bolivia".split(";")); - this.knownCountries.add("ba;Bosnia and Herzegovina".split(";")); - this.knownCountries.add("bw;Botswana".split(";")); - this.knownCountries.add("bv;Bouvet Island".split(";")); - this.knownCountries.add("br;Brazil".split(";")); - this.knownCountries.add("io;British Indian Ocean Territory". - split(";")); - this.knownCountries.add("bn;Brunei".split(";")); - this.knownCountries.add("bg;Bulgaria".split(";")); - this.knownCountries.add("bf;Burkina Faso".split(";")); - this.knownCountries.add("mm;Burma".split(";")); - this.knownCountries.add("bi;Burundi".split(";")); - this.knownCountries.add("kh;Cambodia".split(";")); - this.knownCountries.add("cm;Cameroon".split(";")); - this.knownCountries.add("ca;Canada".split(";")); - this.knownCountries.add("cv;Cape Verde".split(";")); - this.knownCountries.add("ky;Cayman Islands".split(";")); - this.knownCountries.add("cf;Central African Republic".split(";")); - this.knownCountries.add("td;Chad".split(";")); - this.knownCountries.add("cl;Chile".split(";")); - this.knownCountries.add("cn;China".split(";")); - this.knownCountries.add("cx;Christmas Island".split(";")); - this.knownCountries.add("cc;Cocos (Keeling) Islands".split(";")); - this.knownCountries.add("co;Colombia".split(";")); - this.knownCountries.add("km;Comoros".split(";")); - this.knownCountries.add("cd;Congo, The Democratic Republic of the". - split(";")); - this.knownCountries.add("cg;Congo".split(";")); - this.knownCountries.add("ck;Cook Islands".split(";")); - this.knownCountries.add("cr;Costa Rica".split(";")); - this.knownCountries.add("ci:Côte d'Ivoire".split(":")); - this.knownCountries.add("hr;Croatia".split(";")); - this.knownCountries.add("cu;Cuba".split(";")); - this.knownCountries.add("cy;Cyprus".split(";")); - this.knownCountries.add("cz;Czech Republic".split(";")); - this.knownCountries.add("dk;Denmark".split(";")); - this.knownCountries.add("dj;Djibouti".split(";")); - this.knownCountries.add("dm;Dominica".split(";")); - this.knownCountries.add("do;Dominican Republic".split(";")); - this.knownCountries.add("ec;Ecuador".split(";")); - this.knownCountries.add("eg;Egypt".split(";")); - this.knownCountries.add("sv;El Salvador".split(";")); - this.knownCountries.add("gq;Equatorial Guinea".split(";")); - this.knownCountries.add("er;Eritrea".split(";")); - this.knownCountries.add("ee;Estonia".split(";")); - this.knownCountries.add("et;Ethiopia".split(";")); - this.knownCountries.add("fk;Falkland Islands (Malvinas)".split(";")); - this.knownCountries.add("fo;Faroe Islands".split(";")); - this.knownCountries.add("fj;Fiji".split(";")); - this.knownCountries.add("fi;Finland".split(";")); - this.knownCountries.add("fx;France, Metropolitan".split(";")); - this.knownCountries.add("fr;France".split(";")); - this.knownCountries.add("gf;French Guiana".split(";")); - this.knownCountries.add("pf;French Polynesia".split(";")); - this.knownCountries.add("tf;French Southern Territories".split(";")); - this.knownCountries.add("ga;Gabon".split(";")); - this.knownCountries.add("gm;Gambia".split(";")); - this.knownCountries.add("ge;Georgia".split(";")); - this.knownCountries.add("de;Germany".split(";")); - this.knownCountries.add("gh;Ghana".split(";")); - this.knownCountries.add("gi;Gibraltar".split(";")); - this.knownCountries.add("gr;Greece".split(";")); - this.knownCountries.add("gl;Greenland".split(";")); - this.knownCountries.add("gd;Grenada".split(";")); - this.knownCountries.add("gp;Guadeloupe".split(";")); - this.knownCountries.add("gu;Guam".split(";")); - this.knownCountries.add("gt;Guatemala".split(";")); - this.knownCountries.add("gg;Guernsey".split(";")); - this.knownCountries.add("gn;Guinea".split(";")); - this.knownCountries.add("gw;Guinea-Bissau".split(";")); - this.knownCountries.add("gy;Guyana".split(";")); - this.knownCountries.add("ht;Haiti".split(";")); - this.knownCountries.add("hm;Heard Island and McDonald Islands". - split(";")); - this.knownCountries.add("va;Vatican City".split(";")); - this.knownCountries.add("hn;Honduras".split(";")); - this.knownCountries.add("hk;Hong Kong".split(";")); - this.knownCountries.add("hu;Hungary".split(";")); - this.knownCountries.add("is;Iceland".split(";")); - this.knownCountries.add("in;India".split(";")); - this.knownCountries.add("id;Indonesia".split(";")); - this.knownCountries.add("ir;Iran".split(";")); - this.knownCountries.add("iq;Iraq".split(";")); - this.knownCountries.add("ie;Ireland".split(";")); - this.knownCountries.add("im;Isle of Man".split(";")); - this.knownCountries.add("il;Israel".split(";")); - this.knownCountries.add("it;Italy".split(";")); - this.knownCountries.add("jm;Jamaica".split(";")); - this.knownCountries.add("jp;Japan".split(";")); - this.knownCountries.add("je;Jersey".split(";")); - this.knownCountries.add("jo;Jordan".split(";")); - this.knownCountries.add("kz;Kazakhstan".split(";")); - this.knownCountries.add("ke;Kenya".split(";")); - this.knownCountries.add("ki;Kiribati".split(";")); - this.knownCountries.add("kp;North Korea".split(";")); - this.knownCountries.add("kr;Korea, Republic of".split(";")); - this.knownCountries.add("kw;Kuwait".split(";")); - this.knownCountries.add("kg;Kyrgyzstan".split(";")); - this.knownCountries.add("la;Laos".split(";")); - this.knownCountries.add("lv;Latvia".split(";")); - this.knownCountries.add("lb;Lebanon".split(";")); - this.knownCountries.add("ls;Lesotho".split(";")); - this.knownCountries.add("lr;Liberia".split(";")); - this.knownCountries.add("ly;Libya".split(";")); - this.knownCountries.add("li;Liechtenstein".split(";")); - this.knownCountries.add("lt;Lithuania".split(";")); - this.knownCountries.add("lu;Luxembourg".split(";")); - this.knownCountries.add("mo;Macau".split(";")); - this.knownCountries.add("mk;Macedonia".split(";")); - this.knownCountries.add("mg;Madagascar".split(";")); - this.knownCountries.add("mw;Malawi".split(";")); - this.knownCountries.add("my;Malaysia".split(";")); - this.knownCountries.add("mv;Maldives".split(";")); - this.knownCountries.add("ml;Mali".split(";")); - this.knownCountries.add("mt;Malta".split(";")); - this.knownCountries.add("mh;Marshall Islands".split(";")); - this.knownCountries.add("mq;Martinique".split(";")); - this.knownCountries.add("mr;Mauritania".split(";")); - this.knownCountries.add("mu;Mauritius".split(";")); - this.knownCountries.add("yt;Mayotte".split(";")); - this.knownCountries.add("mx;Mexico".split(";")); - this.knownCountries.add("fm;Micronesia, Federated States of". - split(";")); - this.knownCountries.add("md;Moldova, Republic of".split(";")); - this.knownCountries.add("mc;Monaco".split(";")); - this.knownCountries.add("mn;Mongolia".split(";")); - this.knownCountries.add("me;Montenegro".split(";")); - this.knownCountries.add("ms;Montserrat".split(";")); - this.knownCountries.add("ma;Morocco".split(";")); - this.knownCountries.add("mz;Mozambique".split(";")); - this.knownCountries.add("mm;Burma".split(";")); - this.knownCountries.add("na;Namibia".split(";")); - this.knownCountries.add("nr;Nauru".split(";")); - this.knownCountries.add("np;Nepal".split(";")); - this.knownCountries.add("an;Netherlands Antilles".split(";")); - this.knownCountries.add("nl;Netherlands".split(";")); - this.knownCountries.add("nc;New Caledonia".split(";")); - this.knownCountries.add("nz;New Zealand".split(";")); - this.knownCountries.add("ni;Nicaragua".split(";")); - this.knownCountries.add("ne;Niger".split(";")); - this.knownCountries.add("ng;Nigeria".split(";")); - this.knownCountries.add("nu;Niue".split(";")); - this.knownCountries.add("nf;Norfolk Island".split(";")); - this.knownCountries.add("mp;Northern Mariana Islands".split(";")); - this.knownCountries.add("no;Norway".split(";")); - this.knownCountries.add("om;Oman".split(";")); - this.knownCountries.add("pk;Pakistan".split(";")); - this.knownCountries.add("pw;Palau".split(";")); - this.knownCountries.add("ps;Palestinian Territory".split(";")); - this.knownCountries.add("pa;Panama".split(";")); - this.knownCountries.add("pg;Papua New Guinea".split(";")); - this.knownCountries.add("py;Paraguay".split(";")); - this.knownCountries.add("pe;Peru".split(";")); - this.knownCountries.add("ph;Philippines".split(";")); - this.knownCountries.add("pn;Pitcairn Islands".split(";")); - this.knownCountries.add("pl;Poland".split(";")); - this.knownCountries.add("pt;Portugal".split(";")); - this.knownCountries.add("pr;Puerto Rico".split(";")); - this.knownCountries.add("qa;Qatar".split(";")); - this.knownCountries.add("re;Reunion".split(";")); - this.knownCountries.add("ro;Romania".split(";")); - this.knownCountries.add("ru;Russia".split(";")); - this.knownCountries.add("rw;Rwanda".split(";")); - this.knownCountries.add("bl;Saint Bartelemey".split(";")); - this.knownCountries.add("sh;Saint Helena".split(";")); - this.knownCountries.add("kn;Saint Kitts and Nevis".split(";")); - this.knownCountries.add("lc;Saint Lucia".split(";")); - this.knownCountries.add("mf;Saint Martin".split(";")); - this.knownCountries.add("pm;Saint Pierre and Miquelon".split(";")); - this.knownCountries.add("vc;Saint Vincent and the Grenadines". - split(";")); - this.knownCountries.add("ws;Samoa".split(";")); - this.knownCountries.add("sm;San Marino".split(";")); - this.knownCountries.add("st:São Tomé and Príncipe". - split(":")); - this.knownCountries.add("sa;Saudi Arabia".split(";")); - this.knownCountries.add("sn;Senegal".split(";")); - this.knownCountries.add("rs;Serbia".split(";")); - this.knownCountries.add("sc;Seychelles".split(";")); - this.knownCountries.add("sl;Sierra Leone".split(";")); - this.knownCountries.add("sg;Singapore".split(";")); - this.knownCountries.add("sk;Slovakia".split(";")); - this.knownCountries.add("si;Slovenia".split(";")); - this.knownCountries.add("sb;Solomon Islands".split(";")); - this.knownCountries.add("so;Somalia".split(";")); - this.knownCountries.add("za;South Africa".split(";")); - this.knownCountries.add(("gs;South Georgia and the South Sandwich " - + "Islands").split(";")); - this.knownCountries.add("es;Spain".split(";")); - this.knownCountries.add("lk;Sri Lanka".split(";")); - this.knownCountries.add("sd;Sudan".split(";")); - this.knownCountries.add("sr;Suriname".split(";")); - this.knownCountries.add("sj;Svalbard and Jan Mayen".split(";")); - this.knownCountries.add("sz;Swaziland".split(";")); - this.knownCountries.add("se;Sweden".split(";")); - this.knownCountries.add("ch;Switzerland".split(";")); - this.knownCountries.add("sy;Syrian Arab Republic".split(";")); - this.knownCountries.add("tw;Taiwan".split(";")); - this.knownCountries.add("tj;Tajikistan".split(";")); - this.knownCountries.add("tz;Tanzania, United Republic of".split(";")); - this.knownCountries.add("th;Thailand".split(";")); - this.knownCountries.add("tl;East Timor".split(";")); - this.knownCountries.add("tg;Togo".split(";")); - this.knownCountries.add("tk;Tokelau".split(";")); - this.knownCountries.add("to;Tonga".split(";")); - this.knownCountries.add("tt;Trinidad and Tobago".split(";")); - this.knownCountries.add("tn;Tunisia".split(";")); - this.knownCountries.add("tr;Turkey".split(";")); - this.knownCountries.add("tm;Turkmenistan".split(";")); - this.knownCountries.add("tc;Turks and Caicos Islands".split(";")); - this.knownCountries.add("tv;Tuvalu".split(";")); - this.knownCountries.add("ug;Uganda".split(";")); - this.knownCountries.add("ua;Ukraine".split(";")); - this.knownCountries.add("ae;United Arab Emirates".split(";")); - this.knownCountries.add("gb;United Kingdom".split(";")); - this.knownCountries.add("um;United States Minor Outlying Islands". - split(";")); - this.knownCountries.add("us;United States".split(";")); - this.knownCountries.add("uy;Uruguay".split(";")); - this.knownCountries.add("uz;Uzbekistan".split(";")); - this.knownCountries.add("vu;Vanuatu".split(";")); - this.knownCountries.add("ve;Venezuela".split(";")); - this.knownCountries.add("vn;Vietnam".split(";")); - this.knownCountries.add("vg;Virgin Islands, British".split(";")); - this.knownCountries.add("vi;Virgin Islands, U.S.".split(";")); - this.knownCountries.add("wf;Wallis and Futuna".split(";")); - this.knownCountries.add("eh;Western Sahara".split(";")); - this.knownCountries.add("ye;Yemen".split(";")); - this.knownCountries.add("zm;Zambia".split(";")); - this.knownCountries.add("zw;Zimbabwe".split(";")); - } - - public List<String[]> getCountryList() { - return this.knownCountries; - } -} - diff --git a/task-6471/java/src/org/torproject/task6471/Database.java b/task-6471/java/src/org/torproject/task6471/Database.java index 62a5e15..448cda6 100644 --- a/task-6471/java/src/org/torproject/task6471/Database.java +++ b/task-6471/java/src/org/torproject/task6471/Database.java @@ -2,15 +2,16 @@ package org.torproject.task6471; /** - * Database storing multiple GeoIP databases and supporting efficient - * ip-to-country lookups in the most recent of those databases for any - * given date. + * Database storing multiple GeoIP or ASN databases and supporting + * efficient ip-to-country-code or ip-to-AS-number lookups in the most + * recent of those databases for any given date. * - * A typical query for this GeoIP database is: "to which country was IPv4 - * address 1.2.3.4 assigned on date 20120912?" This query is answered by - * looking at the entries from the most recent database published on or - * before 20120912. If the earliest known database was published after - * 20120912, the earliest known database is used to resolve the request. + * A typical query for this GeoIP database is: "to which country code was + * IPv4 address 1.2.3.4 assigned on date 20120912?" This query is + * answered by looking at the entries from the most recent database + * published on or before 20120912. If the earliest known database was + * published after 20120912, the earliest known database is used to + * resolve the request. */ public interface Database { @@ -24,35 +25,13 @@ public interface Database { public boolean loadCombinedDatabases(String path); /** - * Query the database for the country code assigned to an IPv4 address - * and date. + * Query the database for the country code or AS number assigned to an + * IPv4 address on a given date. * * @param address IPv4 address in dotted-quad notation. * @param date Assignment date in format yyyymmdd. - * @return Assigned country code, or null if no assignment could be - * found. + * @return Assigned country code or AS number, or null if no assignment + * could be found. */ - public String lookupCountryCodeFromIpv4AddressAndDate(String address, - String date); - - /** - * Resolve the country code to the corresponding country name. - * - * @param countryCode Country code to be resolved. - * @return Country name. - */ - public String getCountryNameForCountryCode(String countryCode); - - /** - * Query the database for the country name assigned to an IPv4 address - * and date. This is a convenience method that first looks up the - * country code and then resolves it to the country name. - * - * @param address IPv4 address in dotted-quad notation. - * @param date Assignment date in format yyyymmdd. - * @return Assigned country name, or null if no assignment could be - * found. - */ - public String lookupCountryNameFromIpv4AddressAndDate(String address, - String date); + public String lookupIpv4AddressAndDate(String address, String date); } diff --git a/task-6471/java/src/org/torproject/task6471/DatabaseImpl.java b/task-6471/java/src/org/torproject/task6471/DatabaseImpl.java index 7fa6f13..364f0c7 100644 --- a/task-6471/java/src/org/torproject/task6471/DatabaseImpl.java +++ b/task-6471/java/src/org/torproject/task6471/DatabaseImpl.java @@ -8,7 +8,6 @@ import java.io.IOException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Collections; -import java.util.List; import java.util.Map; import java.util.SortedMap; import java.util.SortedSet; @@ -17,7 +16,7 @@ import java.util.TreeMap; import java.util.TreeSet; /** - * Implementation of database holding multiple GeoIP databases with + * Implementation of database holding multiple GeoIP or ASN databases with * special focus on lookup performance, import performance, and memory * consumption (in that order). * @@ -26,7 +25,7 @@ import java.util.TreeSet; * the start IPv4 address in the higher bits and the first database * publication date containing that range in the lower bits. The tree * element itself contains the end IPv4 address, last database publication - * date, and country code. + * date, and country code or AS number. * * Lookups for a given address and random date only require iterating * backwards over ranges with start address smaller than or equaling the @@ -42,20 +41,19 @@ public class DatabaseImpl implements Database { /** * Tree element containing an end IPv4 address, last database date, - * last database index, and country code. Start IPv4 address and first - * database date are encoded in the key under which the element is - * stored. + * last database index, and country code or AS number. Start IPv4 + * address and first database date are encoded in the key under which + * the element is stored. */ protected static class TreeElement { protected long endAddress; protected int lastDbDate; - protected String countryCode; + protected String code; protected boolean modifiedInLastImport; - protected TreeElement(long endAddress, int lastDbDate, - String countryCode) { + protected TreeElement(long endAddress, int lastDbDate, String code) { this.endAddress = endAddress; this.lastDbDate = lastDbDate; - this.countryCode = countryCode; + this.code = code; this.modifiedInLastImport = true; } } @@ -86,7 +84,7 @@ public class DatabaseImpl implements Database { * Look up address and date by iterating backwards over possibly * matching ranges. */ - public String lookupCountryCodeFromIpv4AddressAndDate( + public String lookupIpv4AddressAndDate( String addressString, String dateString) { this.addressLookups++; @@ -124,8 +122,8 @@ public class DatabaseImpl implements Database { } /* Both address and date ranges match, so return the assigned - * country code. */ - return e.getValue().countryCode; + * code. */ + return e.getValue().code; } /* No ranges (left) to look at. We don't have what we were looking @@ -222,7 +220,7 @@ public class DatabaseImpl implements Database { sb.append(String.format("%n %s %s %s %s %s", convertKeyToAddressString(e.getKey()), convertAddressNumberToString(e.getValue().endAddress), - e.getValue().countryCode, + e.getValue().code, convertKeyToDateString(e.getKey()), convertDateNumberToString(e.getValue().lastDbDate))); if (--entries <= 0) { @@ -259,12 +257,12 @@ public class DatabaseImpl implements Database { String[] parts = line.split(","); long startAddress = convertAddressStringToNumber(parts[0]); long endAddress = convertAddressStringToNumber(parts[1]); - String countryCode = parts[2]; + String code = parts[2]; int firstDbDate = convertDateStringToNumber(parts[3]); int lastDbDate = convertDateStringToNumber(parts[4]); this.ranges.put(convertAddressAndDateToKey(startAddress, firstDbDate), new TreeElement(endAddress, lastDbDate, - countryCode)); + code)); } } br.close(); @@ -273,31 +271,4 @@ public class DatabaseImpl implements Database { } return true; } - - /** Mapping from country codes to country names. */ - private static SortedMap<String, String> countryNames; - static { - countryNames = new TreeMap<String, String>(); - List<String[]> countryList = Countries.getInstance().getCountryList(); - for (String[] country : countryList) { - countryNames.put(country[0], country[1]); - } - } - - /** Resolve country codes to country names. */ - public String getCountryNameForCountryCode(String countryCode) { - if (countryCode == null) { - return null; - } else { - return countryNames.get(countryCode); - } - } - - /** Lookup IPv4 address and date and return the country name. */ - public String lookupCountryNameFromIpv4AddressAndDate( - String addressString, String dateString) { - return getCountryNameForCountryCode( - lookupCountryCodeFromIpv4AddressAndDate(addressString, - dateString)); - } } diff --git a/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java b/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java index 6937cfa..4795883 100644 --- a/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java +++ b/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java @@ -94,10 +94,14 @@ public class DatabaseImporterImpl extends DatabaseImpl /* TODO Add support for IPv6 addresses. */ continue; } - String countryCode = parts[1].toLowerCase(); + String code = parts[1].toLowerCase(); + if (code.length() != 2) { + /* Don't import illegal range. */ + continue; + } String startAddressString = parts[3]; long addresses = Long.parseLong(parts[4]); - this.addRange(databaseFileName, countryCode, startAddressString, + this.addRange(databaseFileName, code, startAddressString, addresses); } br.close(); @@ -123,14 +127,9 @@ public class DatabaseImporterImpl extends DatabaseImpl * is called prior to any lookupAddress() calls. No further checks are * performed that the tree is repaired before looking up an address. */ - void addRange(String databaseFileName, String countryCode, + void addRange(String databaseFileName, String code, String startAddressString, long addresses) { - if (countryCode.length() != 2) { - /* Don't import illegal range. */ - return; - } - this.rangeImports++; String databaseDateString = databaseFileName.substring(databaseFileName.length() - 8); @@ -151,8 +150,8 @@ public class DatabaseImporterImpl extends DatabaseImpl * We shouldn't mess with the tree directly while iterating over it, * so let's for now only calculate what changes we want to make. */ SortedMap<Long, TreeElement> updateElements = - this.getUpdatesForAddingRange(databaseDate, countryCode, - startAddress, endAddress); + this.getUpdatesForAddingRange(databaseDate, code, startAddress, + endAddress); /* Apply updates. Elements with non-null values are added, elements * with null values are removed. */ @@ -169,8 +168,7 @@ public class DatabaseImporterImpl extends DatabaseImpl * Calculate necessary changes to the tree to add a range. */ private SortedMap<Long, TreeElement> getUpdatesForAddingRange( - int databaseDate, String countryCode, long startAddress, - long endAddress) { + int databaseDate, String code, long startAddress, long endAddress) { /* Keep updates in a single tree where non-null values will later be * added, possibly replacing existing elements, and null values will @@ -205,7 +203,7 @@ public class DatabaseImporterImpl extends DatabaseImpl long eEndAddress = e.getValue().endAddress; int eFirstDbDate = convertKeyToDate(e.getKey()); int eLastDbDate = e.getValue().lastDbDate; - String eCountryCode = e.getValue().countryCode; + String eCode = e.getValue().code; /* If the next (partial) range starts after the current element * ends, add the new range. */ @@ -213,7 +211,7 @@ public class DatabaseImporterImpl extends DatabaseImpl nextEndAddress >= startAddress) { updateElements.put(convertAddressAndDateToKey(nextStartAddress, nextFirstDbDate), new TreeElement(nextEndAddress, - nextLastDbDate, countryCode)); + nextLastDbDate, code)); nextEndAddress = nextStartAddress - 1L; nextStartAddress = startAddress; nextFirstDbDate = databaseDate; @@ -226,7 +224,7 @@ public class DatabaseImporterImpl extends DatabaseImpl nextEndAddress >= startAddress) { updateElements.put(convertAddressAndDateToKey(eEndAddress + 1L, databaseDate), new TreeElement(nextEndAddress, databaseDate, - countryCode)); + code)); nextEndAddress = eEndAddress; nextStartAddress = startAddress; nextFirstDbDate = databaseDate; @@ -247,10 +245,10 @@ public class DatabaseImporterImpl extends DatabaseImpl if (eStartAddress <= endAddress && eEndAddress > endAddress) { updateElements.put(convertAddressAndDateToKey(endAddress + 1L, eFirstDbDate), new TreeElement(eEndAddress, eLastDbDate, - eCountryCode)); + eCode)); updateElements.put(convertAddressAndDateToKey(eStartAddress, eFirstDbDate), new TreeElement(endAddress, eLastDbDate, - eCountryCode)); + eCode)); eEndAddress = endAddress; } @@ -260,10 +258,10 @@ public class DatabaseImporterImpl extends DatabaseImpl if (eStartAddress < startAddress && eEndAddress >= startAddress) { updateElements.put(convertAddressAndDateToKey(eStartAddress, eFirstDbDate), new TreeElement(startAddress - 1L, eLastDbDate, - eCountryCode)); + eCode)); updateElements.put(convertAddressAndDateToKey(startAddress, eFirstDbDate), new TreeElement(eEndAddress, eLastDbDate, - eCountryCode)); + eCode)); eStartAddress = startAddress; } @@ -272,11 +270,11 @@ public class DatabaseImporterImpl extends DatabaseImpl nextStartAddress = eStartAddress; nextEndAddress = eEndAddress; - /* If the range is already contained and has the same country code, - * mark it as updated. If it's contained with a different country - * code, ignore the update. */ + /* If the range is already contained and has the same code, mark it + * as updated. If it's contained with a different code, ignore the + * update. */ if (eFirstDbDate <= databaseDate && eLastDbDate >= databaseDate) { - if (eCountryCode.equals(countryCode)) { + if (eCode.equals(code)) { nextFirstDbDate = eFirstDbDate; nextLastDbDate = eLastDbDate; } else { @@ -288,7 +286,7 @@ public class DatabaseImporterImpl extends DatabaseImpl /* See if we can merge the new range with the previous or next * range. If so, extend our database range and mark the existing * element for deletion. */ - if (eCountryCode.equals(countryCode)) { + if (eCode.equals(code)) { if (eLastDbDate == previousDatabaseDate) { nextFirstDbDate = eFirstDbDate; updateElements.put(convertAddressAndDateToKey(eStartAddress, @@ -306,7 +304,7 @@ public class DatabaseImporterImpl extends DatabaseImpl while (nextEndAddress >= startAddress) { updateElements.put(convertAddressAndDateToKey(nextStartAddress, nextFirstDbDate), new TreeElement(nextEndAddress, - nextLastDbDate, countryCode)); + nextLastDbDate, code)); nextEndAddress = nextStartAddress - 1L; nextStartAddress = startAddress; nextFirstDbDate = databaseDate; @@ -355,7 +353,7 @@ public class DatabaseImporterImpl extends DatabaseImpl int eLastDbDate = e.getValue().lastDbDate; long eStartAddress = convertKeyToAddress(e.getKey()); long eEndAddress = e.getValue().endAddress; - String eCountryCode = e.getValue().countryCode; + String eCode = e.getValue().code; int start = eFirstDbDate, end = eFirstDbDate; for (int cur : this.databaseDates.tailSet(eFirstDbDate)) { if (cur > eLastDbDate) { @@ -364,8 +362,7 @@ public class DatabaseImporterImpl extends DatabaseImpl if (cur == addedDatabaseDate) { if (start >= 0 && end >= 0) { updateElements.put(convertAddressAndDateToKey(eStartAddress, - start), new TreeElement(eEndAddress, end, - eCountryCode)); + start), new TreeElement(eEndAddress, end, eCode)); start = end = -1; } } else if (start < 0) { @@ -376,7 +373,7 @@ public class DatabaseImporterImpl extends DatabaseImpl } if (start >= 0 && end >= 0) { updateElements.put(convertAddressAndDateToKey(eStartAddress, - start), new TreeElement(eEndAddress, end, eCountryCode)); + start), new TreeElement(eEndAddress, end, eCode)); } } } @@ -422,7 +419,7 @@ public class DatabaseImporterImpl extends DatabaseImpl bw.write(String.format("%s,%s,%s,%s,%s%n", convertKeyToAddressString(e.getKey()), convertAddressNumberToString(e.getValue().endAddress), - e.getValue().countryCode, + e.getValue().code, convertKeyToDateString(e.getKey()), convertDateNumberToString(e.getValue().lastDbDate))); } @@ -452,7 +449,7 @@ public class DatabaseImporterImpl extends DatabaseImpl sb.append(String.format("%n %s %s %s %s %s", convertKeyToAddressString(e.getKey()), convertAddressNumberToString(e.getValue().endAddress), - e.getValue().countryCode, + e.getValue().code, convertKeyToDateString(e.getKey()), convertDateNumberToString(e.getValue().lastDbDate))); if (--entries <= 0) { diff --git a/task-6471/java/src/org/torproject/task6471/DatabasePerformanceExample.java b/task-6471/java/src/org/torproject/task6471/DatabasePerformanceExample.java index 77808c8..5f8573a 100644 --- a/task-6471/java/src/org/torproject/task6471/DatabasePerformanceExample.java +++ b/task-6471/java/src/org/torproject/task6471/DatabasePerformanceExample.java @@ -75,11 +75,10 @@ public class DatabasePerformanceExample { if (testMonth.equals(dbMonth)) { String testAddressString = DatabaseImpl. convertKeyToAddressString(test); - String countryCode = - temp.lookupCountryCodeFromIpv4AddressAndDate( + String code = temp.lookupIpv4AddressAndDate( testAddressString, testDateString); - if (countryCode != null) { - results.put(test, countryCode); + if (code != null) { + results.put(test, code); } } } @@ -91,9 +90,9 @@ public class DatabasePerformanceExample { convertKeyToAddressString(e.getKey()); String testDateString = DatabaseImpl. convertKeyToDateString(e.getKey()); - String countryCode = e.getValue(); - bw.write(testAddressString + "," + testDateString + "," - + countryCode + "\n"); + String code = e.getValue(); + bw.write(testAddressString + "," + testDateString + "," + code + + "\n"); } bw.close(); long endMillis = System.currentTimeMillis(); @@ -124,7 +123,7 @@ public class DatabasePerformanceExample { } String expected = "??".equals(parts[2]) ? null : parts[2]; String result = - combinedDatabase.lookupCountryCodeFromIpv4AddressAndDate( + combinedDatabase.lookupIpv4AddressAndDate( testAddress, testDate); tests++; if ((expected == null && result != null) || @@ -165,7 +164,7 @@ public class DatabasePerformanceExample { String testDate = parts[1]; String expected = parts[2].equals("??") ? null : parts[2]; String result = - combinedDatabase.lookupCountryCodeFromIpv4AddressAndDate( + combinedDatabase.lookupIpv4AddressAndDate( testAddress, testDate); tests++; if ((expected == null && result != null) || diff --git a/task-6471/java/src/org/torproject/task6471/DatabaseTest.java b/task-6471/java/src/org/torproject/task6471/DatabaseTest.java index daba414..fb0f19d 100644 --- a/task-6471/java/src/org/torproject/task6471/DatabaseTest.java +++ b/task-6471/java/src/org/torproject/task6471/DatabaseTest.java @@ -15,48 +15,45 @@ public class DatabaseTest { database.addRange("20120901", "us", "3.0.0.0", 16777216); database.repairTree(); assertEquals(1, database.getNumberOfElements()); - assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals(null, database.lookupIpv4AddressAndDate( "2.255.255.255", "19920901")); - assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals(null, database.lookupIpv4AddressAndDate( "2.255.255.255", "20020901")); - assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals(null, database.lookupIpv4AddressAndDate( "2.255.255.255", "20120901")); - assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals(null, database.lookupIpv4AddressAndDate( "2.255.255.255", "20220901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.0.0.0", "19920901")); - assertEquals("United States", - database.lookupCountryNameFromIpv4AddressAndDate( - "3.0.0.0", "19920901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.0.0.0", "20020901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.0.0.0", "20120901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.0.0.0", "20220901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "19920901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20020901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20120901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20220901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.255.255.255", "19920901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.255.255.255", "20020901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.255.255.255", "20120901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.255.255.255", "20220901")); - assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals(null, database.lookupIpv4AddressAndDate( "4.0.0.0", "19920901")); - assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals(null, database.lookupIpv4AddressAndDate( "4.0.0.0", "20020901")); - assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals(null, database.lookupIpv4AddressAndDate( "4.0.0.0", "20120901")); - assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals(null, database.lookupIpv4AddressAndDate( "4.0.0.0", "20220901")); } @@ -67,21 +64,21 @@ public class DatabaseTest { database.addRange("20120901", "ca", "4.0.0.0", 16777216); database.repairTree(); assertEquals(2, database.getNumberOfElements()); - assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals(null, database.lookupIpv4AddressAndDate( "2.255.255.255", "20120901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20120901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20120901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20120901")); - assertEquals("ca", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("ca", database.lookupIpv4AddressAndDate( "4.127.0.0", "20120901")); - assertEquals("ca", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("ca", database.lookupIpv4AddressAndDate( "4.127.0.0", "20120901")); - assertEquals("ca", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("ca", database.lookupIpv4AddressAndDate( "4.127.0.0", "20120901")); - assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals(null, database.lookupIpv4AddressAndDate( "5.0.0.0", "20120901")); } @@ -92,15 +89,15 @@ public class DatabaseTest { database.addRange("20120901", "ca", "6.0.0.0", 16777216); database.repairTree(); assertEquals(2, database.getNumberOfElements()); - assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals(null, database.lookupIpv4AddressAndDate( "2.255.255.255", "20120901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20120901")); - assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals(null, database.lookupIpv4AddressAndDate( "4.255.255.255", "20120901")); - assertEquals("ca", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("ca", database.lookupIpv4AddressAndDate( "6.127.0.0", "20120901")); - assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals(null, database.lookupIpv4AddressAndDate( "7.0.0.0", "20120901")); } @@ -111,22 +108,22 @@ public class DatabaseTest { database.addRange("20120901", "us", "3.0.0.0", 16777216); database.repairTree(); assertEquals(1, database.getNumberOfElements()); - assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals(null, database.lookupIpv4AddressAndDate( "2.255.255.255", "20120901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20120901")); - assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals(null, database.lookupIpv4AddressAndDate( "4.0.0.0", "20120901")); } @Test() - public void testDuplicateImportDifferentCountryCode() { + public void testDuplicateImportDifferentCode() { DatabaseImporterImpl database = new DatabaseImporterImpl(); database.addRange("20120901", "us", "3.0.0.0", 16777216); database.addRange("20120901", "ca", "3.0.0.0", 16777216); database.repairTree(); assertEquals(1, database.getNumberOfElements()); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20120901")); } @@ -138,13 +135,13 @@ public class DatabaseTest { database.addRange("20121001", "us", "3.0.0.0", 16777216); database.repairTree(); assertEquals(1, database.getNumberOfElements()); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20120801")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20120901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20121001")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20121101")); } @@ -156,13 +153,13 @@ public class DatabaseTest { database.addRange("20120901", "us", "3.0.0.0", 16777216); database.repairTree(); assertEquals(1, database.getNumberOfElements()); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20120801")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20120901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20121001")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20121101")); } @@ -176,13 +173,13 @@ public class DatabaseTest { database.addRange("20121001", "us", "6.0.0.0", 16777216); database.repairTree(); assertEquals(3, database.getNumberOfElements()); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20120801")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20120901")); - assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals(null, database.lookupIpv4AddressAndDate( "3.127.0.0", "20121001")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20121101")); } }

1 0

[metrics-tasks/master] Change combined database file header (#6471).
by karsten＠torproject.org 06 Nov '12

06 Nov '12

commit d84ccdd2a0ad3d3f7e0aed2c65dad3b5e7ad81d0 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Nov 5 20:57:06 2012 -0500 Change combined database file header (#6471). File header now contains database dates and file names. --- .../src/org/torproject/task6471/DatabaseImpl.java | 9 ++-- .../torproject/task6471/DatabaseImporterImpl.java | 55 ++++++++------------ .../task6471/DatabasePerformanceExample.java | 2 +- 3 files changed, 27 insertions(+), 39 deletions(-) diff --git a/task-6471/java/src/org/torproject/task6471/DatabaseImpl.java b/task-6471/java/src/org/torproject/task6471/DatabaseImpl.java index 364f0c7..0338376 100644 --- a/task-6471/java/src/org/torproject/task6471/DatabaseImpl.java +++ b/task-6471/java/src/org/torproject/task6471/DatabaseImpl.java @@ -71,7 +71,7 @@ public class DatabaseImpl implements Database { protected SortedSet<Integer> databaseDates = new TreeSet<Integer>(); /** - * Database file names. + * Database dates and file names, formatted as yyyymmdd!filename. */ protected SortedSet<String> databaseFileNames = new TreeSet<String>(); @@ -243,11 +243,10 @@ public class DatabaseImpl implements Database { if (line.startsWith("!")) { /* First read file header containing database dates. */ - String databaseFileName = line.substring(1); - String databaseDateString = - databaseFileName.substring(databaseFileName.length() - 8); + String[] parts = line.substring(1).split("!"); + this.databaseFileNames.add(line.substring(1)); + String databaseDateString = parts[0]; int dbDate = convertDateStringToNumber(databaseDateString); - this.databaseFileNames.add(databaseFileName); this.databaseDates.add(dbDate); } else { diff --git a/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java b/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java index 2a6c203..0f7df77 100644 --- a/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java +++ b/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java @@ -6,7 +6,6 @@ import java.io.File; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; -import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -17,7 +16,6 @@ import java.util.Map; import java.util.Set; import java.util.SortedMap; import java.util.Stack; -import java.util.TimeZone; import java.util.TreeMap; public class DatabaseImporterImpl extends DatabaseImpl @@ -55,12 +53,6 @@ public class DatabaseImporterImpl extends DatabaseImpl } Collections.sort(allFiles, Collections.reverseOrder()); for (File file : allFiles) { - String databaseFileName = file.getName(); - if (this.databaseFileNames.contains(databaseFileName)) { - /* We already imported this file while loading combined databases - * from disk. */ - continue; - } if (!this.importRegionalRegistryStatsFile(file)) { allImportsSuccessful = false; } @@ -124,8 +116,8 @@ public class DatabaseImporterImpl extends DatabaseImpl int databaseDate = convertDateStringToNumber(databaseDateString); long startAddress = convertAddressStringToNumber(startAddressString); long endAddress = startAddress + addresses - 1L; - this.addRange(databaseFileName, databaseDate, startAddress, - endAddress, code); + this.addDatabase(databaseFileName, databaseDate); + this.addRange(databaseDate, startAddress, endAddress, code); } public boolean importGeoLiteCityFileOrDirectory(String path) { @@ -178,13 +170,11 @@ public class DatabaseImporterImpl extends DatabaseImpl boolean importGeoLiteCityBlocksAndLocationFiles(File blocksFile, File locationFile) { - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); long lastModifiedMillis = blocksFile.lastModified(); - String databaseFileName = blocksFile.getName() + " " - + locationFile.getName() + " " - + dateFormat.format(lastModifiedMillis); + String databaseFileName = blocksFile.getName() + "+" + + locationFile.getName(); int databaseDate = (int) (lastModifiedMillis / 86400000); + this.addDatabase(databaseFileName, databaseDate); try { /* Parse location file first and remember country codes for given * locations. */ @@ -221,8 +211,7 @@ public class DatabaseImporterImpl extends DatabaseImpl break; } String code = locations.get(location); - this.addRange(databaseFileName, databaseDate, startAddress, - endAddress, code); + this.addRange(databaseDate, startAddress, endAddress, code); } br.close(); } catch (IOException e) { @@ -257,12 +246,10 @@ public class DatabaseImporterImpl extends DatabaseImpl } private boolean importGeoIPASNum2File(File file) { - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); long lastModifiedMillis = file.lastModified(); - String databaseFileName = file.getName() + " " - + dateFormat.format(lastModifiedMillis); + String databaseFileName = file.getName(); int databaseDate = (int) (lastModifiedMillis / 86400000); + this.addDatabase(databaseFileName, databaseDate); try { BufferedReader br = new BufferedReader(new FileReader(file)); String line; @@ -275,8 +262,7 @@ public class DatabaseImporterImpl extends DatabaseImpl /* Don't import illegal range. */ continue; } - this.addRange(databaseFileName, databaseDate, startAddress, - endAddress, code); + this.addRange(databaseDate, startAddress, endAddress, code); } br.close(); this.repairTree(); @@ -292,6 +278,17 @@ public class DatabaseImporterImpl extends DatabaseImpl private int rangeImports = 0, rangeImportsKeyLookups = 0; /** + * Add new database date and file name if we didn't know them yet. */ + void addDatabase(String databaseFileName, int databaseDate) { + if (!this.databaseDates.contains(databaseDate)) { + this.databaseDates.add(databaseDate); + this.addedDatabaseDate = databaseDate; + } + this.databaseFileNames.add(convertDateNumberToString(databaseDate) + + "!" + databaseFileName); + } + + /** * Add a single address and date range to the tree, which may require * splitting up existing ranges. * @@ -300,18 +297,10 @@ public class DatabaseImporterImpl extends DatabaseImpl * is called prior to any lookupAddress() calls. No further checks are * performed that the tree is repaired before looking up an address. */ - void addRange(String databaseFileName, int databaseDate, - long startAddress, long endAddress, String code) { + void addRange(int databaseDate, long startAddress, long endAddress, + String code) { this.rangeImports++; - /* Add new database date and file name if we didn't know them yet, - * and note that we need to repair the tree after importing. */ - if (!this.databaseDates.contains(databaseDate)) { - this.databaseDates.add(databaseDate); - this.addedDatabaseDate = databaseDate; - } - this.databaseFileNames.add(databaseFileName); - /* We might have to split existing ranges or the new range before * adding it to the tree, and we might have to remove existing ranges. * We shouldn't mess with the tree directly while iterating over it, diff --git a/task-6471/java/src/org/torproject/task6471/DatabasePerformanceExample.java b/task-6471/java/src/org/torproject/task6471/DatabasePerformanceExample.java index 5f8573a..bdfb140 100644 --- a/task-6471/java/src/org/torproject/task6471/DatabasePerformanceExample.java +++ b/task-6471/java/src/org/torproject/task6471/DatabasePerformanceExample.java @@ -19,7 +19,7 @@ import java.util.TreeSet; public class DatabasePerformanceExample { public static void main(String[] args) throws IOException { - File testCasesCsvFile = new File("test-cases.csv"); + File testCasesCsvFile = new File("test-cases-2007-10-2012-09.csv"); if (!testCasesCsvFile.exists()) { System.out.print("Generating test cases... "); long startMillis = System.currentTimeMillis();

1 0

[metrics-tasks/master] Merge adjacent ranges in city databases (#6471).
by karsten＠torproject.org 06 Nov '12

06 Nov '12

commit 22bd9dce19c603a978c0847cf8647dabcc60bae8 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Nov 5 22:50:15 2012 -0500 Merge adjacent ranges in city databases (#6471). --- .../torproject/task6471/DatabaseImporterImpl.java | 21 ++++++++++++++++--- 1 files changed, 17 insertions(+), 4 deletions(-) diff --git a/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java b/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java index 0f7df77..407d8ca 100644 --- a/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java +++ b/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java @@ -194,6 +194,8 @@ public class DatabaseImporterImpl extends DatabaseImpl } br.close(); /* Parse blocks file and add ranges to the database. */ + long lastStartAddress = 0L, lastEndAddress = -2L; + String lastCode = null; br = new BufferedReader(new FileReader(blocksFile)); while ((line = br.readLine()) != null) { if (!line.startsWith("\"")) { @@ -211,7 +213,21 @@ public class DatabaseImporterImpl extends DatabaseImpl break; } String code = locations.get(location); - this.addRange(databaseDate, startAddress, endAddress, code); + if (lastStartAddress == 0L) { + lastStartAddress = startAddress; + lastCode = code; + } else if (lastEndAddress + 1L != startAddress || + !code.equals(lastCode)) { + this.addRange(databaseDate, lastStartAddress, lastEndAddress, + lastCode); + lastStartAddress = startAddress; + lastCode = code; + } + lastEndAddress = endAddress; + } + if (lastStartAddress != 0L) { + this.addRange(databaseDate, lastStartAddress, lastEndAddress, + lastCode); } br.close(); } catch (IOException e) { @@ -493,9 +509,6 @@ public class DatabaseImporterImpl extends DatabaseImpl * interface, because the caller needs to make sure that repairTree() * is called prior to any lookupAddress() calls. No further checks are * performed that the tree is repaired before look up an address. - * - * TODO While repairing the tree, we might also optimize it by merging - * adjacent address ranges with the same database date ranges. */ void repairTree() { if (this.addedDatabaseDate < 0) {

1 0

[ooni-probe/master] Refactor log file rename configuration to oconfig
by art＠torproject.org 06 Nov '12

06 Nov '12

commit e6ac93347a50096897de673a368a99c557f5cef7 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Nov 6 15:51:29 2012 +0100 Refactor log file rename configuration to oconfig --- ooni/oconfig.py | 7 +++++++ ooni/utils/log.py | 16 ++++++++-------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/ooni/oconfig.py b/ooni/oconfig.py new file mode 100644 index 0000000..96a1ee0 --- /dev/null +++ b/ooni/oconfig.py @@ -0,0 +1,7 @@ +from ooni.utils import Storage + +# XXX move this to an actual configuration file +basic = Storage() +advanced = Storage() + +advanced.debug = True diff --git a/ooni/utils/log.py b/ooni/utils/log.py index 6e99dc6..e93295a 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -11,22 +11,21 @@ from twisted.python import log as txlog from twisted.python.logfile import DailyLogFile from ooni.utils import otime -## error on import with new __init__ file: 31/10/12 -from ooni import config +from ooni import oconfig # XXX make this a config option -log_file = "/tmp/ooniprobe.log" +log_file = oconfig.basic.logfile -log_folder = os.path.join('/', *log_file.split('/')[:-1]) -log_filename = log_file.split('/')[-1] -daily_logfile = DailyLogFile(log_filename, log_folder) +def start(log_file=log_file): + log_folder = os.path.join('/', *log_file.split('/')[:-1]) + log_filename = log_file.split('/')[-1] + daily_logfile = DailyLogFile(log_filename, log_folder) -def start(): txlog.msg("Starting OONI on %s (%s UTC)" % (otime.prettyDateNow(), otime.utcPrettyDateNow())) logging.basicConfig() python_logging = txlog.PythonLoggingObserver() - if config.advanced.debug: + if oconfig.advanced.debug: python_logging.logger.setLevel(logging.DEBUG) else: python_logging.logger.setLevel(logging.INFO) @@ -49,3 +48,4 @@ def err(msg, *arg, **kw): def exception(*msg): logging.exception(msg) +

1 0

[ooni-probe/master] Make important refactoring to various parts.
by art＠torproject.org 06 Nov '12

06 Nov '12

commit 46133a4a1cdc03e741f00c57f2d53fe20095827f Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Nov 6 15:53:45 2012 +0100 Make important refactoring to various parts. * Fix bug in captive portal test --- nettests/core/captiveportal.py | 5 +- nettests/echo.py | 4 +- ooni/__init__.py | 4 +- ooni/config.py | 7 --- ooni/lib/txscapy.py | 50 ++++++++++--------- ooni/oconfig.py | 1 + ooni/runner.py | 2 +- ooni/templates/scapyt.py | 108 +++++++++++++++++++++++++++++++++++---- 8 files changed, 132 insertions(+), 49 deletions(-) diff --git a/nettests/core/captiveportal.py b/nettests/core/captiveportal.py index fdc37a0..6641985 100644 --- a/nettests/core/captiveportal.py +++ b/nettests/core/captiveportal.py @@ -65,7 +65,8 @@ class CaptivePortal(nettest.NetTestCase): name = "captivep" description = "Captive Portal Test" - requirements = None + version = '0.1' + author = "Isis LoveCruft <isis(a)torproject.org>" def http_fetch(self, url, headers={}): """ @@ -636,5 +637,3 @@ class CaptivePortal(nettest.NetTestCase): log.msg("") log.msg("Captive portal test finished!") - self.control(self.report) - diff --git a/nettests/echo.py b/nettests/echo.py index bc47519..611970e 100644 --- a/nettests/echo.py +++ b/nettests/echo.py @@ -70,9 +70,9 @@ class EchoTest(ScapyTest): form (*ifa_name, AF_FAMILY, *ifa_addr) ''' - if self.local_options: + if self.localOptions: log.debug("%s: local_options found" % self.name) - for key, value in self.local_options: + for key, value in self.localOptions.items(): log.debug("%s: setting self.%s = %s" % (key, value)) setattr(self, key, value) diff --git a/ooni/__init__.py b/ooni/__init__.py index f025d30..659d4af 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -1,4 +1,4 @@ -from . import config +from . import oconfig from . import inputunit from . import kit from . import lib @@ -14,7 +14,7 @@ from . import utils #from . import plugoo #from . import plugins -__all__ = ['config', 'inputunit', 'kit', +__all__ = ['oconfig', 'inputunit', 'kit', 'lib', 'nettest', 'oonicli', 'reporter', 'runner', 'templates', 'utils'] # XXX below are legacy related modules diff --git a/ooni/config.py b/ooni/config.py deleted file mode 100644 index 96a1ee0..0000000 --- a/ooni/config.py +++ /dev/null @@ -1,7 +0,0 @@ -from ooni.utils import Storage - -# XXX move this to an actual configuration file -basic = Storage() -advanced = Storage() - -advanced.debug = True diff --git a/ooni/lib/txscapy.py b/ooni/lib/txscapy.py index 4446be0..dffda36 100644 --- a/ooni/lib/txscapy.py +++ b/ooni/lib/txscapy.py @@ -18,7 +18,6 @@ import time from twisted.internet import protocol, base, fdesc, error, defer from twisted.internet import reactor, threads -from twisted.python import log from zope.interface import implements from scapy.all import Gen @@ -77,7 +76,7 @@ class ScapySocket(object): else: return self.ssocket.recv(self.MTU) -class Scapy(object): +class TXScapy(object): """ A twisted based wrapper for scapy send and receive functionality. @@ -90,12 +89,12 @@ class Scapy(object): write_only_answers = False pcapwriter = None recv = False + timeout_call = None + answers = [] + questions = [] def __init__(self, pkts=None, maxPacketSize=8192, reactor=None, filter=None, iface=None, nofilter=None, pcapfile=None, timeout=None, *arg, **kw): - if self.debug: - log.startLogging(sys.stdout) - self.maxPacketSize = maxPacketSize if not reactor: from twisted.internet import reactor @@ -173,13 +172,15 @@ class Scapy(object): @param question: the sent packet that matches that response. """ - if self.pcapwriter and self.write_only_answers: self.pcapwriter.write(question) self.pcapwriter.write(answer) + self.answers.append(answers) + self.questions.append(question) self.answer_count += 1 - if self.answer_count >= self.total_count: - print "Got all the answers I need" + if self.answer_count >= self.total_count and self.running: + log.debug("Got all the answers I need") + self.finalClose() self.deferred.callback(None) def processAnswer(self, pkt, hlst): @@ -211,7 +212,6 @@ class Scapy(object): """ pkt.show() - def doRead(self): """ There is something to be read on the wire. Do all the processing on the @@ -262,10 +262,13 @@ class Scapy(object): self.socket.send(pkt) def timeout(self, *arg, **kw): - log.debug("Caught a timeout with %s %s" % (arg, kw)) if not self.done: - self._reactor.callLater(self.timeoutSeconds, self.timeout, None) - else: + log.debug("I have not finished. Setting to call in %s" % + self.timeoutSeconds) + self.timeout_call = self._reactor.callLater(self.timeoutSeconds, self.timeout, None) + elif self.running: + log.debug("Cancelling timeout call") + self.finalClose() self.deferred.callback(None) def sr(self, pkts, filter=None, iface=None, nofilter=0, timeout=None, *args, **kw): @@ -287,11 +290,11 @@ class Scapy(object): @param filter: provide a BPF filter @param iface: listen answers only on the given interface """ - log.debug("Calling with %s" % pkts) + log.debug("TXScapy sending and receiving packets") self.recv = True if timeout: self.timeoutSeconds = timeout - self._reactor.callLater(timeout, self.timeout, None) + self.timeout_call = self._reactor.callLater(timeout, self.timeout, None) self._sendrcv(pkts, filter=filter, iface=iface, nofilter=nofilter) def send(self, pkts, filter=None, iface=None, nofilter=0, *args, **kw): @@ -327,7 +330,8 @@ class Scapy(object): pkt = self.outqueue.pop() except: self.done = True - if not self.recv: + if not self.recv and self.running: + log.debug("I am not in receiving state running callback") self.deferred.callback(None) return d = threads.deferToThreadPool(reactor, self.threadpool, @@ -353,25 +357,25 @@ class Scapy(object): def finalClose(self): """ - Clean all the thread related stuff up. + Clean all the shutdown related functions. """ self.shutdownID = None self.threadpool.stop() + if self.timeout_call: + self.timeout_call.cancel() + self.timeout_call = None self.running = False @defer.inlineCallbacks -def txsr(*arg, **kw): - tr = Scapy(*arg, **kw) - log.debug("Calling sr with %s, %s" % (arg, kw)) - tr.sr(*arg, **kw) +def txsr(*args, **kw): + tr = TXScapy(*args, **kw) + tr.sr(*args, **kw) yield tr.deferred tr.finalClose() @defer.inlineCallbacks def txsend(*arg, **kw): - tr = Scapy(**kw) - log.debug("Calling send with %s, %s" % (arg, kw)) + tr = TXScapy(*arg, **kw) tr.send(*arg, **kw) yield tr.deferred tr.finalClose() - diff --git a/ooni/oconfig.py b/ooni/oconfig.py index 96a1ee0..202b569 100644 --- a/ooni/oconfig.py +++ b/ooni/oconfig.py @@ -2,6 +2,7 @@ from ooni.utils import Storage # XXX move this to an actual configuration file basic = Storage() +basic.logfile = '/tmp/ooniprobe.log' advanced = Storage() advanced.debug = True diff --git a/ooni/runner.py b/ooni/runner.py index e1ecaac..97b4003 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -205,7 +205,7 @@ class ORunner(object): try: reportFile = open(config['reportfile'], 'a+') - except: + except TypeError: filename = 'report_'+date.timestamp()+'.yaml' reportFile = open(filename, 'a+') self.reporterFactory = ReporterFactory(reportFile, diff --git a/ooni/templates/scapyt.py b/ooni/templates/scapyt.py index 8751587..9e18fbd 100644 --- a/ooni/templates/scapyt.py +++ b/ooni/templates/scapyt.py @@ -7,14 +7,14 @@ import random from zope.interface import implements from twisted.python import usage from twisted.plugin import IPlugin -from twisted.internet import protocol, defer +from twisted.internet import protocol, defer, threads -from scapy.all import IP, TCP +from scapy.all import IP, TCP, send, sr from ooni.nettest import NetTestCase from ooni.utils import log -from ooni.lib.txscapy import txsr, txsend +from ooni.lib.txscapy import TXScapy class ScapyTest(NetTestCase): """ @@ -31,30 +31,116 @@ class ScapyTest(NetTestCase): receive = True timeout = 1 - pcapfile = None + pcapfile = 'packet_capture.pcap' packet = IP()/TCP() reactor = None + + answered = None + unanswered = None + + def setUp(self): if not self.reactor: from twisted.internet import reactor self.reactor = reactor - self.request = {} - self.response = {} + self.questions = [] + self.answers = [] + self.processInputs() + + def processInputs(self): + """ + Place here the logic for validating and processing of inputs and + command line arguments. + """ + pass def tearDown(self): - self.reactor.stop() + log.debug("Tearing down reactor") + + def finished(self, *arg): + log.debug("Calling final close") + + self.questions = self.txscapy.questions + self.answers = self.txscapy.answers + + log.debug("These are the questions: %s" % self.questions) + log.debug("These are the answers: %s" % self.answers) + + self.txscapy.finalClose() + + def sendReceivePackets(self): + packets = self.buildPackets() - def sendReceivePackets(self, *kw): - d = txsr(self.request, pcapfile=self.pcapfile, + log.debug("Sending and receiving %s" % packets) + + self.txscapy = TXScapy(packets, pcapfile=self.pcapfile, + timeout=self.timeout, reactor=self.reactor) + + self.txscapy.sr(packets, pcapfile=self.pcapfile, timeout=self.timeout, reactor=self.reactor) + + d = self.txscapy.deferred + d.addCallback(self.finished) + + return d + + def sendPackets(self): + log.debug("Sending and receiving of packets %s" % packets) + + packets = self.buildPackets() + + self.txscapy = TXScapy(packets, pcapfile=self.pcapfile, + timeout=self.timeout, reactor=self.reactor) + + self.txscapy.send(packets, reactor=self.reactor).deferred + + d = self.txscapy.deferred + d.addCallback(self.finished) + return d + def buildPackets(self): + """ + Override this method to build scapy packets. + """ + pass + +class BlockingScapyTest(ScapyTest): + """ + This is a very basic Scapy Test template that does not do all the + multithreading kung-fu of txscapy, but maintains the same API. + + This will allow tests implemented using the BlockingScapyTest API to easily + migrate to the new API. + """ + name = "Blocking Scapy Test" + version = 0.1 + + timeout = None + + answered = None + unanswered = None + + def sendReceivePackets(self): + packets = self.buildPackets() + + log.debug("Sending and receiving %s" % packets) + + self.answered, self.unanswered = sr(packets, timeout=self.timeout) + + log.debug("%s %s" % (ans, unans)) + def sendPackets(self): - return txsend(self.buildPackets(), reactor=self.reactor) + packets = self.buildPackets() + + log.debug("Sending packets %s" % packets) + + send(packets) + def buildPackets(self): """ Override this method to build scapy packets. """ - return self.packet + pass

1 0