commit 5ffcbdc3713b4e3e698b7fee147e5301d242ef80 Author: Karsten Loesing karsten.loesing@gmx.net Date: Sat Nov 3 21:39:13 2012 -0400
Generalize database to AS numbers (#6471). --- .../src/org/torproject/task6471/Countries.java | 284 -------------------- .../java/src/org/torproject/task6471/Database.java | 49 +--- .../src/org/torproject/task6471/DatabaseImpl.java | 57 +--- .../torproject/task6471/DatabaseImporterImpl.java | 59 ++--- .../task6471/DatabasePerformanceExample.java | 17 +- .../src/org/torproject/task6471/DatabaseTest.java | 103 ++++---- 6 files changed, 114 insertions(+), 455 deletions(-)
diff --git a/task-6471/java/src/org/torproject/task6471/Countries.java b/task-6471/java/src/org/torproject/task6471/Countries.java deleted file mode 100644 index a70c4bf..0000000 --- a/task-6471/java/src/org/torproject/task6471/Countries.java +++ /dev/null @@ -1,284 +0,0 @@ -/* Copyright 2011, 2012 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.task6471; - -import java.util.ArrayList; -import java.util.List; - -public class Countries { - - private static Countries instance = new Countries(); - - public static Countries getInstance() { - return Countries.instance; - } - - /* List of arrays of length 2, containing country codes at [0] and - * country names at [1], alphabetically ordered by country names. */ - private List<String[]> knownCountries; - - private Countries() { - this.knownCountries = new ArrayList<String[]>(); - this.knownCountries.add("af;Afghanistan".split(";")); - this.knownCountries.add("ax;Aland Islands".split(";")); - this.knownCountries.add("al;Albania".split(";")); - this.knownCountries.add("dz;Algeria".split(";")); - this.knownCountries.add("as;American Samoa".split(";")); - this.knownCountries.add("ad;Andorra".split(";")); - this.knownCountries.add("ao;Angola".split(";")); - this.knownCountries.add("ai;Anguilla".split(";")); - this.knownCountries.add("aq;Antarctica".split(";")); - this.knownCountries.add("ag;Antigua and Barbuda".split(";")); - this.knownCountries.add("ar;Argentina".split(";")); - this.knownCountries.add("am;Armenia".split(";")); - this.knownCountries.add("aw;Aruba".split(";")); - this.knownCountries.add("au;Australia".split(";")); - this.knownCountries.add("at;Austria".split(";")); - this.knownCountries.add("az;Azerbaijan".split(";")); - this.knownCountries.add("bs;Bahamas".split(";")); - this.knownCountries.add("bh;Bahrain".split(";")); - this.knownCountries.add("bd;Bangladesh".split(";")); - this.knownCountries.add("bb;Barbados".split(";")); - this.knownCountries.add("by;Belarus".split(";")); - this.knownCountries.add("be;Belgium".split(";")); - this.knownCountries.add("bz;Belize".split(";")); - this.knownCountries.add("bj;Benin".split(";")); - this.knownCountries.add("bm;Bermuda".split(";")); - this.knownCountries.add("bt;Bhutan".split(";")); - this.knownCountries.add("bo;Bolivia".split(";")); - this.knownCountries.add("ba;Bosnia and Herzegovina".split(";")); - this.knownCountries.add("bw;Botswana".split(";")); - this.knownCountries.add("bv;Bouvet Island".split(";")); - this.knownCountries.add("br;Brazil".split(";")); - this.knownCountries.add("io;British Indian Ocean Territory". - split(";")); - this.knownCountries.add("bn;Brunei".split(";")); - this.knownCountries.add("bg;Bulgaria".split(";")); - this.knownCountries.add("bf;Burkina Faso".split(";")); - this.knownCountries.add("mm;Burma".split(";")); - this.knownCountries.add("bi;Burundi".split(";")); - this.knownCountries.add("kh;Cambodia".split(";")); - this.knownCountries.add("cm;Cameroon".split(";")); - this.knownCountries.add("ca;Canada".split(";")); - this.knownCountries.add("cv;Cape Verde".split(";")); - this.knownCountries.add("ky;Cayman Islands".split(";")); - this.knownCountries.add("cf;Central African Republic".split(";")); - this.knownCountries.add("td;Chad".split(";")); - this.knownCountries.add("cl;Chile".split(";")); - this.knownCountries.add("cn;China".split(";")); - this.knownCountries.add("cx;Christmas Island".split(";")); - this.knownCountries.add("cc;Cocos (Keeling) Islands".split(";")); - this.knownCountries.add("co;Colombia".split(";")); - this.knownCountries.add("km;Comoros".split(";")); - this.knownCountries.add("cd;Congo, The Democratic Republic of the". - split(";")); - this.knownCountries.add("cg;Congo".split(";")); - this.knownCountries.add("ck;Cook Islands".split(";")); - this.knownCountries.add("cr;Costa Rica".split(";")); - this.knownCountries.add("ci:Côte d'Ivoire".split(":")); - this.knownCountries.add("hr;Croatia".split(";")); - this.knownCountries.add("cu;Cuba".split(";")); - this.knownCountries.add("cy;Cyprus".split(";")); - this.knownCountries.add("cz;Czech Republic".split(";")); - this.knownCountries.add("dk;Denmark".split(";")); - this.knownCountries.add("dj;Djibouti".split(";")); - this.knownCountries.add("dm;Dominica".split(";")); - this.knownCountries.add("do;Dominican Republic".split(";")); - this.knownCountries.add("ec;Ecuador".split(";")); - this.knownCountries.add("eg;Egypt".split(";")); - this.knownCountries.add("sv;El Salvador".split(";")); - this.knownCountries.add("gq;Equatorial Guinea".split(";")); - this.knownCountries.add("er;Eritrea".split(";")); - this.knownCountries.add("ee;Estonia".split(";")); - this.knownCountries.add("et;Ethiopia".split(";")); - this.knownCountries.add("fk;Falkland Islands (Malvinas)".split(";")); - this.knownCountries.add("fo;Faroe Islands".split(";")); - this.knownCountries.add("fj;Fiji".split(";")); - this.knownCountries.add("fi;Finland".split(";")); - this.knownCountries.add("fx;France, Metropolitan".split(";")); - this.knownCountries.add("fr;France".split(";")); - this.knownCountries.add("gf;French Guiana".split(";")); - this.knownCountries.add("pf;French Polynesia".split(";")); - this.knownCountries.add("tf;French Southern Territories".split(";")); - this.knownCountries.add("ga;Gabon".split(";")); - this.knownCountries.add("gm;Gambia".split(";")); - this.knownCountries.add("ge;Georgia".split(";")); - this.knownCountries.add("de;Germany".split(";")); - this.knownCountries.add("gh;Ghana".split(";")); - this.knownCountries.add("gi;Gibraltar".split(";")); - this.knownCountries.add("gr;Greece".split(";")); - this.knownCountries.add("gl;Greenland".split(";")); - this.knownCountries.add("gd;Grenada".split(";")); - this.knownCountries.add("gp;Guadeloupe".split(";")); - this.knownCountries.add("gu;Guam".split(";")); - this.knownCountries.add("gt;Guatemala".split(";")); - this.knownCountries.add("gg;Guernsey".split(";")); - this.knownCountries.add("gn;Guinea".split(";")); - this.knownCountries.add("gw;Guinea-Bissau".split(";")); - this.knownCountries.add("gy;Guyana".split(";")); - this.knownCountries.add("ht;Haiti".split(";")); - this.knownCountries.add("hm;Heard Island and McDonald Islands". - split(";")); - this.knownCountries.add("va;Vatican City".split(";")); - this.knownCountries.add("hn;Honduras".split(";")); - this.knownCountries.add("hk;Hong Kong".split(";")); - this.knownCountries.add("hu;Hungary".split(";")); - this.knownCountries.add("is;Iceland".split(";")); - this.knownCountries.add("in;India".split(";")); - this.knownCountries.add("id;Indonesia".split(";")); - this.knownCountries.add("ir;Iran".split(";")); - this.knownCountries.add("iq;Iraq".split(";")); - this.knownCountries.add("ie;Ireland".split(";")); - this.knownCountries.add("im;Isle of Man".split(";")); - this.knownCountries.add("il;Israel".split(";")); - this.knownCountries.add("it;Italy".split(";")); - this.knownCountries.add("jm;Jamaica".split(";")); - this.knownCountries.add("jp;Japan".split(";")); - this.knownCountries.add("je;Jersey".split(";")); - this.knownCountries.add("jo;Jordan".split(";")); - this.knownCountries.add("kz;Kazakhstan".split(";")); - this.knownCountries.add("ke;Kenya".split(";")); - this.knownCountries.add("ki;Kiribati".split(";")); - this.knownCountries.add("kp;North Korea".split(";")); - this.knownCountries.add("kr;Korea, Republic of".split(";")); - this.knownCountries.add("kw;Kuwait".split(";")); - this.knownCountries.add("kg;Kyrgyzstan".split(";")); - this.knownCountries.add("la;Laos".split(";")); - this.knownCountries.add("lv;Latvia".split(";")); - this.knownCountries.add("lb;Lebanon".split(";")); - this.knownCountries.add("ls;Lesotho".split(";")); - this.knownCountries.add("lr;Liberia".split(";")); - this.knownCountries.add("ly;Libya".split(";")); - this.knownCountries.add("li;Liechtenstein".split(";")); - this.knownCountries.add("lt;Lithuania".split(";")); - this.knownCountries.add("lu;Luxembourg".split(";")); - this.knownCountries.add("mo;Macau".split(";")); - this.knownCountries.add("mk;Macedonia".split(";")); - this.knownCountries.add("mg;Madagascar".split(";")); - this.knownCountries.add("mw;Malawi".split(";")); - this.knownCountries.add("my;Malaysia".split(";")); - this.knownCountries.add("mv;Maldives".split(";")); - this.knownCountries.add("ml;Mali".split(";")); - this.knownCountries.add("mt;Malta".split(";")); - this.knownCountries.add("mh;Marshall Islands".split(";")); - this.knownCountries.add("mq;Martinique".split(";")); - this.knownCountries.add("mr;Mauritania".split(";")); - this.knownCountries.add("mu;Mauritius".split(";")); - this.knownCountries.add("yt;Mayotte".split(";")); - this.knownCountries.add("mx;Mexico".split(";")); - this.knownCountries.add("fm;Micronesia, Federated States of". - split(";")); - this.knownCountries.add("md;Moldova, Republic of".split(";")); - this.knownCountries.add("mc;Monaco".split(";")); - this.knownCountries.add("mn;Mongolia".split(";")); - this.knownCountries.add("me;Montenegro".split(";")); - this.knownCountries.add("ms;Montserrat".split(";")); - this.knownCountries.add("ma;Morocco".split(";")); - this.knownCountries.add("mz;Mozambique".split(";")); - this.knownCountries.add("mm;Burma".split(";")); - this.knownCountries.add("na;Namibia".split(";")); - this.knownCountries.add("nr;Nauru".split(";")); - this.knownCountries.add("np;Nepal".split(";")); - this.knownCountries.add("an;Netherlands Antilles".split(";")); - this.knownCountries.add("nl;Netherlands".split(";")); - this.knownCountries.add("nc;New Caledonia".split(";")); - this.knownCountries.add("nz;New Zealand".split(";")); - this.knownCountries.add("ni;Nicaragua".split(";")); - this.knownCountries.add("ne;Niger".split(";")); - this.knownCountries.add("ng;Nigeria".split(";")); - this.knownCountries.add("nu;Niue".split(";")); - this.knownCountries.add("nf;Norfolk Island".split(";")); - this.knownCountries.add("mp;Northern Mariana Islands".split(";")); - this.knownCountries.add("no;Norway".split(";")); - this.knownCountries.add("om;Oman".split(";")); - this.knownCountries.add("pk;Pakistan".split(";")); - this.knownCountries.add("pw;Palau".split(";")); - this.knownCountries.add("ps;Palestinian Territory".split(";")); - this.knownCountries.add("pa;Panama".split(";")); - this.knownCountries.add("pg;Papua New Guinea".split(";")); - this.knownCountries.add("py;Paraguay".split(";")); - this.knownCountries.add("pe;Peru".split(";")); - this.knownCountries.add("ph;Philippines".split(";")); - this.knownCountries.add("pn;Pitcairn Islands".split(";")); - this.knownCountries.add("pl;Poland".split(";")); - this.knownCountries.add("pt;Portugal".split(";")); - this.knownCountries.add("pr;Puerto Rico".split(";")); - this.knownCountries.add("qa;Qatar".split(";")); - this.knownCountries.add("re;Reunion".split(";")); - this.knownCountries.add("ro;Romania".split(";")); - this.knownCountries.add("ru;Russia".split(";")); - this.knownCountries.add("rw;Rwanda".split(";")); - this.knownCountries.add("bl;Saint Bartelemey".split(";")); - this.knownCountries.add("sh;Saint Helena".split(";")); - this.knownCountries.add("kn;Saint Kitts and Nevis".split(";")); - this.knownCountries.add("lc;Saint Lucia".split(";")); - this.knownCountries.add("mf;Saint Martin".split(";")); - this.knownCountries.add("pm;Saint Pierre and Miquelon".split(";")); - this.knownCountries.add("vc;Saint Vincent and the Grenadines". - split(";")); - this.knownCountries.add("ws;Samoa".split(";")); - this.knownCountries.add("sm;San Marino".split(";")); - this.knownCountries.add("st:São Tomé and Príncipe". - split(":")); - this.knownCountries.add("sa;Saudi Arabia".split(";")); - this.knownCountries.add("sn;Senegal".split(";")); - this.knownCountries.add("rs;Serbia".split(";")); - this.knownCountries.add("sc;Seychelles".split(";")); - this.knownCountries.add("sl;Sierra Leone".split(";")); - this.knownCountries.add("sg;Singapore".split(";")); - this.knownCountries.add("sk;Slovakia".split(";")); - this.knownCountries.add("si;Slovenia".split(";")); - this.knownCountries.add("sb;Solomon Islands".split(";")); - this.knownCountries.add("so;Somalia".split(";")); - this.knownCountries.add("za;South Africa".split(";")); - this.knownCountries.add(("gs;South Georgia and the South Sandwich " - + "Islands").split(";")); - this.knownCountries.add("es;Spain".split(";")); - this.knownCountries.add("lk;Sri Lanka".split(";")); - this.knownCountries.add("sd;Sudan".split(";")); - this.knownCountries.add("sr;Suriname".split(";")); - this.knownCountries.add("sj;Svalbard and Jan Mayen".split(";")); - this.knownCountries.add("sz;Swaziland".split(";")); - this.knownCountries.add("se;Sweden".split(";")); - this.knownCountries.add("ch;Switzerland".split(";")); - this.knownCountries.add("sy;Syrian Arab Republic".split(";")); - this.knownCountries.add("tw;Taiwan".split(";")); - this.knownCountries.add("tj;Tajikistan".split(";")); - this.knownCountries.add("tz;Tanzania, United Republic of".split(";")); - this.knownCountries.add("th;Thailand".split(";")); - this.knownCountries.add("tl;East Timor".split(";")); - this.knownCountries.add("tg;Togo".split(";")); - this.knownCountries.add("tk;Tokelau".split(";")); - this.knownCountries.add("to;Tonga".split(";")); - this.knownCountries.add("tt;Trinidad and Tobago".split(";")); - this.knownCountries.add("tn;Tunisia".split(";")); - this.knownCountries.add("tr;Turkey".split(";")); - this.knownCountries.add("tm;Turkmenistan".split(";")); - this.knownCountries.add("tc;Turks and Caicos Islands".split(";")); - this.knownCountries.add("tv;Tuvalu".split(";")); - this.knownCountries.add("ug;Uganda".split(";")); - this.knownCountries.add("ua;Ukraine".split(";")); - this.knownCountries.add("ae;United Arab Emirates".split(";")); - this.knownCountries.add("gb;United Kingdom".split(";")); - this.knownCountries.add("um;United States Minor Outlying Islands". - split(";")); - this.knownCountries.add("us;United States".split(";")); - this.knownCountries.add("uy;Uruguay".split(";")); - this.knownCountries.add("uz;Uzbekistan".split(";")); - this.knownCountries.add("vu;Vanuatu".split(";")); - this.knownCountries.add("ve;Venezuela".split(";")); - this.knownCountries.add("vn;Vietnam".split(";")); - this.knownCountries.add("vg;Virgin Islands, British".split(";")); - this.knownCountries.add("vi;Virgin Islands, U.S.".split(";")); - this.knownCountries.add("wf;Wallis and Futuna".split(";")); - this.knownCountries.add("eh;Western Sahara".split(";")); - this.knownCountries.add("ye;Yemen".split(";")); - this.knownCountries.add("zm;Zambia".split(";")); - this.knownCountries.add("zw;Zimbabwe".split(";")); - } - - public List<String[]> getCountryList() { - return this.knownCountries; - } -} - diff --git a/task-6471/java/src/org/torproject/task6471/Database.java b/task-6471/java/src/org/torproject/task6471/Database.java index 62a5e15..448cda6 100644 --- a/task-6471/java/src/org/torproject/task6471/Database.java +++ b/task-6471/java/src/org/torproject/task6471/Database.java @@ -2,15 +2,16 @@ package org.torproject.task6471;
/** - * Database storing multiple GeoIP databases and supporting efficient - * ip-to-country lookups in the most recent of those databases for any - * given date. + * Database storing multiple GeoIP or ASN databases and supporting + * efficient ip-to-country-code or ip-to-AS-number lookups in the most + * recent of those databases for any given date. * - * A typical query for this GeoIP database is: "to which country was IPv4 - * address 1.2.3.4 assigned on date 20120912?" This query is answered by - * looking at the entries from the most recent database published on or - * before 20120912. If the earliest known database was published after - * 20120912, the earliest known database is used to resolve the request. + * A typical query for this GeoIP database is: "to which country code was + * IPv4 address 1.2.3.4 assigned on date 20120912?" This query is + * answered by looking at the entries from the most recent database + * published on or before 20120912. If the earliest known database was + * published after 20120912, the earliest known database is used to + * resolve the request. */ public interface Database {
@@ -24,35 +25,13 @@ public interface Database { public boolean loadCombinedDatabases(String path);
/** - * Query the database for the country code assigned to an IPv4 address - * and date. + * Query the database for the country code or AS number assigned to an + * IPv4 address on a given date. * * @param address IPv4 address in dotted-quad notation. * @param date Assignment date in format yyyymmdd. - * @return Assigned country code, or null if no assignment could be - * found. + * @return Assigned country code or AS number, or null if no assignment + * could be found. */ - public String lookupCountryCodeFromIpv4AddressAndDate(String address, - String date); - - /** - * Resolve the country code to the corresponding country name. - * - * @param countryCode Country code to be resolved. - * @return Country name. - */ - public String getCountryNameForCountryCode(String countryCode); - - /** - * Query the database for the country name assigned to an IPv4 address - * and date. This is a convenience method that first looks up the - * country code and then resolves it to the country name. - * - * @param address IPv4 address in dotted-quad notation. - * @param date Assignment date in format yyyymmdd. - * @return Assigned country name, or null if no assignment could be - * found. - */ - public String lookupCountryNameFromIpv4AddressAndDate(String address, - String date); + public String lookupIpv4AddressAndDate(String address, String date); } diff --git a/task-6471/java/src/org/torproject/task6471/DatabaseImpl.java b/task-6471/java/src/org/torproject/task6471/DatabaseImpl.java index 7fa6f13..364f0c7 100644 --- a/task-6471/java/src/org/torproject/task6471/DatabaseImpl.java +++ b/task-6471/java/src/org/torproject/task6471/DatabaseImpl.java @@ -8,7 +8,6 @@ import java.io.IOException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Collections; -import java.util.List; import java.util.Map; import java.util.SortedMap; import java.util.SortedSet; @@ -17,7 +16,7 @@ import java.util.TreeMap; import java.util.TreeSet;
/** - * Implementation of database holding multiple GeoIP databases with + * Implementation of database holding multiple GeoIP or ASN databases with * special focus on lookup performance, import performance, and memory * consumption (in that order). * @@ -26,7 +25,7 @@ import java.util.TreeSet; * the start IPv4 address in the higher bits and the first database * publication date containing that range in the lower bits. The tree * element itself contains the end IPv4 address, last database publication - * date, and country code. + * date, and country code or AS number. * * Lookups for a given address and random date only require iterating * backwards over ranges with start address smaller than or equaling the @@ -42,20 +41,19 @@ public class DatabaseImpl implements Database {
/** * Tree element containing an end IPv4 address, last database date, - * last database index, and country code. Start IPv4 address and first - * database date are encoded in the key under which the element is - * stored. + * last database index, and country code or AS number. Start IPv4 + * address and first database date are encoded in the key under which + * the element is stored. */ protected static class TreeElement { protected long endAddress; protected int lastDbDate; - protected String countryCode; + protected String code; protected boolean modifiedInLastImport; - protected TreeElement(long endAddress, int lastDbDate, - String countryCode) { + protected TreeElement(long endAddress, int lastDbDate, String code) { this.endAddress = endAddress; this.lastDbDate = lastDbDate; - this.countryCode = countryCode; + this.code = code; this.modifiedInLastImport = true; } } @@ -86,7 +84,7 @@ public class DatabaseImpl implements Database { * Look up address and date by iterating backwards over possibly * matching ranges. */ - public String lookupCountryCodeFromIpv4AddressAndDate( + public String lookupIpv4AddressAndDate( String addressString, String dateString) { this.addressLookups++;
@@ -124,8 +122,8 @@ public class DatabaseImpl implements Database { }
/* Both address and date ranges match, so return the assigned - * country code. */ - return e.getValue().countryCode; + * code. */ + return e.getValue().code; }
/* No ranges (left) to look at. We don't have what we were looking @@ -222,7 +220,7 @@ public class DatabaseImpl implements Database { sb.append(String.format("%n %s %s %s %s %s", convertKeyToAddressString(e.getKey()), convertAddressNumberToString(e.getValue().endAddress), - e.getValue().countryCode, + e.getValue().code, convertKeyToDateString(e.getKey()), convertDateNumberToString(e.getValue().lastDbDate))); if (--entries <= 0) { @@ -259,12 +257,12 @@ public class DatabaseImpl implements Database { String[] parts = line.split(","); long startAddress = convertAddressStringToNumber(parts[0]); long endAddress = convertAddressStringToNumber(parts[1]); - String countryCode = parts[2]; + String code = parts[2]; int firstDbDate = convertDateStringToNumber(parts[3]); int lastDbDate = convertDateStringToNumber(parts[4]); this.ranges.put(convertAddressAndDateToKey(startAddress, firstDbDate), new TreeElement(endAddress, lastDbDate, - countryCode)); + code)); } } br.close(); @@ -273,31 +271,4 @@ public class DatabaseImpl implements Database { } return true; } - - /** Mapping from country codes to country names. */ - private static SortedMap<String, String> countryNames; - static { - countryNames = new TreeMap<String, String>(); - List<String[]> countryList = Countries.getInstance().getCountryList(); - for (String[] country : countryList) { - countryNames.put(country[0], country[1]); - } - } - - /** Resolve country codes to country names. */ - public String getCountryNameForCountryCode(String countryCode) { - if (countryCode == null) { - return null; - } else { - return countryNames.get(countryCode); - } - } - - /** Lookup IPv4 address and date and return the country name. */ - public String lookupCountryNameFromIpv4AddressAndDate( - String addressString, String dateString) { - return getCountryNameForCountryCode( - lookupCountryCodeFromIpv4AddressAndDate(addressString, - dateString)); - } } diff --git a/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java b/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java index 6937cfa..4795883 100644 --- a/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java +++ b/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java @@ -94,10 +94,14 @@ public class DatabaseImporterImpl extends DatabaseImpl /* TODO Add support for IPv6 addresses. */ continue; } - String countryCode = parts[1].toLowerCase(); + String code = parts[1].toLowerCase(); + if (code.length() != 2) { + /* Don't import illegal range. */ + continue; + } String startAddressString = parts[3]; long addresses = Long.parseLong(parts[4]); - this.addRange(databaseFileName, countryCode, startAddressString, + this.addRange(databaseFileName, code, startAddressString, addresses); } br.close(); @@ -123,14 +127,9 @@ public class DatabaseImporterImpl extends DatabaseImpl * is called prior to any lookupAddress() calls. No further checks are * performed that the tree is repaired before looking up an address. */ - void addRange(String databaseFileName, String countryCode, + void addRange(String databaseFileName, String code, String startAddressString, long addresses) {
- if (countryCode.length() != 2) { - /* Don't import illegal range. */ - return; - } - this.rangeImports++; String databaseDateString = databaseFileName.substring(databaseFileName.length() - 8); @@ -151,8 +150,8 @@ public class DatabaseImporterImpl extends DatabaseImpl * We shouldn't mess with the tree directly while iterating over it, * so let's for now only calculate what changes we want to make. */ SortedMap<Long, TreeElement> updateElements = - this.getUpdatesForAddingRange(databaseDate, countryCode, - startAddress, endAddress); + this.getUpdatesForAddingRange(databaseDate, code, startAddress, + endAddress);
/* Apply updates. Elements with non-null values are added, elements * with null values are removed. */ @@ -169,8 +168,7 @@ public class DatabaseImporterImpl extends DatabaseImpl * Calculate necessary changes to the tree to add a range. */ private SortedMap<Long, TreeElement> getUpdatesForAddingRange( - int databaseDate, String countryCode, long startAddress, - long endAddress) { + int databaseDate, String code, long startAddress, long endAddress) {
/* Keep updates in a single tree where non-null values will later be * added, possibly replacing existing elements, and null values will @@ -205,7 +203,7 @@ public class DatabaseImporterImpl extends DatabaseImpl long eEndAddress = e.getValue().endAddress; int eFirstDbDate = convertKeyToDate(e.getKey()); int eLastDbDate = e.getValue().lastDbDate; - String eCountryCode = e.getValue().countryCode; + String eCode = e.getValue().code;
/* If the next (partial) range starts after the current element * ends, add the new range. */ @@ -213,7 +211,7 @@ public class DatabaseImporterImpl extends DatabaseImpl nextEndAddress >= startAddress) { updateElements.put(convertAddressAndDateToKey(nextStartAddress, nextFirstDbDate), new TreeElement(nextEndAddress, - nextLastDbDate, countryCode)); + nextLastDbDate, code)); nextEndAddress = nextStartAddress - 1L; nextStartAddress = startAddress; nextFirstDbDate = databaseDate; @@ -226,7 +224,7 @@ public class DatabaseImporterImpl extends DatabaseImpl nextEndAddress >= startAddress) { updateElements.put(convertAddressAndDateToKey(eEndAddress + 1L, databaseDate), new TreeElement(nextEndAddress, databaseDate, - countryCode)); + code)); nextEndAddress = eEndAddress; nextStartAddress = startAddress; nextFirstDbDate = databaseDate; @@ -247,10 +245,10 @@ public class DatabaseImporterImpl extends DatabaseImpl if (eStartAddress <= endAddress && eEndAddress > endAddress) { updateElements.put(convertAddressAndDateToKey(endAddress + 1L, eFirstDbDate), new TreeElement(eEndAddress, eLastDbDate, - eCountryCode)); + eCode)); updateElements.put(convertAddressAndDateToKey(eStartAddress, eFirstDbDate), new TreeElement(endAddress, eLastDbDate, - eCountryCode)); + eCode)); eEndAddress = endAddress; }
@@ -260,10 +258,10 @@ public class DatabaseImporterImpl extends DatabaseImpl if (eStartAddress < startAddress && eEndAddress >= startAddress) { updateElements.put(convertAddressAndDateToKey(eStartAddress, eFirstDbDate), new TreeElement(startAddress - 1L, eLastDbDate, - eCountryCode)); + eCode)); updateElements.put(convertAddressAndDateToKey(startAddress, eFirstDbDate), new TreeElement(eEndAddress, eLastDbDate, - eCountryCode)); + eCode)); eStartAddress = startAddress; }
@@ -272,11 +270,11 @@ public class DatabaseImporterImpl extends DatabaseImpl nextStartAddress = eStartAddress; nextEndAddress = eEndAddress;
- /* If the range is already contained and has the same country code, - * mark it as updated. If it's contained with a different country - * code, ignore the update. */ + /* If the range is already contained and has the same code, mark it + * as updated. If it's contained with a different code, ignore the + * update. */ if (eFirstDbDate <= databaseDate && eLastDbDate >= databaseDate) { - if (eCountryCode.equals(countryCode)) { + if (eCode.equals(code)) { nextFirstDbDate = eFirstDbDate; nextLastDbDate = eLastDbDate; } else { @@ -288,7 +286,7 @@ public class DatabaseImporterImpl extends DatabaseImpl /* See if we can merge the new range with the previous or next * range. If so, extend our database range and mark the existing * element for deletion. */ - if (eCountryCode.equals(countryCode)) { + if (eCode.equals(code)) { if (eLastDbDate == previousDatabaseDate) { nextFirstDbDate = eFirstDbDate; updateElements.put(convertAddressAndDateToKey(eStartAddress, @@ -306,7 +304,7 @@ public class DatabaseImporterImpl extends DatabaseImpl while (nextEndAddress >= startAddress) { updateElements.put(convertAddressAndDateToKey(nextStartAddress, nextFirstDbDate), new TreeElement(nextEndAddress, - nextLastDbDate, countryCode)); + nextLastDbDate, code)); nextEndAddress = nextStartAddress - 1L; nextStartAddress = startAddress; nextFirstDbDate = databaseDate; @@ -355,7 +353,7 @@ public class DatabaseImporterImpl extends DatabaseImpl int eLastDbDate = e.getValue().lastDbDate; long eStartAddress = convertKeyToAddress(e.getKey()); long eEndAddress = e.getValue().endAddress; - String eCountryCode = e.getValue().countryCode; + String eCode = e.getValue().code; int start = eFirstDbDate, end = eFirstDbDate; for (int cur : this.databaseDates.tailSet(eFirstDbDate)) { if (cur > eLastDbDate) { @@ -364,8 +362,7 @@ public class DatabaseImporterImpl extends DatabaseImpl if (cur == addedDatabaseDate) { if (start >= 0 && end >= 0) { updateElements.put(convertAddressAndDateToKey(eStartAddress, - start), new TreeElement(eEndAddress, end, - eCountryCode)); + start), new TreeElement(eEndAddress, end, eCode)); start = end = -1; } } else if (start < 0) { @@ -376,7 +373,7 @@ public class DatabaseImporterImpl extends DatabaseImpl } if (start >= 0 && end >= 0) { updateElements.put(convertAddressAndDateToKey(eStartAddress, - start), new TreeElement(eEndAddress, end, eCountryCode)); + start), new TreeElement(eEndAddress, end, eCode)); } } } @@ -422,7 +419,7 @@ public class DatabaseImporterImpl extends DatabaseImpl bw.write(String.format("%s,%s,%s,%s,%s%n", convertKeyToAddressString(e.getKey()), convertAddressNumberToString(e.getValue().endAddress), - e.getValue().countryCode, + e.getValue().code, convertKeyToDateString(e.getKey()), convertDateNumberToString(e.getValue().lastDbDate))); } @@ -452,7 +449,7 @@ public class DatabaseImporterImpl extends DatabaseImpl sb.append(String.format("%n %s %s %s %s %s", convertKeyToAddressString(e.getKey()), convertAddressNumberToString(e.getValue().endAddress), - e.getValue().countryCode, + e.getValue().code, convertKeyToDateString(e.getKey()), convertDateNumberToString(e.getValue().lastDbDate))); if (--entries <= 0) { diff --git a/task-6471/java/src/org/torproject/task6471/DatabasePerformanceExample.java b/task-6471/java/src/org/torproject/task6471/DatabasePerformanceExample.java index 77808c8..5f8573a 100644 --- a/task-6471/java/src/org/torproject/task6471/DatabasePerformanceExample.java +++ b/task-6471/java/src/org/torproject/task6471/DatabasePerformanceExample.java @@ -75,11 +75,10 @@ public class DatabasePerformanceExample { if (testMonth.equals(dbMonth)) { String testAddressString = DatabaseImpl. convertKeyToAddressString(test); - String countryCode = - temp.lookupCountryCodeFromIpv4AddressAndDate( + String code = temp.lookupIpv4AddressAndDate( testAddressString, testDateString); - if (countryCode != null) { - results.put(test, countryCode); + if (code != null) { + results.put(test, code); } } } @@ -91,9 +90,9 @@ public class DatabasePerformanceExample { convertKeyToAddressString(e.getKey()); String testDateString = DatabaseImpl. convertKeyToDateString(e.getKey()); - String countryCode = e.getValue(); - bw.write(testAddressString + "," + testDateString + "," - + countryCode + "\n"); + String code = e.getValue(); + bw.write(testAddressString + "," + testDateString + "," + code + + "\n"); } bw.close(); long endMillis = System.currentTimeMillis(); @@ -124,7 +123,7 @@ public class DatabasePerformanceExample { } String expected = "??".equals(parts[2]) ? null : parts[2]; String result = - combinedDatabase.lookupCountryCodeFromIpv4AddressAndDate( + combinedDatabase.lookupIpv4AddressAndDate( testAddress, testDate); tests++; if ((expected == null && result != null) || @@ -165,7 +164,7 @@ public class DatabasePerformanceExample { String testDate = parts[1]; String expected = parts[2].equals("??") ? null : parts[2]; String result = - combinedDatabase.lookupCountryCodeFromIpv4AddressAndDate( + combinedDatabase.lookupIpv4AddressAndDate( testAddress, testDate); tests++; if ((expected == null && result != null) || diff --git a/task-6471/java/src/org/torproject/task6471/DatabaseTest.java b/task-6471/java/src/org/torproject/task6471/DatabaseTest.java index daba414..fb0f19d 100644 --- a/task-6471/java/src/org/torproject/task6471/DatabaseTest.java +++ b/task-6471/java/src/org/torproject/task6471/DatabaseTest.java @@ -15,48 +15,45 @@ public class DatabaseTest { database.addRange("20120901", "us", "3.0.0.0", 16777216); database.repairTree(); assertEquals(1, database.getNumberOfElements()); - assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals(null, database.lookupIpv4AddressAndDate( "2.255.255.255", "19920901")); - assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals(null, database.lookupIpv4AddressAndDate( "2.255.255.255", "20020901")); - assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals(null, database.lookupIpv4AddressAndDate( "2.255.255.255", "20120901")); - assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals(null, database.lookupIpv4AddressAndDate( "2.255.255.255", "20220901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.0.0.0", "19920901")); - assertEquals("United States", - database.lookupCountryNameFromIpv4AddressAndDate( - "3.0.0.0", "19920901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.0.0.0", "20020901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.0.0.0", "20120901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.0.0.0", "20220901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "19920901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20020901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20120901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20220901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.255.255.255", "19920901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.255.255.255", "20020901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.255.255.255", "20120901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.255.255.255", "20220901")); - assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals(null, database.lookupIpv4AddressAndDate( "4.0.0.0", "19920901")); - assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals(null, database.lookupIpv4AddressAndDate( "4.0.0.0", "20020901")); - assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals(null, database.lookupIpv4AddressAndDate( "4.0.0.0", "20120901")); - assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals(null, database.lookupIpv4AddressAndDate( "4.0.0.0", "20220901")); }
@@ -67,21 +64,21 @@ public class DatabaseTest { database.addRange("20120901", "ca", "4.0.0.0", 16777216); database.repairTree(); assertEquals(2, database.getNumberOfElements()); - assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals(null, database.lookupIpv4AddressAndDate( "2.255.255.255", "20120901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20120901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20120901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20120901")); - assertEquals("ca", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("ca", database.lookupIpv4AddressAndDate( "4.127.0.0", "20120901")); - assertEquals("ca", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("ca", database.lookupIpv4AddressAndDate( "4.127.0.0", "20120901")); - assertEquals("ca", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("ca", database.lookupIpv4AddressAndDate( "4.127.0.0", "20120901")); - assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals(null, database.lookupIpv4AddressAndDate( "5.0.0.0", "20120901")); }
@@ -92,15 +89,15 @@ public class DatabaseTest { database.addRange("20120901", "ca", "6.0.0.0", 16777216); database.repairTree(); assertEquals(2, database.getNumberOfElements()); - assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals(null, database.lookupIpv4AddressAndDate( "2.255.255.255", "20120901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20120901")); - assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals(null, database.lookupIpv4AddressAndDate( "4.255.255.255", "20120901")); - assertEquals("ca", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("ca", database.lookupIpv4AddressAndDate( "6.127.0.0", "20120901")); - assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals(null, database.lookupIpv4AddressAndDate( "7.0.0.0", "20120901")); }
@@ -111,22 +108,22 @@ public class DatabaseTest { database.addRange("20120901", "us", "3.0.0.0", 16777216); database.repairTree(); assertEquals(1, database.getNumberOfElements()); - assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals(null, database.lookupIpv4AddressAndDate( "2.255.255.255", "20120901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20120901")); - assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals(null, database.lookupIpv4AddressAndDate( "4.0.0.0", "20120901")); }
@Test() - public void testDuplicateImportDifferentCountryCode() { + public void testDuplicateImportDifferentCode() { DatabaseImporterImpl database = new DatabaseImporterImpl(); database.addRange("20120901", "us", "3.0.0.0", 16777216); database.addRange("20120901", "ca", "3.0.0.0", 16777216); database.repairTree(); assertEquals(1, database.getNumberOfElements()); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20120901")); }
@@ -138,13 +135,13 @@ public class DatabaseTest { database.addRange("20121001", "us", "3.0.0.0", 16777216); database.repairTree(); assertEquals(1, database.getNumberOfElements()); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20120801")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20120901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20121001")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20121101")); }
@@ -156,13 +153,13 @@ public class DatabaseTest { database.addRange("20120901", "us", "3.0.0.0", 16777216); database.repairTree(); assertEquals(1, database.getNumberOfElements()); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20120801")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20120901")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20121001")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20121101")); }
@@ -176,13 +173,13 @@ public class DatabaseTest { database.addRange("20121001", "us", "6.0.0.0", 16777216); database.repairTree(); assertEquals(3, database.getNumberOfElements()); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20120801")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20120901")); - assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals(null, database.lookupIpv4AddressAndDate( "3.127.0.0", "20121001")); - assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate( + assertEquals("us", database.lookupIpv4AddressAndDate( "3.127.0.0", "20121101")); } }
tor-commits@lists.torproject.org
-
karsten＠torproject.org