commit 04a718c38e44d0d47223fcffeec8e31a7308d3f8 Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Thu Oct 25 12:35:29 2012 -0400 Export test cases to a file (#6471). --- .../torproject/task6471/DatabaseImporterImpl.java | 5 + .../task6471/DatabasePerformanceExample.java | 190 ++++++++++++-------- 2 files changed, 116 insertions(+), 79 deletions(-) diff --git a/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java b/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java index 384b2d0..b562c29 100644 --- a/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java +++ b/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java @@ -123,6 +123,11 @@ public class DatabaseImporterImpl extends DatabaseImpl void addRange(String databaseFileName, String countryCode, String startAddressString, long addresses) { + if (countryCode.length() != 2) { + /* Don't import illegal range. */ + return; + } + this.rangeImports++; String databaseDateString = databaseFileName.substring(databaseFileName.length() - 8); diff --git a/task-6471/java/src/org/torproject/task6471/DatabasePerformanceExample.java b/task-6471/java/src/org/torproject/task6471/DatabasePerformanceExample.java index 4338fad..77808c8 100644 --- a/task-6471/java/src/org/torproject/task6471/DatabasePerformanceExample.java +++ b/task-6471/java/src/org/torproject/task6471/DatabasePerformanceExample.java @@ -1,104 +1,132 @@ package org.torproject.task6471; +import java.io.BufferedReader; +import java.io.BufferedWriter; import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Random; -import java.util.SortedMap; import java.util.SortedSet; import java.util.Stack; -import java.util.TreeMap; import java.util.TreeSet; public class DatabasePerformanceExample { - public static void main(String[] args) { + public static void main(String[] args) throws IOException { - System.out.print("Generating test cases... "); - long startMillis = System.currentTimeMillis(); - List<Long> tests = new ArrayList<Long>(); - SortedMap<Long, String> results = new TreeMap<Long, String>(); - Random rnd = new Random(1L); - int startDate = DatabaseImpl.convertDateStringToNumber("20071001"); - int endDate = DatabaseImpl.convertDateStringToNumber("20120930"); - /* Skipping Dec 1--3, 2009, because the first available database from - * December 2009 was published on the 4th, and generating test cases - * was just too confusing when taking that into account. */ - List<Integer> skipDates = new ArrayList<Integer>(); - skipDates.add(DatabaseImpl.convertDateStringToNumber("20091201")); - skipDates.add(DatabaseImpl.convertDateStringToNumber("20091202")); - skipDates.add(DatabaseImpl.convertDateStringToNumber("20091203")); - for (int i = 0; i < 100000; i++) { - long testAddress = rnd.nextLong() & ((1L << 32) - 1L); - int testDate = startDate + rnd.nextInt(endDate - startDate); - if (skipDates.contains(testDate)) { - i--; - } else { - tests.add((testAddress << 16) + testDate); + File testCasesCsvFile = new File("test-cases.csv"); + if (!testCasesCsvFile.exists()) { + System.out.print("Generating test cases... "); + long startMillis = System.currentTimeMillis(); + List<Long> tests = new ArrayList<Long>(); + Random rnd = new Random(1L); + int startDate = DatabaseImpl.convertDateStringToNumber("20071001"); + int endDate = DatabaseImpl.convertDateStringToNumber("20120930"); + /* Skipping Dec 1--3, 2009, because the first available database + * from December 2009 was published on the 4th, and generating test + * cases was just too confusing when taking that into account. */ + List<Integer> skipDates = new ArrayList<Integer>(); + skipDates.add(DatabaseImpl.convertDateStringToNumber("20091201")); + skipDates.add(DatabaseImpl.convertDateStringToNumber("20091202")); + skipDates.add(DatabaseImpl.convertDateStringToNumber("20091203")); + for (int i = 0; i < 100000; i++) { + long testAddress = rnd.nextLong() & ((1L << 32) - 1L); + int testDate = startDate + rnd.nextInt(endDate - startDate); + if (skipDates.contains(testDate)) { + i--; + } else { + tests.add((testAddress << 16) + testDate); + } } - } - Stack<File> stackedFiles = new Stack<File>(); - stackedFiles.add(new File("../data")); - SortedSet<File> files = new TreeSet<File>(); - while (!stackedFiles.isEmpty()) { - File file = stackedFiles.pop(); - if (file.isDirectory()) { - stackedFiles.addAll(Arrays.asList(file.listFiles())); - } else if (!file.getName().endsWith(".md5") && - !file.getName().endsWith(".md5.gz") && - !file.getName().endsWith(".asc") && - !file.getName().endsWith(".asc.gz")) { - files.add(file); + Stack<File> stackedFiles = new Stack<File>(); + stackedFiles.add(new File("../data")); + SortedSet<File> files = new TreeSet<File>(); + while (!stackedFiles.isEmpty()) { + File file = stackedFiles.pop(); + if (file.isDirectory()) { + stackedFiles.addAll(Arrays.asList(file.listFiles())); + } else if (!file.getName().endsWith(".md5") && + !file.getName().endsWith(".md5.gz") && + !file.getName().endsWith(".asc") && + !file.getName().endsWith(".asc.gz")) { + files.add(file); + } } - } - for (File file : files) { - String dbMonth = file.getName().substring( - file.getName().length() - 8); - dbMonth = dbMonth.substring(0, 6); - DatabaseImporter temp = new DatabaseImporterImpl(); - temp.importRegionalRegistryStatsFileOrDirectory( - file.getAbsolutePath()); + Map<Long, String> results = new HashMap<Long, String>(); for (long test : tests) { - int testDate = (int) (test & ((1 << 16) - 1)); - String testMonth = DatabaseImpl.convertDateNumberToString( - testDate).substring(0, 6); - if (testMonth.equals(dbMonth)) { - String testAddressString = DatabaseImpl. - convertAddressNumberToString(test >> 16); - String testDateString = DatabaseImpl.convertDateNumberToString( - testDate); - String countryCode = - temp.lookupCountryCodeFromIpv4AddressAndDate( - testAddressString, testDateString); - if (countryCode != null) { - results.put(test, countryCode); + results.put(test, "??"); + } + for (File file : files) { + String dbMonth = file.getName().substring( + file.getName().length() - 8); + dbMonth = dbMonth.substring(0, 6); + DatabaseImporter temp = new DatabaseImporterImpl(); + temp.importRegionalRegistryStatsFileOrDirectory( + file.getAbsolutePath()); + for (long test : tests) { + String testDateString = DatabaseImpl. + convertKeyToDateString(test); + String testMonth = testDateString.substring(0, 6); + if (testMonth.equals(dbMonth)) { + String testAddressString = DatabaseImpl. + convertKeyToAddressString(test); + String countryCode = + temp.lookupCountryCodeFromIpv4AddressAndDate( + testAddressString, testDateString); + if (countryCode != null) { + results.put(test, countryCode); + } } } } + BufferedWriter bw = new BufferedWriter(new FileWriter( + testCasesCsvFile)); + for (Map.Entry<Long, String> e : results.entrySet()) { + String testAddressString = DatabaseImpl. + convertKeyToAddressString(e.getKey()); + String testDateString = DatabaseImpl. + convertKeyToDateString(e.getKey()); + String countryCode = e.getValue(); + bw.write(testAddressString + "," + testDateString + "," + + countryCode + "\n"); + } + bw.close(); + long endMillis = System.currentTimeMillis(); + System.out.println((endMillis - startMillis) + " millis."); } - long endMillis = System.currentTimeMillis(); - System.out.println((endMillis - startMillis) + " millis."); System.out.print("Importing files... "); - startMillis = endMillis; + long startMillis = System.currentTimeMillis(); DatabaseImporter combinedDatabase = new DatabaseImporterImpl(); combinedDatabase.importRegionalRegistryStatsFileOrDirectory( "../data"); - endMillis = System.currentTimeMillis(); + long endMillis = System.currentTimeMillis(); System.out.println((endMillis - startMillis) + " millis."); System.out.print("Making test requests... "); startMillis = endMillis; - int failures = 0; - for (long test : tests) { - String testAddress = DatabaseImpl.convertAddressNumberToString( - test >> 16); - String testDate = DatabaseImpl.convertDateNumberToString( - (int) (test & ((1 << 16) - 1))); - String expected = results.get(test); + BufferedReader br = new BufferedReader(new FileReader( + testCasesCsvFile)); + String line; + int tests = 0, failures = 0; + while ((line = br.readLine()) != null) { + String[] parts = line.split(","); + String testAddress = parts[0]; + String testDate = parts[1]; + if (parts.length != 3) { + System.out.println(line); + System.exit(1); + } + String expected = "??".equals(parts[2]) ? null : parts[2]; String result = combinedDatabase.lookupCountryCodeFromIpv4AddressAndDate( testAddress, testDate); + tests++; if ((expected == null && result != null) || (expected != null && !expected.equals(result))) { //System.out.println("Expected " + expected + " for " @@ -106,9 +134,10 @@ public class DatabasePerformanceExample { failures++; } } + br.close(); endMillis = System.currentTimeMillis(); System.out.println((endMillis - startMillis) + " millis, " + failures - + " out of " + tests.size() + " tests failed."); + + " out of " + tests + " tests failed."); System.out.println(combinedDatabase); @@ -128,15 +157,17 @@ public class DatabasePerformanceExample { System.out.print("Making a second round of test requests... "); startMillis = endMillis; - failures = 0; - for (long test : tests) { - String testAddress = DatabaseImpl.convertAddressNumberToString( - test >> 16); - String testDate = DatabaseImpl.convertDateNumberToString( - (int) (test & ((1 << 16) - 1))); - String expected = results.get(test); - String result = database.lookupCountryCodeFromIpv4AddressAndDate( + br = new BufferedReader(new FileReader(testCasesCsvFile)); + tests = failures = 0; + while ((line = br.readLine()) != null) { + String[] parts = line.split(","); + String testAddress = parts[0]; + String testDate = parts[1]; + String expected = parts[2].equals("??") ? null : parts[2]; + String result = + combinedDatabase.lookupCountryCodeFromIpv4AddressAndDate( testAddress, testDate); + tests++; if ((expected == null && result != null) || (expected != null && !expected.equals(result))) { //System.out.println("Expected " + expected + " for " @@ -144,8 +175,9 @@ public class DatabasePerformanceExample { failures++; } } + br.close(); endMillis = System.currentTimeMillis(); System.out.println((endMillis - startMillis) + " millis, " + failures - + " out of " + tests.size() + " tests failed."); + + " out of " + tests + " tests failed."); } }