[tor-commits] [metrics-tasks/master] Export test cases to a file (#6471).
karsten at torproject.org
karsten at torproject.org
Thu Oct 25 23:06:23 UTC 2012
commit 04a718c38e44d0d47223fcffeec8e31a7308d3f8
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Thu Oct 25 12:35:29 2012 -0400
Export test cases to a file (#6471).
---
.../torproject/task6471/DatabaseImporterImpl.java | 5 +
.../task6471/DatabasePerformanceExample.java | 190 ++++++++++++--------
2 files changed, 116 insertions(+), 79 deletions(-)
diff --git a/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java b/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java
index 384b2d0..b562c29 100644
--- a/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java
+++ b/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java
@@ -123,6 +123,11 @@ public class DatabaseImporterImpl extends DatabaseImpl
void addRange(String databaseFileName, String countryCode,
String startAddressString, long addresses) {
+ if (countryCode.length() != 2) {
+ /* Don't import illegal range. */
+ return;
+ }
+
this.rangeImports++;
String databaseDateString =
databaseFileName.substring(databaseFileName.length() - 8);
diff --git a/task-6471/java/src/org/torproject/task6471/DatabasePerformanceExample.java b/task-6471/java/src/org/torproject/task6471/DatabasePerformanceExample.java
index 4338fad..77808c8 100644
--- a/task-6471/java/src/org/torproject/task6471/DatabasePerformanceExample.java
+++ b/task-6471/java/src/org/torproject/task6471/DatabasePerformanceExample.java
@@ -1,104 +1,132 @@
package org.torproject.task6471;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Random;
-import java.util.SortedMap;
import java.util.SortedSet;
import java.util.Stack;
-import java.util.TreeMap;
import java.util.TreeSet;
public class DatabasePerformanceExample {
- public static void main(String[] args) {
+ public static void main(String[] args) throws IOException {
- System.out.print("Generating test cases... ");
- long startMillis = System.currentTimeMillis();
- List<Long> tests = new ArrayList<Long>();
- SortedMap<Long, String> results = new TreeMap<Long, String>();
- Random rnd = new Random(1L);
- int startDate = DatabaseImpl.convertDateStringToNumber("20071001");
- int endDate = DatabaseImpl.convertDateStringToNumber("20120930");
- /* Skipping Dec 1--3, 2009, because the first available database from
- * December 2009 was published on the 4th, and generating test cases
- * was just too confusing when taking that into account. */
- List<Integer> skipDates = new ArrayList<Integer>();
- skipDates.add(DatabaseImpl.convertDateStringToNumber("20091201"));
- skipDates.add(DatabaseImpl.convertDateStringToNumber("20091202"));
- skipDates.add(DatabaseImpl.convertDateStringToNumber("20091203"));
- for (int i = 0; i < 100000; i++) {
- long testAddress = rnd.nextLong() & ((1L << 32) - 1L);
- int testDate = startDate + rnd.nextInt(endDate - startDate);
- if (skipDates.contains(testDate)) {
- i--;
- } else {
- tests.add((testAddress << 16) + testDate);
+ File testCasesCsvFile = new File("test-cases.csv");
+ if (!testCasesCsvFile.exists()) {
+ System.out.print("Generating test cases... ");
+ long startMillis = System.currentTimeMillis();
+ List<Long> tests = new ArrayList<Long>();
+ Random rnd = new Random(1L);
+ int startDate = DatabaseImpl.convertDateStringToNumber("20071001");
+ int endDate = DatabaseImpl.convertDateStringToNumber("20120930");
+ /* Skipping Dec 1--3, 2009, because the first available database
+ * from December 2009 was published on the 4th, and generating test
+ * cases was just too confusing when taking that into account. */
+ List<Integer> skipDates = new ArrayList<Integer>();
+ skipDates.add(DatabaseImpl.convertDateStringToNumber("20091201"));
+ skipDates.add(DatabaseImpl.convertDateStringToNumber("20091202"));
+ skipDates.add(DatabaseImpl.convertDateStringToNumber("20091203"));
+ for (int i = 0; i < 100000; i++) {
+ long testAddress = rnd.nextLong() & ((1L << 32) - 1L);
+ int testDate = startDate + rnd.nextInt(endDate - startDate);
+ if (skipDates.contains(testDate)) {
+ i--;
+ } else {
+ tests.add((testAddress << 16) + testDate);
+ }
}
- }
- Stack<File> stackedFiles = new Stack<File>();
- stackedFiles.add(new File("../data"));
- SortedSet<File> files = new TreeSet<File>();
- while (!stackedFiles.isEmpty()) {
- File file = stackedFiles.pop();
- if (file.isDirectory()) {
- stackedFiles.addAll(Arrays.asList(file.listFiles()));
- } else if (!file.getName().endsWith(".md5") &&
- !file.getName().endsWith(".md5.gz") &&
- !file.getName().endsWith(".asc") &&
- !file.getName().endsWith(".asc.gz")) {
- files.add(file);
+ Stack<File> stackedFiles = new Stack<File>();
+ stackedFiles.add(new File("../data"));
+ SortedSet<File> files = new TreeSet<File>();
+ while (!stackedFiles.isEmpty()) {
+ File file = stackedFiles.pop();
+ if (file.isDirectory()) {
+ stackedFiles.addAll(Arrays.asList(file.listFiles()));
+ } else if (!file.getName().endsWith(".md5") &&
+ !file.getName().endsWith(".md5.gz") &&
+ !file.getName().endsWith(".asc") &&
+ !file.getName().endsWith(".asc.gz")) {
+ files.add(file);
+ }
}
- }
- for (File file : files) {
- String dbMonth = file.getName().substring(
- file.getName().length() - 8);
- dbMonth = dbMonth.substring(0, 6);
- DatabaseImporter temp = new DatabaseImporterImpl();
- temp.importRegionalRegistryStatsFileOrDirectory(
- file.getAbsolutePath());
+ Map<Long, String> results = new HashMap<Long, String>();
for (long test : tests) {
- int testDate = (int) (test & ((1 << 16) - 1));
- String testMonth = DatabaseImpl.convertDateNumberToString(
- testDate).substring(0, 6);
- if (testMonth.equals(dbMonth)) {
- String testAddressString = DatabaseImpl.
- convertAddressNumberToString(test >> 16);
- String testDateString = DatabaseImpl.convertDateNumberToString(
- testDate);
- String countryCode =
- temp.lookupCountryCodeFromIpv4AddressAndDate(
- testAddressString, testDateString);
- if (countryCode != null) {
- results.put(test, countryCode);
+ results.put(test, "??");
+ }
+ for (File file : files) {
+ String dbMonth = file.getName().substring(
+ file.getName().length() - 8);
+ dbMonth = dbMonth.substring(0, 6);
+ DatabaseImporter temp = new DatabaseImporterImpl();
+ temp.importRegionalRegistryStatsFileOrDirectory(
+ file.getAbsolutePath());
+ for (long test : tests) {
+ String testDateString = DatabaseImpl.
+ convertKeyToDateString(test);
+ String testMonth = testDateString.substring(0, 6);
+ if (testMonth.equals(dbMonth)) {
+ String testAddressString = DatabaseImpl.
+ convertKeyToAddressString(test);
+ String countryCode =
+ temp.lookupCountryCodeFromIpv4AddressAndDate(
+ testAddressString, testDateString);
+ if (countryCode != null) {
+ results.put(test, countryCode);
+ }
}
}
}
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ testCasesCsvFile));
+ for (Map.Entry<Long, String> e : results.entrySet()) {
+ String testAddressString = DatabaseImpl.
+ convertKeyToAddressString(e.getKey());
+ String testDateString = DatabaseImpl.
+ convertKeyToDateString(e.getKey());
+ String countryCode = e.getValue();
+ bw.write(testAddressString + "," + testDateString + ","
+ + countryCode + "\n");
+ }
+ bw.close();
+ long endMillis = System.currentTimeMillis();
+ System.out.println((endMillis - startMillis) + " millis.");
}
- long endMillis = System.currentTimeMillis();
- System.out.println((endMillis - startMillis) + " millis.");
System.out.print("Importing files... ");
- startMillis = endMillis;
+ long startMillis = System.currentTimeMillis();
DatabaseImporter combinedDatabase = new DatabaseImporterImpl();
combinedDatabase.importRegionalRegistryStatsFileOrDirectory(
"../data");
- endMillis = System.currentTimeMillis();
+ long endMillis = System.currentTimeMillis();
System.out.println((endMillis - startMillis) + " millis.");
System.out.print("Making test requests... ");
startMillis = endMillis;
- int failures = 0;
- for (long test : tests) {
- String testAddress = DatabaseImpl.convertAddressNumberToString(
- test >> 16);
- String testDate = DatabaseImpl.convertDateNumberToString(
- (int) (test & ((1 << 16) - 1)));
- String expected = results.get(test);
+ BufferedReader br = new BufferedReader(new FileReader(
+ testCasesCsvFile));
+ String line;
+ int tests = 0, failures = 0;
+ while ((line = br.readLine()) != null) {
+ String[] parts = line.split(",");
+ String testAddress = parts[0];
+ String testDate = parts[1];
+ if (parts.length != 3) {
+ System.out.println(line);
+ System.exit(1);
+ }
+ String expected = "??".equals(parts[2]) ? null : parts[2];
String result =
combinedDatabase.lookupCountryCodeFromIpv4AddressAndDate(
testAddress, testDate);
+ tests++;
if ((expected == null && result != null) ||
(expected != null && !expected.equals(result))) {
//System.out.println("Expected " + expected + " for "
@@ -106,9 +134,10 @@ public class DatabasePerformanceExample {
failures++;
}
}
+ br.close();
endMillis = System.currentTimeMillis();
System.out.println((endMillis - startMillis) + " millis, " + failures
- + " out of " + tests.size() + " tests failed.");
+ + " out of " + tests + " tests failed.");
System.out.println(combinedDatabase);
@@ -128,15 +157,17 @@ public class DatabasePerformanceExample {
System.out.print("Making a second round of test requests... ");
startMillis = endMillis;
- failures = 0;
- for (long test : tests) {
- String testAddress = DatabaseImpl.convertAddressNumberToString(
- test >> 16);
- String testDate = DatabaseImpl.convertDateNumberToString(
- (int) (test & ((1 << 16) - 1)));
- String expected = results.get(test);
- String result = database.lookupCountryCodeFromIpv4AddressAndDate(
+ br = new BufferedReader(new FileReader(testCasesCsvFile));
+ tests = failures = 0;
+ while ((line = br.readLine()) != null) {
+ String[] parts = line.split(",");
+ String testAddress = parts[0];
+ String testDate = parts[1];
+ String expected = parts[2].equals("??") ? null : parts[2];
+ String result =
+ combinedDatabase.lookupCountryCodeFromIpv4AddressAndDate(
testAddress, testDate);
+ tests++;
if ((expected == null && result != null) ||
(expected != null && !expected.equals(result))) {
//System.out.println("Expected " + expected + " for "
@@ -144,8 +175,9 @@ public class DatabasePerformanceExample {
failures++;
}
}
+ br.close();
endMillis = System.currentTimeMillis();
System.out.println((endMillis - startMillis) + " millis, " + failures
- + " out of " + tests.size() + " tests failed.");
+ + " out of " + tests + " tests failed.");
}
}
More information about the tor-commits
mailing list