November 2013 - tor-commits - lists.torproject.org

[onionoo/master] Simplify tests by getting rid of test helpers.
by karsten＠torproject.org 28 Nov '13

28 Nov '13

commit 91c2cba598d0a359b9a97ac784330403a396219d Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Nov 27 19:12:24 2013 +0100 Simplify tests by getting rid of test helpers. --- src/org/torproject/onionoo/ResponseBuilder.java | 4 - test/org/torproject/onionoo/LookupServiceTest.java | 392 ++++++----- .../torproject/onionoo/ResourceServletTest.java | 702 ++++++++++---------- 3 files changed, 533 insertions(+), 565 deletions(-) diff --git a/src/org/torproject/onionoo/ResponseBuilder.java b/src/org/torproject/onionoo/ResponseBuilder.java index 5cc1e04..b79623e 100644 --- a/src/org/torproject/onionoo/ResponseBuilder.java +++ b/src/org/torproject/onionoo/ResponseBuilder.java @@ -337,8 +337,6 @@ public class ResponseBuilder { relayFingerprintSummaryLines); Map<String, String> filteredBridges = new HashMap<String, String>( bridgeFingerprintSummaryLines); - System.out.println(filteredRelays.size() + " relays left, " - + filteredBridges.size() + " bridges left."); filterByType(filteredRelays, filteredBridges); filterByRunning(filteredRelays, filteredBridges); filterBySearchTerms(filteredRelays, filteredBridges); @@ -349,8 +347,6 @@ public class ResponseBuilder { filterNodesByFirstSeenDays(filteredRelays, filteredBridges); filterNodesByLastSeenDays(filteredRelays, filteredBridges); filterByContact(filteredRelays, filteredBridges); - System.out.println(filteredRelays.size() + " relays left, " - + filteredBridges.size() + " bridges left."); /* Re-order and limit results. */ List<String> orderedRelays = new ArrayList<String>(); diff --git a/test/org/torproject/onionoo/LookupServiceTest.java b/test/org/torproject/onionoo/LookupServiceTest.java index 42df61f..4834092 100644 --- a/test/org/torproject/onionoo/LookupServiceTest.java +++ b/test/org/torproject/onionoo/LookupServiceTest.java @@ -26,195 +26,181 @@ import org.torproject.onionoo.LookupService.LookupResult; public class LookupServiceTest { - private static class LookupServiceTestHelper { + private List<String> manualGeoLiteCityBlocksLines, + automaticGeoLiteCityBlocksLines, geoLiteCityBlocksLines, + geoLiteCityLocationLines, iso3166Lines, regionLines, + geoipASNum2Lines; - private File tempGeoipDir; + private LookupService lookupService; - private List<String> manualGeoLiteCityBlocksLines, - automaticGeoLiteCityBlocksLines, geoLiteCityBlocksLines, - geoLiteCityLocationLines, iso3166Lines, regionLines, - geoipASNum2Lines; + private SortedSet<String> addressStrings = new TreeSet<String>(); - private LookupService lookupService; + private SortedMap<String, LookupResult> lookupResults; - private SortedSet<String> addressStrings; - - private SortedMap<String, LookupResult> lookupResults; - - private LookupServiceTestHelper(File tempGeoipDir, - SortedSet<String> addressStrings) { - this.tempGeoipDir = tempGeoipDir; - this.addressStrings = addressStrings; - } + private void populateLines() { + this.manualGeoLiteCityBlocksLines = new ArrayList<String>(); + this.manualGeoLiteCityBlocksLines.add( + "Copyright (c) 2011 MaxMind Inc. All Rights Reserved."); + this.manualGeoLiteCityBlocksLines.add("startIpNum,endIpNum,locId"); + this.manualGeoLiteCityBlocksLines.add("\"134739200\",\"134744063\"," + + "\"223\""); + this.manualGeoLiteCityBlocksLines.add("\"134744064\",\"134744319\"," + + "\"32191\""); + this.manualGeoLiteCityBlocksLines.add("\"134744320\",\"134751743\"," + + "\"223\""); + this.geoLiteCityLocationLines = new ArrayList<String>(); + this.geoLiteCityLocationLines.add("Copyright (c) 2012 MaxMind " + + "LLC. All Rights Reserved."); + this.geoLiteCityLocationLines.add("locId,country,region,city," + + "postalCode,latitude,longitude,metroCode,areaCode"); + this.geoLiteCityLocationLines.add("223,\"US\",\"\",\"\",\"\"," + + "38.0000,-97.0000,,"); + this.geoLiteCityLocationLines.add("32191,\"US\",\"CA\"," + + "\"Mountain View\",\"\",37.3860,-122.0838,807,650"); + this.iso3166Lines = new ArrayList<String>(); + this.iso3166Lines.add("US,\"United States\""); + this.regionLines = new ArrayList<String>(); + this.regionLines.add("US,CA,\"California\""); + this.geoipASNum2Lines = new ArrayList<String>(); + this.geoipASNum2Lines.add("134743296,134744063,\"AS3356 Level 3 " + + "Communications\""); + this.geoipASNum2Lines.add("134744064,134744319,\"AS15169 Google " + + "Inc.\""); + this.geoipASNum2Lines.add("134744320,134750463,\"AS3356 Level 3 " + + "Communications\""); + } - private void populateLines() { - this.manualGeoLiteCityBlocksLines = new ArrayList<String>(); - this.manualGeoLiteCityBlocksLines.add( - "Copyright (c) 2011 MaxMind Inc. All Rights Reserved."); - this.manualGeoLiteCityBlocksLines.add("startIpNum,endIpNum,locId"); - this.manualGeoLiteCityBlocksLines.add("\"134739200\",\"134744063\"," - + "\"223\""); - this.manualGeoLiteCityBlocksLines.add("\"134744064\",\"134744319\"," - + "\"32191\""); - this.manualGeoLiteCityBlocksLines.add("\"134744320\",\"134751743\"," - + "\"223\""); - this.geoLiteCityLocationLines = new ArrayList<String>(); - this.geoLiteCityLocationLines.add("Copyright (c) 2012 MaxMind " - + "LLC. All Rights Reserved."); - this.geoLiteCityLocationLines.add("locId,country,region,city," - + "postalCode,latitude,longitude,metroCode,areaCode"); - this.geoLiteCityLocationLines.add("223,\"US\",\"\",\"\",\"\"," - + "38.0000,-97.0000,,"); - this.geoLiteCityLocationLines.add("32191,\"US\",\"CA\"," - + "\"Mountain View\",\"\",37.3860,-122.0838,807,650"); - this.iso3166Lines = new ArrayList<String>(); - this.iso3166Lines.add("US,\"United States\""); - this.regionLines = new ArrayList<String>(); - this.regionLines.add("US,CA,\"California\""); - this.geoipASNum2Lines = new ArrayList<String>(); - this.geoipASNum2Lines.add("134743296,134744063,\"AS3356 Level 3 " - + "Communications\""); - this.geoipASNum2Lines.add("134744064,134744319,\"AS15169 Google " - + "Inc.\""); - this.geoipASNum2Lines.add("134744320,134750463,\"AS3356 Level 3 " - + "Communications\""); + private void writeCsvFiles() { + try { + this.writeCsvFile(this.manualGeoLiteCityBlocksLines, + "Manual-GeoLiteCity-Blocks.csv"); + this.writeCsvFile(this.automaticGeoLiteCityBlocksLines, + "Automatic-GeoLiteCity-Blocks.csv"); + this.writeCsvFile(this.geoLiteCityBlocksLines, + "GeoLiteCity-Blocks.csv"); + this.writeCsvFile(this.geoLiteCityLocationLines, + "GeoLiteCity-Location.csv"); + this.writeCsvFile(this.iso3166Lines, "iso3166.csv"); + this.writeCsvFile(this.regionLines, "region.csv"); + this.writeCsvFile(this.geoipASNum2Lines, "GeoIPASNum2.csv"); + } catch (IOException e) { + throw new RuntimeException(e); } + } - private void writeCsvFiles() { - try { - this.writeCsvFile(this.manualGeoLiteCityBlocksLines, - "Manual-GeoLiteCity-Blocks.csv"); - this.writeCsvFile(this.automaticGeoLiteCityBlocksLines, - "Automatic-GeoLiteCity-Blocks.csv"); - this.writeCsvFile(this.geoLiteCityBlocksLines, - "GeoLiteCity-Blocks.csv"); - this.writeCsvFile(this.geoLiteCityLocationLines, - "GeoLiteCity-Location.csv"); - this.writeCsvFile(this.iso3166Lines, "iso3166.csv"); - this.writeCsvFile(this.regionLines, "region.csv"); - this.writeCsvFile(this.geoipASNum2Lines, "GeoIPASNum2.csv"); - } catch (IOException e) { - throw new RuntimeException(e); + private void writeCsvFile(List<String> lines, String fileName) + throws IOException { + if (lines != null && !lines.isEmpty()) { + BufferedWriter bw = new BufferedWriter(new FileWriter( + new File(this.tempGeoipDir, fileName))); + for (String line : lines) { + bw.write(line + "\n"); } + bw.close(); } + } - private void writeCsvFile(List<String> lines, String fileName) - throws IOException { - if (lines != null && !lines.isEmpty()) { - BufferedWriter bw = new BufferedWriter(new FileWriter( - new File(this.tempGeoipDir, fileName))); - for (String line : lines) { - bw.write(line + "\n"); - } - bw.close(); - } - } + private void performLookups() { + this.lookupService = new LookupService(this.tempGeoipDir); + this.lookupResults = this.lookupService.lookup(this.addressStrings); + } - private void performLookups() { - this.lookupService = new LookupService(this.tempGeoipDir); - this.lookupResults = this.lookupService.lookup(this.addressStrings); + private void assertLookupResult( + List<String> manualGeoLiteCityBlocksLines, + List<String> automaticGeoLiteCityBlocksLines, + List<String> geoLiteCityBlocksLines, + List<String> geoLiteCityLocationLines, List<String> iso3166Lines, + List<String> regionLines, List<String> geoipASNum2Lines, + String addressString, String countryCode, String countryName, + String regionName, String cityName, String latitude, + String longitude, String aSNumber, String aSName) { + this.addressStrings.add(addressString); + this.populateLines(); + if (manualGeoLiteCityBlocksLines != null) { + this.manualGeoLiteCityBlocksLines = + manualGeoLiteCityBlocksLines; } - - private static void assertLookupResult(File tempGeoipDir, - List<String> manualGeoLiteCityBlocksLines, - List<String> automaticGeoLiteCityBlocksLines, - List<String> geoLiteCityBlocksLines, - List<String> geoLiteCityLocationLines, List<String> iso3166Lines, - List<String> regionLines, List<String> geoipASNum2Lines, - String addressString, String countryCode, String countryName, - String regionName, String cityName, String latitude, - String longitude, String aSNumber, String aSName) { - SortedSet<String> addressStrings = new TreeSet<String>(); - addressStrings.add(addressString); - LookupServiceTestHelper helper = new LookupServiceTestHelper( - tempGeoipDir, addressStrings); - helper.populateLines(); - if (manualGeoLiteCityBlocksLines != null) { - helper.manualGeoLiteCityBlocksLines = - manualGeoLiteCityBlocksLines; - } - if (automaticGeoLiteCityBlocksLines != null) { - helper.automaticGeoLiteCityBlocksLines = - automaticGeoLiteCityBlocksLines; - } - if (geoLiteCityBlocksLines != null) { - helper.geoLiteCityBlocksLines = geoLiteCityBlocksLines; - } - if (geoLiteCityLocationLines != null) { - helper.geoLiteCityLocationLines = geoLiteCityLocationLines; - } - if (iso3166Lines != null) { - helper.iso3166Lines = iso3166Lines; - } - if (regionLines != null) { - helper.regionLines = regionLines; - } - if (geoipASNum2Lines != null) { - helper.geoipASNum2Lines = geoipASNum2Lines; - } - helper.writeCsvFiles(); - /* Disable log messages printed to System.err. */ - System.setErr(new PrintStream(new OutputStream() { - public void write(int b) { - } - })); - helper.performLookups(); - if (countryCode == null) { - assertTrue(!helper.lookupResults.containsKey(addressString) || - helper.lookupResults.get(addressString).countryCode == null); - } else { - assertEquals(countryCode, - helper.lookupResults.get(addressString).countryCode); - } - if (countryName == null) { - assertTrue(!helper.lookupResults.containsKey(addressString) || - helper.lookupResults.get(addressString).countryName == null); - } else { - assertEquals(countryName, - helper.lookupResults.get(addressString).countryName); - } - if (regionName == null) { - assertTrue(!helper.lookupResults.containsKey(addressString) || - helper.lookupResults.get(addressString).regionName == null); - } else { - assertEquals(regionName, - helper.lookupResults.get(addressString).regionName); - } - if (cityName == null) { - assertTrue(!helper.lookupResults.containsKey(addressString) || - helper.lookupResults.get(addressString).cityName == null); - } else { - assertEquals(cityName, - helper.lookupResults.get(addressString).cityName); - } - if (latitude == null) { - assertTrue(!helper.lookupResults.containsKey(addressString) || - helper.lookupResults.get(addressString).latitude == null); - } else { - assertEquals(latitude, - helper.lookupResults.get(addressString).latitude); - } - if (longitude == null) { - assertTrue(!helper.lookupResults.containsKey(addressString) || - helper.lookupResults.get(addressString).longitude == null); - } else { - assertEquals(longitude, - helper.lookupResults.get(addressString).longitude); - } - if (aSNumber == null) { - assertTrue(!helper.lookupResults.containsKey(addressString) || - helper.lookupResults.get(addressString).aSNumber == null); - } else { - assertEquals(aSNumber, - helper.lookupResults.get(addressString).aSNumber); - } - if (aSName == null) { - assertTrue(!helper.lookupResults.containsKey(addressString) || - helper.lookupResults.get(addressString).aSName == null); - } else { - assertEquals(aSName, - helper.lookupResults.get(addressString).aSName); + if (automaticGeoLiteCityBlocksLines != null) { + this.automaticGeoLiteCityBlocksLines = + automaticGeoLiteCityBlocksLines; + } + if (geoLiteCityBlocksLines != null) { + this.geoLiteCityBlocksLines = geoLiteCityBlocksLines; + } + if (geoLiteCityLocationLines != null) { + this.geoLiteCityLocationLines = geoLiteCityLocationLines; + } + if (iso3166Lines != null) { + this.iso3166Lines = iso3166Lines; + } + if (regionLines != null) { + this.regionLines = regionLines; + } + if (geoipASNum2Lines != null) { + this.geoipASNum2Lines = geoipASNum2Lines; + } + this.writeCsvFiles(); + /* Disable log messages printed to System.err. */ + System.setErr(new PrintStream(new OutputStream() { + public void write(int b) { } + })); + this.performLookups(); + if (countryCode == null) { + assertTrue(!this.lookupResults.containsKey(addressString) || + this.lookupResults.get(addressString).countryCode == null); + } else { + assertEquals(countryCode, + this.lookupResults.get(addressString).countryCode); + } + if (countryName == null) { + assertTrue(!this.lookupResults.containsKey(addressString) || + this.lookupResults.get(addressString).countryName == null); + } else { + assertEquals(countryName, + this.lookupResults.get(addressString).countryName); + } + if (regionName == null) { + assertTrue(!this.lookupResults.containsKey(addressString) || + this.lookupResults.get(addressString).regionName == null); + } else { + assertEquals(regionName, + this.lookupResults.get(addressString).regionName); + } + if (cityName == null) { + assertTrue(!this.lookupResults.containsKey(addressString) || + this.lookupResults.get(addressString).cityName == null); + } else { + assertEquals(cityName, + this.lookupResults.get(addressString).cityName); + } + if (latitude == null) { + assertTrue(!this.lookupResults.containsKey(addressString) || + this.lookupResults.get(addressString).latitude == null); + } else { + assertEquals(latitude, + this.lookupResults.get(addressString).latitude); + } + if (longitude == null) { + assertTrue(!this.lookupResults.containsKey(addressString) || + this.lookupResults.get(addressString).longitude == null); + } else { + assertEquals(longitude, + this.lookupResults.get(addressString).longitude); + } + if (aSNumber == null) { + assertTrue(!this.lookupResults.containsKey(addressString) || + this.lookupResults.get(addressString).aSNumber == null); + } else { + assertEquals(aSNumber, + this.lookupResults.get(addressString).aSNumber); + } + if (aSName == null) { + assertTrue(!this.lookupResults.containsKey(addressString) || + this.lookupResults.get(addressString).aSName == null); + } else { + assertEquals(aSName, + this.lookupResults.get(addressString).aSName); } } @@ -230,7 +216,7 @@ public class LookupServiceTest { @Test() public void testLookup8888() { - LookupServiceTestHelper.assertLookupResult(this.tempGeoipDir, null, + this.assertLookupResult(null, null, null, null, null, null, null, "8.8.8.8", "us", "United States", "California", "Mountain View", "37.3860", "-122.0838", "AS15169", "Google Inc."); @@ -238,7 +224,7 @@ public class LookupServiceTest { @Test() public void testLookup8880() { - LookupServiceTestHelper.assertLookupResult(this.tempGeoipDir, null, + this.assertLookupResult(null, null, null, null, null, null, null, "8.8.8.0", "us", "United States", "California", "Mountain View", "37.3860", "-122.0838", "AS15169", "Google Inc."); @@ -246,7 +232,7 @@ public class LookupServiceTest { @Test() public void testLookup888255() { - LookupServiceTestHelper.assertLookupResult(this.tempGeoipDir, null, + this.assertLookupResult(null, null, null, null, null, null, null, "8.8.8.255", "us", "United States", "California", "Mountain View", "37.3860", "-122.0838", "AS15169", "Google Inc."); @@ -254,56 +240,56 @@ public class LookupServiceTest { @Test() public void testLookup888256() { - LookupServiceTestHelper.assertLookupResult(this.tempGeoipDir, null, + this.assertLookupResult(null, null, null, null, null, null, null, "8.8.8.256", null, null, null, null, null, null, null, null); } @Test() public void testLookup888Minus1() { - LookupServiceTestHelper.assertLookupResult(this.tempGeoipDir, null, + this.assertLookupResult(null, null, null, null, null, null, null, "8.8.8.-1", null, null, null, null, null, null, null, null); } @Test() public void testLookup000() { - LookupServiceTestHelper.assertLookupResult(this.tempGeoipDir, null, + this.assertLookupResult(null, null, null, null, null, null, null, "0.0.0.0", null, null, null, null, null, null, null, null); } @Test() public void testLookupNoBlocksLines() { - LookupServiceTestHelper.assertLookupResult(this.tempGeoipDir, + this.assertLookupResult( new ArrayList<String>(), null, null, null, null, null, null, "8.8.8.8", null, null, null, null, null, null, null, null); } @Test() public void testLookupNoLocationLines() { - LookupServiceTestHelper.assertLookupResult(this.tempGeoipDir, null, + this.assertLookupResult(null, null, null, new ArrayList<String>(), null, null, null, "8.8.8.8", null, null, null, null, null, null, null, null); } @Test() public void testLookupNoIso3166Lines() { - LookupServiceTestHelper.assertLookupResult(this.tempGeoipDir, null, + this.assertLookupResult(null, null, null, null, new ArrayList<String>(), null, null, "8.8.8.8", null, null, null, null, null, null, null, null); } @Test() public void testLookupNoRegionLines() { - LookupServiceTestHelper.assertLookupResult(this.tempGeoipDir, null, + this.assertLookupResult(null, null, null, null, null, new ArrayList<String>(), null, "8.8.8.8", null, null, null, null, null, null, null, null); } @Test() public void testLookupNoGeoipASNum2Lines() { - LookupServiceTestHelper.assertLookupResult(this.tempGeoipDir, null, + this.assertLookupResult(null, null, null, null, null, null, new ArrayList<String>(), "8.8.8.8", null, null, null, null, null, null, null, null); } @@ -317,7 +303,7 @@ public class LookupServiceTest { + "latitude,longitude,metroCode,areaCode"); geoLiteCityLocationLines.add("223,\"US\",\"\",\"\",\"\",38.0000," + "-97.0000,,"); - LookupServiceTestHelper.assertLookupResult(this.tempGeoipDir, null, + this.assertLookupResult(null, null, null, geoLiteCityLocationLines, null, null, null, "8.8.8.8", null, null, null, null, null, null, "AS15169", "Google Inc."); } @@ -326,7 +312,7 @@ public class LookupServiceTest { public void testLookupNoCorrespondingCountryName() { List<String> iso3166Lines = new ArrayList<String>(); iso3166Lines.add("UY,\"Uruguay\""); - LookupServiceTestHelper.assertLookupResult(this.tempGeoipDir, null, + this.assertLookupResult(null, null, null, null, iso3166Lines, null, null, "8.8.8.8", "us", null, "California", "Mountain View", "37.3860", "-122.0838", "AS15169", "Google Inc."); @@ -336,7 +322,7 @@ public class LookupServiceTest { public void testLookupNoCorrespondingRegionName() { List<String> regionLines = new ArrayList<String>(); regionLines.add("US,CO,\"Colorado\""); - LookupServiceTestHelper.assertLookupResult(this.tempGeoipDir, null, + this.assertLookupResult(null, null, null, null, null, regionLines, null, "8.8.8.8", "us", "United States", null, "Mountain View", "37.3860", "-122.0838", "AS15169", "Google Inc."); @@ -354,7 +340,7 @@ public class LookupServiceTest { + "\"32191\""); manualGeoLiteCityBlocksLines.add("\"134744320\",\"134751743\"," + "\"223\""); - LookupServiceTestHelper.assertLookupResult(this.tempGeoipDir, + this.assertLookupResult( manualGeoLiteCityBlocksLines, null, null, null, null, null, null, "8.8.8.8", null, null, null, null, null, null, "AS15169", "Google Inc."); @@ -368,7 +354,7 @@ public class LookupServiceTest { manualGeoLiteCityBlocksLines.add("startIpNum,endIpNum,locId"); manualGeoLiteCityBlocksLines.add("\"one\",\"134744319\"," + "\"32191\""); - LookupServiceTestHelper.assertLookupResult(this.tempGeoipDir, + this.assertLookupResult( manualGeoLiteCityBlocksLines, null, null, null, null, null, null, "8.8.8.8", null, null, null, null, null, null, null, null); } @@ -381,7 +367,7 @@ public class LookupServiceTest { manualGeoLiteCityBlocksLines.add("startIpNum,endIpNum,locId"); manualGeoLiteCityBlocksLines.add("\"1" + String.valueOf(Long.MAX_VALUE) + "\",\"134744319\",\"32191\""); - LookupServiceTestHelper.assertLookupResult(this.tempGeoipDir, + this.assertLookupResult( manualGeoLiteCityBlocksLines, null, null, null, null, null, null, "8.8.8.8", null, null, null, null, null, null, null, null); } @@ -393,7 +379,7 @@ public class LookupServiceTest { + "All Rights Reserved."); manualGeoLiteCityBlocksLines.add("startIpNum,endIpNum,locId"); manualGeoLiteCityBlocksLines.add("\"134744064\",\"134744319\",\"X\""); - LookupServiceTestHelper.assertLookupResult(this.tempGeoipDir, + this.assertLookupResult( manualGeoLiteCityBlocksLines, null, null, null, null, null, null, "8.8.8.8", null, null, null, null, null, null, null, null); } @@ -405,7 +391,7 @@ public class LookupServiceTest { + "All Rights Reserved."); manualGeoLiteCityBlocksLines.add("startIpNum,endIpNum,locId"); manualGeoLiteCityBlocksLines.add("\"134744064\",\"134744319\""); - LookupServiceTestHelper.assertLookupResult(this.tempGeoipDir, + this.assertLookupResult( manualGeoLiteCityBlocksLines, null, null, null, null, null, null, "8.8.8.8", null, null, null, null, null, null, null, null); } @@ -419,7 +405,7 @@ public class LookupServiceTest { + "latitude,longitude,metroCode,areaCode"); geoLiteCityLocationLines.add("threetwoonenineone,\"US\",\"CA\"," + "\"Mountain View\",\"\",37.3860,-122.0838,807,650"); - LookupServiceTestHelper.assertLookupResult(this.tempGeoipDir, null, + this.assertLookupResult(null, null, null, geoLiteCityLocationLines, null, null, null, "8.8.8.8", null, null, null, null, null, null, null, null); } @@ -433,7 +419,7 @@ public class LookupServiceTest { + "latitude,longitude,metroCode,areaCode"); geoLiteCityLocationLines.add("32191,\"US\",\"CA\",\"Mountain View\"," + "\"\",37.3860,-122.0838,807"); - LookupServiceTestHelper.assertLookupResult(this.tempGeoipDir, null, + this.assertLookupResult(null, null, null, geoLiteCityLocationLines, null, null, null, "8.8.8.8", null, null, null, null, null, null, null, null); } @@ -442,7 +428,7 @@ public class LookupServiceTest { public void testLookupIso3166TooFewFields() { List<String> iso3166Lines = new ArrayList<String>(); iso3166Lines.add("US"); - LookupServiceTestHelper.assertLookupResult(this.tempGeoipDir, null, + this.assertLookupResult(null, null, null, null, iso3166Lines, null, null, "8.8.8.8", null, null, null, null, null, null, null, null); } @@ -451,7 +437,7 @@ public class LookupServiceTest { public void testLookupRegionTooFewFields() { List<String> regionLines = new ArrayList<String>(); regionLines.add("US,CA"); - LookupServiceTestHelper.assertLookupResult(this.tempGeoipDir, null, + this.assertLookupResult(null, null, null, null, null, regionLines, null, "8.8.8.8", null, null, null, null, null, null, null, null); } @@ -464,7 +450,7 @@ public class LookupServiceTest { geoipASNum2Lines.add("134744319,134744064,\"AS15169 Google Inc.\""); geoipASNum2Lines.add("134744320,134750463,\"AS3356 Level 3 " + "Communications\""); - LookupServiceTestHelper.assertLookupResult(this.tempGeoipDir, null, + this.assertLookupResult(null, null, null, null, null, null, geoipASNum2Lines, "8.8.8.8", "us", "United States", "California", "Mountain View", "37.3860", "-122.0838", null, null); @@ -474,7 +460,7 @@ public class LookupServiceTest { public void testLookupGeoipASNum2StartNotANumber() { List<String> geoipASNum2Lines = new ArrayList<String>(); geoipASNum2Lines.add("one,134744319,\"AS15169 Google Inc.\""); - LookupServiceTestHelper.assertLookupResult(this.tempGeoipDir, null, + this.assertLookupResult(null, null, null, null, null, null, geoipASNum2Lines, "8.8.8.8", null, null, null, null, null, null, null, null); } @@ -484,7 +470,7 @@ public class LookupServiceTest { List<String> geoipASNum2Lines = new ArrayList<String>(); geoipASNum2Lines.add("1" + String.valueOf(Long.MAX_VALUE) + ",134744319,\"AS15169 Google Inc.\""); - LookupServiceTestHelper.assertLookupResult(this.tempGeoipDir, null, + this.assertLookupResult(null, null, null, null, null, null, geoipASNum2Lines, "8.8.8.8", null, null, null, null, null, null, null, null); } @@ -493,7 +479,7 @@ public class LookupServiceTest { public void testLookupGeoipASNum2TooFewFields() { List<String> geoipASNum2Lines = new ArrayList<String>(); geoipASNum2Lines.add("134744064,134744319"); - LookupServiceTestHelper.assertLookupResult(this.tempGeoipDir, null, + this.assertLookupResult(null, null, null, null, null, null, geoipASNum2Lines, "8.8.8.8", null, null, null, null, null, null, null, null); } diff --git a/test/org/torproject/onionoo/ResourceServletTest.java b/test/org/torproject/onionoo/ResourceServletTest.java index fbe7db7..ca7cd69 100644 --- a/test/org/torproject/onionoo/ResourceServletTest.java +++ b/test/org/torproject/onionoo/ResourceServletTest.java @@ -32,243 +32,233 @@ import com.google.gson.Gson; public class ResourceServletTest { - private static class ResourceServletTestHelper { + private SortedMap<String, String> relays, bridges; - private File tempOutDir; + // 2013-04-24 12:22:22 + private static long lastModified = 1366806142000L; - private SortedMap<String, String> - relays = new TreeMap<String, String>(), - bridges = new TreeMap<String, String>(); + private long currentTimeMillis = 1366806142000L; - // 2013-04-24 12:22:22 - private static long lastModified = 1366806142000L; - - private static long currentTimeMillis = 1366806142000L; - - private class TestingTime extends Time { - public long currentTimeMillis() { - return currentTimeMillis; - } + private class TestingTime extends Time { + public long currentTimeMillis() { + return currentTimeMillis; } + } - private Time testingTime = new TestingTime(); + private Time testingTime = new TestingTime(); - private boolean maintenanceMode = false; + private boolean maintenanceMode = false; - private class TestingHttpServletRequestWrapper - extends HttpServletRequestWrapper { - private String requestURI; - private Map<String, String[]> parameterMap; - private TestingHttpServletRequestWrapper(String requestURI, - Map<String, String[]> parameterMap) { - super(null); - this.requestURI = requestURI; - this.parameterMap = parameterMap == null - ? new HashMap<String, String[]>() : parameterMap; - } - protected String getRequestURI() { - return this.requestURI; - } - protected Map getParameterMap() { - return this.parameterMap; - } - protected String[] getParameterValues(String parameterKey) { - return this.parameterMap.get(parameterKey); - } + private class TestingHttpServletRequestWrapper + extends HttpServletRequestWrapper { + private String requestURI; + private Map<String, String[]> parameterMap; + private TestingHttpServletRequestWrapper(String requestURI, + Map<String, String[]> parameterMap) { + super(null); + this.requestURI = requestURI; + this.parameterMap = parameterMap == null + ? new HashMap<String, String[]>() : parameterMap; + } + protected String getRequestURI() { + return this.requestURI; + } + protected Map getParameterMap() { + return this.parameterMap; + } + protected String[] getParameterValues(String parameterKey) { + return this.parameterMap.get(parameterKey); } + } - private class TestingHttpServletResponseWrapper extends - HttpServletResponseWrapper { - private TestingHttpServletResponseWrapper() { - super(null); - } - private int errorStatusCode; - protected void sendError(int errorStatusCode) throws IOException { - this.errorStatusCode = errorStatusCode; - } - private Map<String, String> headers = new HashMap<String, String>(); - protected void setHeader(String headerName, String headerValue) { - this.headers.put(headerName, headerValue); - } - protected void setContentType(String contentType) { - } - protected void setCharacterEncoding(String characterEncoding) { - } - private StringWriter stringWriter; - protected PrintWriter getWriter() throws IOException { - if (this.stringWriter == null) { - this.stringWriter = new StringWriter(); - return new PrintWriter(this.stringWriter); - } else { - throw new IOException("Can only request writer once"); - } - } - private String getWrittenContent() { - return this.stringWriter == null ? null - : this.stringWriter.toString(); + private class TestingHttpServletResponseWrapper extends + HttpServletResponseWrapper { + private TestingHttpServletResponseWrapper() { + super(null); + } + private int errorStatusCode; + protected void sendError(int errorStatusCode) throws IOException { + this.errorStatusCode = errorStatusCode; + } + private Map<String, String> headers = new HashMap<String, String>(); + protected void setHeader(String headerName, String headerValue) { + this.headers.put(headerName, headerValue); + } + protected void setContentType(String contentType) { + } + protected void setCharacterEncoding(String characterEncoding) { + } + private StringWriter stringWriter; + protected PrintWriter getWriter() throws IOException { + if (this.stringWriter == null) { + this.stringWriter = new StringWriter(); + return new PrintWriter(this.stringWriter); + } else { + throw new IOException("Can only request writer once"); } } + private String getWrittenContent() { + return this.stringWriter == null ? null + : this.stringWriter.toString(); + } + } - private TestingHttpServletRequestWrapper request; + private TestingHttpServletRequestWrapper request; - private TestingHttpServletResponseWrapper response; + private TestingHttpServletResponseWrapper response; - private String responseString; + private String responseString; - private SummaryDocument summaryDocument; + private SummaryDocument summaryDocument; - private ResourceServletTestHelper(File tempOutDir, String requestURI, - Map<String, String[]> parameterMap) { - this.tempOutDir = tempOutDir; - this.relays = new TreeMap<String, String>(); - this.relays.put("000C5F55", "r\tTorkaZ\t" - + "000C5F55BD4814B917CC474BD537F1A3B33CCE2A\t" - + "62.216.201.221;;62.216.201.222+62.216.201.223\t" - + "2013-04-19\t05:00:00\t9001\t0\tRunning,Valid\t20\tde\tnull\t" - + "-1\treject\t1-65535\t2013-04-18\t05:00:00\t" - + "2013-04-19\t05:00:00\tAS8767\ttorkaz <klaus dot zufall at " - + "gmx dot de> <fb-token:np5_g_83jmf=>"); - this.relays.put("001C13B3", "r\tFerrari458\t" - + "001C13B3A55A71B977CA65EC85539D79C653A3FC\t" - + "68.38.171.200;[2001:4f8:3:2e::51]:9001;\t" - + "2013-04-24\t12:00:00\t9001\t9030\t" - + "Fast,Named,Running,V2Dir,Valid\t1140\tus\t" - + "c-68-38-171-200.hsd1.pa.comcast.net\t1366805763009\treject\t" - + "1-65535\t2013-02-12\t16:00:00\t2013-02-26\t18:00:00\t" - + "AS7922\t"); - this.relays.put("0025C136", "r\tTimMayTribute\t" - + "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B\t89.69.68.246;;\t" - + "2013-04-22\t20:00:00\t9001\t9030\t" - + "Fast,Running,Unnamed,V2Dir,Valid\t63\ta1\tnull\t-1\treject\t" - + "1-65535\t2013-04-16\t18:00:00\t2013-04-16\t18:00:00\t" - + "AS6830\t1024D/51E2A1C7 steven j. murdoch " - + "<tor+steven.murdoch(a)cl.cam.ac.uk> <fb-token:5sr_k_zs2wm=>"); - this.bridges.put("0000831B", "b\tec2bridgercc7f31fe\t" - + "0000831B236DFF73D409AD17B40E2A728A53994F\t10.199.7.176;;\t" - + "2013-04-21\t18:07:03\t443\t0\tValid\t-1\t??\tnull\t-1\t" - + "null\tnull\t2013-04-20\t15:37:04\tnull\tnull\tnull\tnull"); - this.bridges.put("0002D9BD", "b\tUnnamed\t" - + "0002D9BDBBC230BD9C78FF502A16E0033EF87E0C\t10.0.52.84;;\t" - + "2013-04-20\t17:37:04\t443\t0\tValid\t-1\t??\tnull\t-1\t" - + "null\tnull\t2013-04-14\t07:07:05\tnull\tnull\tnull\tnull"); - this.bridges.put("0010D49C", "b\tgummy\t" - + "1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB756\t10.63.169.98;;\t" - + "2013-04-24\t01:07:04\t9001\t0\tRunning,Valid\t-1\t??\tnull\t" - + "-1\tnull\tnull\t2013-01-16\t21:07:04\tnull\tnull\tnull\t" - + "null"); - this.request = new TestingHttpServletRequestWrapper(requestURI, - parameterMap); - this.response = new TestingHttpServletResponseWrapper(); + @Before + public void createSampleRelaysAndBridges() { + this.relays = new TreeMap<String, String>(); + this.relays.put("000C5F55", "r\tTorkaZ\t" + + "000C5F55BD4814B917CC474BD537F1A3B33CCE2A\t" + + "62.216.201.221;;62.216.201.222+62.216.201.223\t" + + "2013-04-19\t05:00:00\t9001\t0\tRunning,Valid\t20\tde\tnull\t" + + "-1\treject\t1-65535\t2013-04-18\t05:00:00\t" + + "2013-04-19\t05:00:00\tAS8767\ttorkaz <klaus dot zufall at " + + "gmx dot de> <fb-token:np5_g_83jmf=>"); + this.relays.put("001C13B3", "r\tFerrari458\t" + + "001C13B3A55A71B977CA65EC85539D79C653A3FC\t" + + "68.38.171.200;[2001:4f8:3:2e::51]:9001;\t" + + "2013-04-24\t12:00:00\t9001\t9030\t" + + "Fast,Named,Running,V2Dir,Valid\t1140\tus\t" + + "c-68-38-171-200.hsd1.pa.comcast.net\t1366805763009\treject\t" + + "1-65535\t2013-02-12\t16:00:00\t2013-02-26\t18:00:00\t" + + "AS7922\t"); + this.relays.put("0025C136", "r\tTimMayTribute\t" + + "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B\t89.69.68.246;;\t" + + "2013-04-22\t20:00:00\t9001\t9030\t" + + "Fast,Running,Unnamed,V2Dir,Valid\t63\ta1\tnull\t-1\treject\t" + + "1-65535\t2013-04-16\t18:00:00\t2013-04-16\t18:00:00\t" + + "AS6830\t1024D/51E2A1C7 steven j. murdoch " + + "<tor+steven.murdoch(a)cl.cam.ac.uk> <fb-token:5sr_k_zs2wm=>"); + this.bridges = new TreeMap<String, String>(); + this.bridges.put("0000831B", "b\tec2bridgercc7f31fe\t" + + "0000831B236DFF73D409AD17B40E2A728A53994F\t10.199.7.176;;\t" + + "2013-04-21\t18:07:03\t443\t0\tValid\t-1\t??\tnull\t-1\t" + + "null\tnull\t2013-04-20\t15:37:04\tnull\tnull\tnull\tnull"); + this.bridges.put("0002D9BD", "b\tUnnamed\t" + + "0002D9BDBBC230BD9C78FF502A16E0033EF87E0C\t10.0.52.84;;\t" + + "2013-04-20\t17:37:04\t443\t0\tValid\t-1\t??\tnull\t-1\t" + + "null\tnull\t2013-04-14\t07:07:05\tnull\tnull\tnull\tnull"); + this.bridges.put("0010D49C", "b\tgummy\t" + + "1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB756\t10.63.169.98;;\t" + + "2013-04-24\t01:07:04\t9001\t0\tRunning,Valid\t-1\t??\tnull\t" + + "-1\tnull\tnull\t2013-01-16\t21:07:04\tnull\tnull\tnull\t" + + "null"); + } + + private void runTest(String requestURI, + Map<String, String[]> parameterMap) { + try { + this.writeSummaryFile(); + this.makeRequest(requestURI, parameterMap); + this.parseResponse(); + } catch (IOException e) { + throw new RuntimeException(e); } + } - private void runTest() { - try { - this.writeSummaryFile(); - this.makeRequest(); - this.parseResponse(); - } catch (IOException e) { - throw new RuntimeException(e); - } + /* TODO Instead of writing out/summary and out/update to a temp + * directory, we could also write our own DocumentStore instance. */ + private void writeSummaryFile() throws IOException { + File summaryFile = new File(this.tempOutDir, "summary"); + BufferedWriter bw = new BufferedWriter(new FileWriter(summaryFile)); + for (String relay : this.relays.values()) { + bw.write(relay + "\n"); } - - /* TODO Instead of writing out/summary and out/update to a temp - * directory, we could also write our own DocumentStore instance. */ - private void writeSummaryFile() throws IOException { - File summaryFile = new File(this.tempOutDir, "summary"); - BufferedWriter bw = new BufferedWriter(new FileWriter(summaryFile)); - for (String relay : relays.values()) { - bw.write(relay + "\n"); - } - for (String bridge : bridges.values()) { - bw.write(bridge + "\n"); - } - bw.close(); - File updateFile = new File(this.tempOutDir, "update"); - bw = new BufferedWriter(new FileWriter(updateFile)); - bw.write(String.valueOf(lastModified++)); - bw.close(); + for (String bridge : this.bridges.values()) { + bw.write(bridge + "\n"); } - - private void makeRequest() throws IOException { - ResourceServlet rs = new ResourceServlet(); - rs.init(maintenanceMode, this.tempOutDir, this.testingTime); - rs.doGet(this.request, this.response); + bw.close(); + File updateFile = new File(this.tempOutDir, "update"); + bw = new BufferedWriter(new FileWriter(updateFile)); + bw.write(String.valueOf(lastModified++)); + bw.close(); + } + + private void makeRequest(String requestURI, + Map<String, String[]> parameterMap) throws IOException { + ResourceServlet rs = new ResourceServlet(); + rs.init(maintenanceMode, this.tempOutDir, this.testingTime); + this.request = new TestingHttpServletRequestWrapper(requestURI, + parameterMap); + this.response = new TestingHttpServletResponseWrapper(); + rs.doGet(this.request, this.response); + } + + private void parseResponse() { + this.responseString = this.response.getWrittenContent(); + if (this.responseString != null) { + Gson gson = new Gson(); + this.summaryDocument = gson.fromJson(this.responseString, + SummaryDocument.class); } + } - private void parseResponse() { - this.responseString = this.response.getWrittenContent(); - if (this.responseString != null) { - Gson gson = new Gson(); - this.summaryDocument = gson.fromJson(this.responseString, - SummaryDocument.class); + private void assertErrorStatusCode(String request, + int errorStatusCode) { + String requestURI = parseRequestURI(request); + Map<String, String[]> parameters = parseParameters(request); + this.runTest(requestURI, parameters); + assertEquals(errorStatusCode, this.response.errorStatusCode); + } + + private void assertSummaryDocument(String request, + int expectedRelaysNumber, String[] expectedRelaysNicknames, + int expectedBridgesNumber, String[] expectedBridgesNicknames) { + String requestURI = parseRequestURI(request); + Map<String, String[]> parameters = parseParameters(request); + this.runTest(requestURI, parameters); + assertNotNull(this.summaryDocument); + assertEquals(expectedRelaysNumber, + this.summaryDocument.relays.length); + if (expectedRelaysNicknames != null) { + for (int i = 0; i < expectedRelaysNumber; i++) { + assertEquals(expectedRelaysNicknames[i], + this.summaryDocument.relays[i].n); } } - - private static void assertErrorStatusCode(File tempOutDir, - String request, int errorStatusCode) { - String requestURI = parseRequestURI(request); - Map<String, String[]> parameters = parseParameters(request); - ResourceServletTestHelper helper = new ResourceServletTestHelper( - tempOutDir, requestURI, parameters); - helper.runTest(); - assertEquals(errorStatusCode, helper.response.errorStatusCode); - } - - private static void assertSummaryDocument(File tempOutDir, - String request, int expectedRelaysNumber, - String[] expectedRelaysNicknames, int expectedBridgesNumber, - String[] expectedBridgesNicknames) { - String requestURI = parseRequestURI(request); - Map<String, String[]> parameters = parseParameters(request); - ResourceServletTestHelper helper = new ResourceServletTestHelper( - tempOutDir, requestURI, parameters); - helper.runTest(); - assertNotNull(helper.summaryDocument); - assertEquals(expectedRelaysNumber, - helper.summaryDocument.relays.length); - if (expectedRelaysNicknames != null) { - for (int i = 0; i < expectedRelaysNumber; i++) { - assertEquals(expectedRelaysNicknames[i], - helper.summaryDocument.relays[i].n); - } - } - assertEquals(expectedBridgesNumber, - helper.summaryDocument.bridges.length); - if (expectedBridgesNicknames != null) { - for (int i = 0; i < expectedBridgesNumber; i++) { - assertEquals(expectedBridgesNicknames[i], - helper.summaryDocument.bridges[i].n); - } + assertEquals(expectedBridgesNumber, + this.summaryDocument.bridges.length); + if (expectedBridgesNicknames != null) { + for (int i = 0; i < expectedBridgesNumber; i++) { + assertEquals(expectedBridgesNicknames[i], + this.summaryDocument.bridges[i].n); } } + } - private static String parseRequestURI(String request) { - return request.split("\\?")[0]; - } + private String parseRequestURI(String request) { + return request.split("\\?")[0]; + } - private static Map<String, String[]> parseParameters(String request) { - Map<String, String[]> parameters = null; - String[] uriParts = request.split("\\?"); - if (uriParts.length == 2) { - Map<String, List<String>> parameterLists = - new HashMap<String, List<String>>(); - for (String parameter : uriParts[1].split("&")) { - String[] parameterParts = parameter.split("="); - if (!parameterLists.containsKey(parameterParts[0])) { - parameterLists.put(parameterParts[0], - new ArrayList<String>()); - } - parameterLists.get(parameterParts[0]).add(parameterParts[1]); - } - parameters = new HashMap<String, String[]>(); - for (Map.Entry<String, List<String>> e : - parameterLists.entrySet()) { - parameters.put(e.getKey(), - e.getValue().toArray(new String[e.getValue().size()])); + private Map<String, String[]> parseParameters(String request) { + Map<String, String[]> parameters = null; + String[] uriParts = request.split("\\?"); + if (uriParts.length == 2) { + Map<String, List<String>> parameterLists = + new HashMap<String, List<String>>(); + for (String parameter : uriParts[1].split("&")) { + String[] parameterParts = parameter.split("="); + if (!parameterLists.containsKey(parameterParts[0])) { + parameterLists.put(parameterParts[0], + new ArrayList<String>()); } + parameterLists.get(parameterParts[0]).add(parameterParts[1]); + } + parameters = new HashMap<String, String[]>(); + for (Map.Entry<String, List<String>> e : + parameterLists.entrySet()) { + parameters.put(e.getKey(), + e.getValue().toArray(new String[e.getValue().size()])); } - return parameters; } + return parameters; } @Rule @@ -278,7 +268,7 @@ public class ResourceServletTest { @Before public void createTempOutDir() throws IOException { - tempOutDir = this.tempFolder.newFolder("out"); + this.tempOutDir = this.tempFolder.newFolder("out"); } private static class SummaryDocument { @@ -303,19 +293,17 @@ public class ResourceServletTest { @Test() public void testValidSummaryRelay() throws IOException { - ResourceServletTestHelper helper = new ResourceServletTestHelper( - this.tempOutDir, "/summary", null); - helper.relays.clear(); - helper.relays.put("000C5F55", "r TorkaZ " + this.relays.clear(); + this.relays.put("000C5F55", "r TorkaZ " + "000C5F55BD4814B917CC474BD537F1A3B33CCE2A 62.216.201.221;; " + "2013-04-19 05:00:00 9001 0 Running,Valid 20 de null -1 " + "reject 1-65535 2013-04-18 05:00:00 2013-04-19 05:00:00 " + "AS8767"); - helper.runTest(); + this.runTest("/summary", null); assertEquals("2013-04-19 05:00:00", - helper.summaryDocument.relays_published); - assertEquals(1, helper.summaryDocument.relays.length); - RelaySummary relay = helper.summaryDocument.relays[0]; + this.summaryDocument.relays_published); + assertEquals(1, this.summaryDocument.relays.length); + RelaySummary relay = this.summaryDocument.relays[0]; assertEquals("TorkaZ", relay.n); assertEquals("000C5F55BD4814B917CC474BD537F1A3B33CCE2A", relay.f); assertEquals(1, relay.a.length); @@ -325,18 +313,16 @@ public class ResourceServletTest { @Test() public void testValidSummaryBridge() { - ResourceServletTestHelper helper = new ResourceServletTestHelper( - this.tempOutDir, "/summary", null); - helper.bridges.clear(); - helper.bridges.put("0000831", "b ec2bridgercc7f31fe " + this.bridges.clear(); + this.bridges.put("0000831", "b ec2bridgercc7f31fe " + "0000831B236DFF73D409AD17B40E2A728A53994F 10.199.7.176;; " + "2013-04-21 18:07:03 443 0 Valid -1 ?? null -1 null null " + "2013-04-20 15:37:04 null null null"); - helper.runTest(); + this.runTest("/summary", null); assertEquals("2013-04-21 18:07:03", - helper.summaryDocument.bridges_published); - assertEquals(1, helper.summaryDocument.bridges.length); - BridgeSummary bridge = helper.summaryDocument.bridges[0]; + this.summaryDocument.bridges_published); + assertEquals(1, this.summaryDocument.bridges.length); + BridgeSummary bridge = this.summaryDocument.bridges[0]; assertEquals("ec2bridgercc7f31fe", bridge.n); assertEquals("0000831B236DFF73D409AD17B40E2A728A53994F", bridge.h); assertFalse(bridge.r); @@ -344,708 +330,708 @@ public class ResourceServletTest { @Test() public void testNonExistantDocumentType() { - ResourceServletTestHelper.assertErrorStatusCode(this.tempOutDir, + this.assertErrorStatusCode( "/doesnotexist", 400); } @Test() public void testSUMMARYDocument() { - ResourceServletTestHelper.assertErrorStatusCode(this.tempOutDir, + this.assertErrorStatusCode( "/SUMMARY", 400); } @Test() public void testTypeRelay() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?type=relay", 3, null, 0, null); } @Test() public void testTypeBridge() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?type=bridge", 0, null, 3, null); } @Test() public void testTypeBridgerelay() { - ResourceServletTestHelper.assertErrorStatusCode(this.tempOutDir, + this.assertErrorStatusCode( "/summary?type=bridgerelay", 400); } @Test() public void testTypeRelayBridge() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?type=relay&type=bridge", 3, null, 0, null); } @Test() public void testTypeBridgeRelay() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?type=bridge&type=relay", 0, null, 3, null); } @Test() public void testTypeRelayRelay() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?type=relay&type=relay", 3, null, 0, null); } @Test() public void testTYPERelay() { - ResourceServletTestHelper.assertErrorStatusCode(this.tempOutDir, + this.assertErrorStatusCode( "/summary?TYPE=relay", 400); } @Test() public void testTypeRELAY() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?type=RELAY", 3, null, 0, null); } @Test() public void testRunningTrue() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?running=true", 1, new String[] { "Ferrari458" }, 1, new String[] { "gummy" }); } @Test() public void testRunningFalse() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?running=false", 2, null, 2, null); } @Test() public void testRunningTruefalse() { - ResourceServletTestHelper.assertErrorStatusCode(this.tempOutDir, + this.assertErrorStatusCode( "/summary?running=truefalse", 400); } @Test() public void testRunningTrueFalse() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?running=true&running=false", 1, new String[] { "Ferrari458" }, 1, new String[] { "gummy" }); } @Test() public void testRunningFalseTrue() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?running=false&running=true", 2, null, 2, null); } @Test() public void testRunningTrueTrue() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?running=true&running=true", 1, new String[] { "Ferrari458" }, 1, new String[] { "gummy" }); } @Test() public void testRUNNINGTrue() { - ResourceServletTestHelper.assertErrorStatusCode(this.tempOutDir, + this.assertErrorStatusCode( "/summary?RUNNING=true", 400); } @Test() public void testRunningTRUE() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?running=TRUE", 1, null, 1, null); } @Test() public void testSearchTorkaZ() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=TorkaZ", 1, new String[] { "TorkaZ" }, 0, null); } @Test() public void testSearchTorkaX() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=TorkaX", 0, null, 0, null); } @Test() public void testSearchOrkaZ() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=orkaZ", 1, new String[] { "TorkaZ" }, 0, null); } @Test() public void testSearchTorka() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=Torka", 1, new String[] { "TorkaZ" }, 0, null); } @Test() public void testSearchTORKAZ() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=TORKAZ", 1, new String[] { "TorkaZ" }, 0, null); } @Test() public void testSearchDollarFingerprint() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=$000C5F55BD4814B917CC474BD537F1A3B33CCE2A", 1, new String[] { "TorkaZ" }, 0, null); } @Test() public void testSearchFingerprint() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=000C5F55BD4814B917CC474BD537F1A3B33CCE2A", 1, new String[] { "TorkaZ" }, 0, null); } @Test() public void testSearchDollarFingerprint39() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=$000C5F55BD4814B917CC474BD537F1A3B33CCE2", 1, new String[] { "TorkaZ" }, 0, null); } @Test() public void testSearchDollarFingerprintLowerCase39() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=$000c5f55bd4814b917cc474bd537f1a3b33cce2", 1, new String[] { "TorkaZ" }, 0, null); } @Test() public void testSearchFingerprintLowerCase39() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=000c5f55bd4814b917cc474bd537f1a3b33cce2", 1, new String[] { "TorkaZ" }, 0, null); } @Test() public void testSearchDollarHashedFingerprint() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=$5aa14c08d62913e0057a9ad5863b458c0ce94cee", 1, new String[] { "TorkaZ" }, 0, null); } @Test() public void testSearchDollarHashedFingerprint39() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=$5aa14c08d62913e0057a9ad5863b458c0ce94ce", 1, new String[] { "TorkaZ" }, 0, null); } @Test() public void testSearchDollarHashedFingerprint41() { - ResourceServletTestHelper.assertErrorStatusCode(this.tempOutDir, + this.assertErrorStatusCode( "/summary?search=$5aa14c08d62913e0057a9ad5863b458c0ce94ceee", 400); } @Test() public void testSearchIp() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=62.216.201.221", 1, new String[] { "TorkaZ" }, 0, null); } @Test() public void testSearchIp24Network() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=62.216.201", 1, new String[] { "TorkaZ" }, 0, null); } @Test() public void testSearchIpExit() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=62.216.201.222", 1, new String[] { "TorkaZ" }, 0, null); } @Test() public void testSearchIpv6() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=[2001:4f8:3:2e::51]", 1, new String[] { "Ferrari458" }, 0, null); } @Test() public void testSearchIpv6Slash64NoTrailingBracket() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=[2001:4f8:3:2e::", 1, new String[] { "Ferrari458" }, 0, null); } @Test() public void testSearchIpv6Slash64TrailingBracket() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=[2001:4f8:3:2e::]", 0, null, 0, null); } @Test() public void testSearchIpv6Uncompressed() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=[2001:04f8:0003:002e:0000:0000:0000:0051]", 0, null, 0, null); } @Test() public void testSearchIpv6UpperCase() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=[2001:4F8:3:2E::51]", 1, new String[] { "Ferrari458" }, 0, null); } @Test() public void testSearchIpv6ThreeColons() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=[2001:4f8:3:2e:::51]", 0, null, 0, null); } @Test() public void testSearchIpv6FiveHex() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=[20014:f80:3:2e::51]", 0, null, 0, null); } @Test() public void testSearchIpv6NineGroups() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=[1:2:3:4:5:6:7:8:9]", 0, null, 0, null); } @Test() public void testSearchIpv6TcpPort() { - ResourceServletTestHelper.assertErrorStatusCode(this.tempOutDir, + this.assertErrorStatusCode( "/summary?search=[2001:4f8:3:2e::51]:9001", 400); } @Test() public void testSearchGummy() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=gummy", 0, null, 1, new String[] { "gummy" }); } @Test() public void testSearchGummi() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=gummi", 0, null, 0, null); } @Test() public void testSearchUmmy() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=ummy", 0, null, 1, new String[] { "gummy" }); } @Test() public void testSearchGumm() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=gumm", 0, null, 1, new String[] { "gummy" }); } @Test() public void testSearchGUMMY() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=GUMMY", 0, null, 1, new String[] { "gummy" }); } @Test() public void testSearchBridgeDollarHashedFingerprint() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=$1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB756", 0, null, 1, new String[] { "gummy" }); } @Test() public void testSearchBridgeHashedFingerprint() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB756", 0, null, 1, new String[] { "gummy" }); } @Test() public void testSearchBridgeDollarHashedFingerprint39() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=$1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB75", 0, null, 1, new String[] { "gummy" }); } @Test() public void testSearchBridgeDollarHashedFingerprintLowerCase39() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=$1fede50ed8dba1dd9f9165f78c8131e4a44ab75", 0, null, 1, new String[] { "gummy" }); } @Test() public void testSearchBridgeHashedFingerprintLowerCase39() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=1fede50ed8dba1dd9f9165f78c8131e4a44ab75", 0, null, 1, new String[] { "gummy" }); } @Test() public void testSearchBridgeDollarHashedHashedFingerprint() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=$CE52F898DB3678BCE33FAC28C92774DE90D618B5", 0, null, 1, new String[] { "gummy" }); } @Test() public void testSearchBridgeDollarHashedHashedFingerprint39() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=$CE52F898DB3678BCE33FAC28C92774DE90D618B", 0, null, 1, new String[] { "gummy" }); } @Test() public void testSearchBridgeDollarOriginalFingerprint() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?search=$0010D49C6DA1E46A316563099F41BFE40B6C7183", 0, null, 0, null); } @Test() public void testSearchUnderscore() { - ResourceServletTestHelper.assertErrorStatusCode(this.tempOutDir, + this.assertErrorStatusCode( "/summary?search=_", 400); } @Test() public void testLookupFingerprint() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?lookup=000C5F55BD4814B917CC474BD537F1A3B33CCE2A", 1, new String[] { "TorkaZ" }, 0, null); } @Test() public void testLookupDollarFingerprint() { - ResourceServletTestHelper.assertErrorStatusCode(this.tempOutDir, + this.assertErrorStatusCode( "/summary?lookup=$000C5F55BD4814B917CC474BD537F1A3B33CCE2A", 400); } @Test() public void testLookupDollarFingerprint39() { - ResourceServletTestHelper.assertErrorStatusCode(this.tempOutDir, + this.assertErrorStatusCode( "/summary?lookup=$000C5F55BD4814B917CC474BD537F1A3B33CCE2", 400); } @Test() public void testLookupFingerprintLowerCase39() { - ResourceServletTestHelper.assertErrorStatusCode(this.tempOutDir, + this.assertErrorStatusCode( "/summary?lookup=000c5f55bd4814b917cc474bd537f1a3b33cce2", 400); } @Test() public void testLookupHashedFingerprint() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?lookup=5aa14c08d62913e0057a9ad5863b458c0ce94cee", 1, new String[] { "TorkaZ" }, 0, null); } @Test() public void testLookupBridgeHashedFingerprint() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?lookup=1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB756", 0, null, 1, new String[] { "gummy" }); } @Test() public void testSearchBridgeHashedHashedFingerprint() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?lookup=CE52F898DB3678BCE33FAC28C92774DE90D618B5", 0, null, 1, new String[] { "gummy" }); } @Test() public void testLookupBridgeOriginalFingerprint() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?lookup=0010D49C6DA1E46A316563099F41BFE40B6C7183", 0, null, 0, null); } @Test() public void testCountryDe() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?country=de", 1, new String[] { "TorkaZ" }, 0, null); } @Test() public void testCountryFr() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?country=fr", 0, null, 0, null); } @Test() public void testCountryZz() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?country=zz", 0, null, 0, null); } @Test() public void testCountryDE() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?country=DE", 1, new String[] { "TorkaZ" }, 0, null); } @Test() public void testCountryDeu() { - ResourceServletTestHelper.assertErrorStatusCode(this.tempOutDir, + this.assertErrorStatusCode( "/summary?country=deu", 400); } @Test() public void testCountryD() { - ResourceServletTestHelper.assertErrorStatusCode(this.tempOutDir, + this.assertErrorStatusCode( "/summary?country=d", 400); } @Test() public void testCountryA1() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?country=a1", 1, new String[] { "TimMayTribute" }, 0, null); } @Test() public void testCountryDeDe() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?country=de&country=de", 1, new String[] { "TorkaZ" }, 0, null); } @Test() public void testAsAS8767() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?as=AS8767", 1, new String[] { "TorkaZ" }, 0, null); } @Test() public void testAs8767() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?as=8767", 1, new String[] { "TorkaZ" }, 0, null); } @Test() public void testAsAS() { - ResourceServletTestHelper.assertErrorStatusCode(this.tempOutDir, + this.assertErrorStatusCode( "/summary?as=AS", 400); } @Test() public void testAsas8767() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?as=as8767", 1, new String[] { "TorkaZ" }, 0, null); } @Test() public void testAsASSpace8767() { - ResourceServletTestHelper.assertErrorStatusCode(this.tempOutDir, + this.assertErrorStatusCode( "/summary?as=AS 8767", 400); } @Test() public void testFlagRunning() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?flag=Running", 3, null, 0, null); } @Test() public void testFlagValid() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?flag=Valid", 3, null, 0, null); } @Test() public void testFlagFast() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?flag=Fast", 2, null, 0, null); } @Test() public void testFlagNamed() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?flag=Named", 1, null, 0, null); } @Test() public void testFlagUnnamed() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?flag=Unnamed", 1, null, 0, null); } @Test() public void testFlagV2Dir() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?flag=V2Dir", 2, null, 0, null); } @Test() public void testFlagGuard() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?flag=Guard", 0, null, 0, null); } @Test() public void testFlagCool() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?flag=Cool", 0, null, 0, null); } @Test() public void testFirstSeenDaysZeroToTwo() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?first_seen_days=0-2", 0, null, 0, null); } @Test() public void testFirstSeenDaysUpToThree() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?first_seen_days=-3", 0, null, 1, null); } @Test() public void testFirstSeenDaysThree() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?first_seen_days=3", 0, null, 1, null); } @Test() public void testFirstSeenDaysTwoToFive() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?first_seen_days=2-5", 0, null, 1, null); } @Test() public void testFirstSeenDaysSixToSixteen() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?first_seen_days=6-16", 2, null, 1, null); } @Test() public void testFirstSeenDaysNinetysevenOrMore() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?first_seen_days=97-", 0, null, 1, null); } @Test() public void testFirstSeenDaysNinetyeightOrMore() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?first_seen_days=98-", 0, null, 0, null); } @Test() public void testFirstSeenDaysDashDash() { - ResourceServletTestHelper.assertErrorStatusCode(this.tempOutDir, + this.assertErrorStatusCode( "/summary?first_seen_days=--", 400); } @Test() public void testFirstSeenDaysDashOneDash() { - ResourceServletTestHelper.assertErrorStatusCode(this.tempOutDir, + this.assertErrorStatusCode( "/summary?first_seen_days=-1-", 400); } @Test() public void testFirstSeenDaysZeroDotDotOne() { - ResourceServletTestHelper.assertErrorStatusCode(this.tempOutDir, + this.assertErrorStatusCode( "/summary?first_seen_days=0..1", 400); } @Test() public void testFirstSeenDaysElevenDigits() { - ResourceServletTestHelper.assertErrorStatusCode(this.tempOutDir, + this.assertErrorStatusCode( "/summary?first_seen_days=12345678901", 400); } @Test() public void testFirstSeenDaysLargeTenDigitNumber() { - ResourceServletTestHelper.assertErrorStatusCode(this.tempOutDir, + this.assertErrorStatusCode( "/summary?first_seen_days=9999999999", 400); } @Test() public void testFirstSeenDaysMaxInt() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?last_seen_days=" + String.valueOf(Integer.MAX_VALUE), 0, null, 0, null); } @Test() public void testFirstSeenDaysMaxIntPlusOne() { - ResourceServletTestHelper.assertErrorStatusCode(this.tempOutDir, + this.assertErrorStatusCode( "/summary?first_seen_days=" + String.valueOf(Integer.MAX_VALUE + 1), 400); } @Test() public void testLastSeenDaysZero() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?last_seen_days=0", 1, null, 1, null); } @Test() public void testLastSeenDaysUpToZero() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?last_seen_days=-0", 1, null, 1, null); } @Test() public void testLastSeenDaysOneToThree() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?last_seen_days=1-3", 1, null, 2, null); } @Test() public void testLastSeenDaysSixOrMore() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?last_seen_days=6-", 0, null, 0, null); } @Test() public void testContactSteven() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?contact=Steven", 1, null, 0, null); } @Test() public void testContactStevenMurdoch() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?contact=Steven Murdoch", 1, null, 0, null); } @Test() public void testContactMurdochSteven() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?contact=Murdoch Steven", 1, null, 0, null); } @Test() public void testContactStevenDotMurdoch() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?contact=Steven.Murdoch", 1, null, 0, null); } @Test() public void testContactFbTokenFive() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?contact=<fb-token:5sR_K_zs2wM=>", 1, null, 0, null); } @Test() public void testContactFbToken() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?contact=<fb-token:", 2, null, 0, null); } @Test() public void testContactDash() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?contact=-", 2, null, 0, null); } @Test() public void testOrderConsensusWeightAscending() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?order=consensus_weight", 3, new String[] { "TorkaZ", "TimMayTribute", "Ferrari458" }, 3, null); @@ -1053,7 +1039,7 @@ public class ResourceServletTest { @Test() public void testOrderConsensusWeightDescending() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?order=-consensus_weight", 3, new String[] { "Ferrari458", "TimMayTribute", "TorkaZ" }, 3, null); @@ -1061,25 +1047,25 @@ public class ResourceServletTest { @Test() public void testOrderConsensusWeightAscendingTwice() { - ResourceServletTestHelper.assertErrorStatusCode(this.tempOutDir, + this.assertErrorStatusCode( "/summary?order=consensus_weight,consensus_weight", 400); } @Test() public void testOrderConsensusWeightAscendingThenDescending() { - ResourceServletTestHelper.assertErrorStatusCode(this.tempOutDir, + this.assertErrorStatusCode( "/summary?order=consensus_weight,-consensus_weight", 400); } @Test() public void testOrderConsensusWeightThenNickname() { - ResourceServletTestHelper.assertErrorStatusCode(this.tempOutDir, + this.assertErrorStatusCode( "/summary?order=consensus_weight,nickname", 400); } @Test() public void testOrderCONSENSUS_WEIGHT() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?order=CONSENSUS_WEIGHT", 3, new String[] { "TorkaZ", "TimMayTribute", "Ferrari458" }, 3, null); @@ -1087,111 +1073,111 @@ public class ResourceServletTest { @Test() public void testOrderConsensusWeightAscendingLimit1() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?order=consensus_weight&limit=1", 1, new String[] { "TorkaZ" }, 0, null); } @Test() public void testOrderConsensusWeightDecendingLimit1() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?order=-consensus_weight&limit=1", 1, new String[] { "Ferrari458" }, 0, null); } @Test() public void testOffsetOne() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?offset=1", 2, null, 3, null); } @Test() public void testOffsetAllRelays() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?offset=3", 0, null, 3, null); } @Test() public void testOffsetAllRelaysAndOneBridge() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?offset=4", 0, null, 2, null); } @Test() public void testOffsetAllRelaysAndAllBridges() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?offset=6", 0, null, 0, null); } @Test() public void testOffsetMoreThanAllRelaysAndAllBridges() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?offset=7", 0, null, 0, null); } @Test() public void testOffsetZero() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?offset=0", 3, null, 3, null); } @Test() public void testOffsetMinusOne() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?offset=-1", 3, null, 3, null); } @Test() public void testOffsetOneWord() { - ResourceServletTestHelper.assertErrorStatusCode(this.tempOutDir, + this.assertErrorStatusCode( "/summary?offset=one", 400); } @Test() public void testLimitOne() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?limit=1", 1, null, 0, null); } @Test() public void testLimitAllRelays() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?limit=3", 3, null, 0, null); } @Test() public void testLimitAllRelaysAndOneBridge() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?limit=4", 3, null, 1, null); } @Test() public void testLimitAllRelaysAndAllBridges() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?limit=6", 3, null, 3, null); } @Test() public void testLimitMoreThanAllRelaysAndAllBridges() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?limit=7", 3, null, 3, null); } @Test() public void testLimitZero() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?limit=0", 0, null, 0, null); } @Test() public void testLimitMinusOne() { - ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + this.assertSummaryDocument( "/summary?limit=-1", 0, null, 0, null); } @Test() public void testLimitOneWord() { - ResourceServletTestHelper.assertErrorStatusCode(this.tempOutDir, + this.assertErrorStatusCode( "/summary?limit=one", 400); } }

1 0

[onionoo/master] Merge two writer classes to speed up rDNS lookups.
by karsten＠torproject.org 28 Nov '13

28 Nov '13

commit f337c14e52f8290a0ce993339cca757cf5249153 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Sep 20 17:27:52 2013 +0200 Merge two writer classes to speed up rDNS lookups. The NodeDataWriter and DetailsDataWriter did very similar things. They processed various descriptors and performed GeoIP and rDNS lookups to learn the latest status of a relay or bridge for summary and details documents. Merging these classes makes sense, because we can start rDNS lookups much earlier which then run in the background. --- src/org/torproject/onionoo/DetailsDataWriter.java | 683 --------------------- src/org/torproject/onionoo/Main.java | 32 +- src/org/torproject/onionoo/NodeDataWriter.java | 649 +++++++++++++++++++- 3 files changed, 655 insertions(+), 709 deletions(-) diff --git a/src/org/torproject/onionoo/DetailsDataWriter.java b/src/org/torproject/onionoo/DetailsDataWriter.java deleted file mode 100644 index 4954dfc..0000000 --- a/src/org/torproject/onionoo/DetailsDataWriter.java +++ /dev/null @@ -1,683 +0,0 @@ -/* Copyright 2011, 2012 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Scanner; -import java.util.Set; -import java.util.SortedMap; -import java.util.SortedSet; -import java.util.TimeZone; -import java.util.TreeMap; -import java.util.TreeSet; - -import org.apache.commons.lang.StringEscapeUtils; - -import org.torproject.descriptor.BridgePoolAssignment; -import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.ExitList; -import org.torproject.descriptor.ExitListEntry; -import org.torproject.descriptor.ServerDescriptor; - -/* Write updated detail data files to disk and delete files of relays or - * bridges that fell out of the summary list. - * - * The parts of details files coming from server descriptors always come - * from the last known descriptor of a relay or bridge, not from the - * descriptor that was last referenced in a network status. */ -public class DetailsDataWriter implements DescriptorListener { - - private DescriptorSource descriptorSource; - - private ReverseDomainNameResolver reverseDomainNameResolver; - - private DocumentStore documentStore; - - private SortedMap<String, NodeStatus> relays; - - private SortedMap<String, NodeStatus> bridges; - - public DetailsDataWriter(DescriptorSource descriptorSource, - ReverseDomainNameResolver reverseDomainNameResolver, - DocumentStore documentStore) { - this.descriptorSource = descriptorSource; - this.reverseDomainNameResolver = reverseDomainNameResolver; - this.documentStore = documentStore; - this.registerDescriptorListeners(); - } - - private void registerDescriptorListeners() { - this.descriptorSource.registerListener(this, - DescriptorType.RELAY_SERVER_DESCRIPTORS); - this.descriptorSource.registerListener(this, - DescriptorType.BRIDGE_SERVER_DESCRIPTORS); - this.descriptorSource.registerListener(this, - DescriptorType.BRIDGE_POOL_ASSIGNMENTS); - this.descriptorSource.registerListener(this, - DescriptorType.EXIT_LISTS); - } - - public void processDescriptor(Descriptor descriptor, boolean relay) { - if (descriptor instanceof ServerDescriptor && relay) { - this.processRelayServerDescriptor((ServerDescriptor) descriptor); - } else if (descriptor instanceof ServerDescriptor && !relay) { - this.processBridgeServerDescriptor((ServerDescriptor) descriptor); - } else if (descriptor instanceof BridgePoolAssignment) { - this.processBridgePoolAssignment((BridgePoolAssignment) descriptor); - } else if (descriptor instanceof ExitList) { - this.processExitList((ExitList) descriptor); - } - } - - private void processRelayServerDescriptor( - ServerDescriptor descriptor) { - String fingerprint = descriptor.getFingerprint(); - DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, false, fingerprint); - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setLenient(false); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - String publishedDateTime = - dateTimeFormat.format(descriptor.getPublishedMillis()); - if (detailsStatus != null) { - String detailsString = detailsStatus.documentString; - String descPublishedLine = "\"desc_published\":\"" - + publishedDateTime + "\","; - Scanner s = new Scanner(detailsString); - while (s.hasNextLine()) { - String line = s.nextLine(); - if (line.startsWith("\"desc_published\":\"")) { - if (descPublishedLine.compareTo(line) < 0) { - return; - } else { - break; - } - } - } - s.close(); - } - StringBuilder sb = new StringBuilder(); - String lastRestartedString = dateTimeFormat.format( - descriptor.getPublishedMillis() - descriptor.getUptime() * 1000L); - int bandwidthRate = descriptor.getBandwidthRate(); - int bandwidthBurst = descriptor.getBandwidthBurst(); - int observedBandwidth = descriptor.getBandwidthObserved(); - int advertisedBandwidth = Math.min(bandwidthRate, - Math.min(bandwidthBurst, observedBandwidth)); - sb.append("\"desc_published\":\"" + publishedDateTime + "\",\n" - + "\"last_restarted\":\"" + lastRestartedString + "\",\n" - + "\"bandwidth_rate\":" + bandwidthRate + ",\n" - + "\"bandwidth_burst\":" + bandwidthBurst + ",\n" - + "\"observed_bandwidth\":" + observedBandwidth + ",\n" - + "\"advertised_bandwidth\":" + advertisedBandwidth + ",\n" - + "\"exit_policy\":["); - int written = 0; - for (String exitPolicyLine : descriptor.getExitPolicyLines()) { - sb.append((written++ > 0 ? "," : "") + "\n \"" + exitPolicyLine - + "\""); - } - sb.append("\n]"); - if (descriptor.getContact() != null) { - sb.append(",\n\"contact\":\"" - + escapeJSON(descriptor.getContact()) + "\""); - } - if (descriptor.getPlatform() != null) { - sb.append(",\n\"platform\":\"" - + escapeJSON(descriptor.getPlatform()) + "\""); - } - if (descriptor.getFamilyEntries() != null) { - sb.append(",\n\"family\":["); - written = 0; - for (String familyEntry : descriptor.getFamilyEntries()) { - sb.append((written++ > 0 ? "," : "") + "\n \"" + familyEntry - + "\""); - } - sb.append("\n]"); - } - detailsStatus = new DetailsStatus(); - detailsStatus.documentString = sb.toString(); - this.documentStore.store(detailsStatus, fingerprint); - } - - private void processBridgeServerDescriptor( - ServerDescriptor descriptor) { - String fingerprint = descriptor.getFingerprint(); - DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, false, fingerprint); - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setLenient(false); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - String publishedDateTime = - dateTimeFormat.format(descriptor.getPublishedMillis()); - String poolAssignmentLine = null; - if (detailsStatus != null) { - String detailsString = detailsStatus.documentString; - String descPublishedLine = "\"desc_published\":\"" - + publishedDateTime + "\","; - Scanner s = new Scanner(detailsString); - while (s.hasNextLine()) { - String line = s.nextLine(); - if (line.startsWith("\"pool_assignment\":")) { - poolAssignmentLine = line; - } else if (line.startsWith("\"desc_published\":") && - descPublishedLine.compareTo(line) < 0) { - return; - } - } - s.close(); - } - StringBuilder sb = new StringBuilder(); - String lastRestartedString = dateTimeFormat.format( - descriptor.getPublishedMillis() - descriptor.getUptime() * 1000L); - int advertisedBandwidth = Math.min(descriptor.getBandwidthRate(), - Math.min(descriptor.getBandwidthBurst(), - descriptor.getBandwidthObserved())); - sb.append("\"desc_published\":\"" + publishedDateTime + "\",\n" - + "\"last_restarted\":\"" + lastRestartedString + "\",\n" - + "\"advertised_bandwidth\":" + advertisedBandwidth + ",\n" - + "\"platform\":\"" + escapeJSON(descriptor.getPlatform()) - + "\""); - if (poolAssignmentLine != null) { - sb.append(",\n" + poolAssignmentLine); - } - detailsStatus = new DetailsStatus(); - detailsStatus.documentString = sb.toString(); - this.documentStore.store(detailsStatus, fingerprint); - } - - private void processBridgePoolAssignment( - BridgePoolAssignment bridgePoolAssignment) { - for (Map.Entry<String, String> e : - bridgePoolAssignment.getEntries().entrySet()) { - String fingerprint = e.getKey(); - String details = e.getValue(); - StringBuilder sb = new StringBuilder(); - DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, false, fingerprint); - if (detailsStatus != null) { - String detailsString = detailsStatus.documentString; - Scanner s = new Scanner(detailsString); - int linesWritten = 0; - boolean endsWithComma = false; - while (s.hasNextLine()) { - String line = s.nextLine(); - if (!line.startsWith("\"pool_assignment\":")) { - sb.append((linesWritten++ > 0 ? "\n" : "") + line); - endsWithComma = line.endsWith(","); - } - } - s.close(); - if (sb.length() > 0) { - sb.append((endsWithComma ? "" : ",") + "\n"); - } - } - sb.append("\"pool_assignment\":\"" + details + "\""); - detailsStatus = new DetailsStatus(); - detailsStatus.documentString = sb.toString(); - this.documentStore.store(detailsStatus, fingerprint); - } - } - - private long now = System.currentTimeMillis(); - - private Map<String, Set<ExitListEntry>> exitListEntries = - new HashMap<String, Set<ExitListEntry>>(); - - private void processExitList(ExitList exitList) { - for (ExitListEntry exitListEntry : exitList.getExitListEntries()) { - if (exitListEntry.getScanMillis() < - this.now - 24L * 60L * 60L * 1000L) { - continue; - } - String fingerprint = exitListEntry.getFingerprint(); - if (!this.exitListEntries.containsKey(fingerprint)) { - this.exitListEntries.put(fingerprint, - new HashSet<ExitListEntry>()); - } - this.exitListEntries.get(fingerprint).add(exitListEntry); - } - } - - public void setCurrentNodes( - SortedMap<String, NodeStatus> currentNodes) { - this.relays = new TreeMap<String, NodeStatus>(); - this.bridges = new TreeMap<String, NodeStatus>(); - for (Map.Entry<String, NodeStatus> e : currentNodes.entrySet()) { - if (e.getValue().isRelay()) { - this.relays.put(e.getKey(), e.getValue()); - } else { - this.bridges.put(e.getKey(), e.getValue()); - } - } - } - - public void startReverseDomainNameLookups() { - Map<String, Long> addressLastLookupTimes = - new HashMap<String, Long>(); - for (NodeStatus relay : relays.values()) { - addressLastLookupTimes.put(relay.getAddress(), - relay.getLastRdnsLookup()); - } - this.reverseDomainNameResolver.setAddresses(addressLastLookupTimes); - this.reverseDomainNameResolver.startReverseDomainNameLookups(); - } - - public void finishReverseDomainNameLookups() { - this.reverseDomainNameResolver.finishReverseDomainNameLookups(); - Map<String, String> lookupResults = - this.reverseDomainNameResolver.getLookupResults(); - long startedRdnsLookups = - this.reverseDomainNameResolver.getLookupStartMillis(); - for (NodeStatus relay : relays.values()) { - if (lookupResults.containsKey(relay.getAddress())) { - relay.setHostName(lookupResults.get(relay.getAddress())); - relay.setLastRdnsLookup(startedRdnsLookups); - } - } - } - - public void calculatePathSelectionProbabilities( - SortedMap<String, Integer> bandwidthWeights) { - boolean consensusContainsBandwidthWeights = false; - double wgg = 0.0, wgd = 0.0, wmg = 0.0, wmm = 0.0, wme = 0.0, - wmd = 0.0, wee = 0.0, wed = 0.0; - if (bandwidthWeights != null) { - SortedSet<String> weightKeys = new TreeSet<String>(Arrays.asList( - "Wgg,Wgd,Wmg,Wmm,Wme,Wmd,Wee,Wed".split(","))); - weightKeys.removeAll(bandwidthWeights.keySet()); - if (weightKeys.isEmpty()) { - consensusContainsBandwidthWeights = true; - wgg = ((double) bandwidthWeights.get("Wgg")) / 10000.0; - wgd = ((double) bandwidthWeights.get("Wgd")) / 10000.0; - wmg = ((double) bandwidthWeights.get("Wmg")) / 10000.0; - wmm = ((double) bandwidthWeights.get("Wmm")) / 10000.0; - wme = ((double) bandwidthWeights.get("Wme")) / 10000.0; - wmd = ((double) bandwidthWeights.get("Wmd")) / 10000.0; - wee = ((double) bandwidthWeights.get("Wee")) / 10000.0; - wed = ((double) bandwidthWeights.get("Wed")) / 10000.0; - } - } else { - System.err.println("Could not determine most recent Wxx parameter " - + "values, probably because we didn't parse a consensus in " - + "this execution. All relays' guard/middle/exit weights are " - + "going to be 0.0."); - } - SortedMap<String, Double> - advertisedBandwidths = new TreeMap<String, Double>(), - consensusWeights = new TreeMap<String, Double>(), - guardWeights = new TreeMap<String, Double>(), - middleWeights = new TreeMap<String, Double>(), - exitWeights = new TreeMap<String, Double>(); - double totalAdvertisedBandwidth = 0.0; - double totalConsensusWeight = 0.0; - double totalGuardWeight = 0.0; - double totalMiddleWeight = 0.0; - double totalExitWeight = 0.0; - for (Map.Entry<String, NodeStatus> e : this.relays.entrySet()) { - String fingerprint = e.getKey(); - NodeStatus relay = e.getValue(); - if (!relay.getRunning()) { - continue; - } - boolean isExit = relay.getRelayFlags().contains("Exit") && - !relay.getRelayFlags().contains("BadExit"); - boolean isGuard = relay.getRelayFlags().contains("Guard"); - DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, false, fingerprint); - if (detailsStatus != null) { - double advertisedBandwidth = -1.0; - String detailsString = detailsStatus.documentString; - Scanner s = new Scanner(detailsString); - while (s.hasNextLine()) { - String line = s.nextLine(); - if (!line.startsWith("\"advertised_bandwidth\":")) { - continue; - } - try { - advertisedBandwidth = (double) Integer.parseInt( - line.split(":")[1].replaceAll(",", "")); - } catch (NumberFormatException ex) { - /* Handle below. */ - } - break; - } - s.close(); - if (advertisedBandwidth >= 0.0) { - advertisedBandwidths.put(fingerprint, advertisedBandwidth); - totalAdvertisedBandwidth += advertisedBandwidth; - } - } - double consensusWeight = (double) relay.getConsensusWeight(); - consensusWeights.put(fingerprint, consensusWeight); - totalConsensusWeight += consensusWeight; - if (consensusContainsBandwidthWeights) { - double guardWeight = consensusWeight, - middleWeight = consensusWeight, - exitWeight = consensusWeight; - if (isGuard && isExit) { - guardWeight *= wgd; - middleWeight *= wmd; - exitWeight *= wed; - } else if (isGuard) { - guardWeight *= wgg; - middleWeight *= wmg; - exitWeight = 0.0; - } else if (isExit) { - guardWeight = 0.0; - middleWeight *= wme; - exitWeight *= wee; - } else { - guardWeight = 0.0; - middleWeight *= wmm; - exitWeight = 0.0; - } - guardWeights.put(fingerprint, guardWeight); - middleWeights.put(fingerprint, middleWeight); - exitWeights.put(fingerprint, exitWeight); - totalGuardWeight += guardWeight; - totalMiddleWeight += middleWeight; - totalExitWeight += exitWeight; - } - } - for (Map.Entry<String, NodeStatus> e : this.relays.entrySet()) { - String fingerprint = e.getKey(); - NodeStatus relay = e.getValue(); - if (advertisedBandwidths.containsKey(fingerprint)) { - relay.setAdvertisedBandwidthFraction(advertisedBandwidths.get( - fingerprint) / totalAdvertisedBandwidth); - } - if (consensusWeights.containsKey(fingerprint)) { - relay.setConsensusWeightFraction(consensusWeights.get(fingerprint) - / totalConsensusWeight); - } - if (guardWeights.containsKey(fingerprint)) { - relay.setGuardProbability(guardWeights.get(fingerprint) - / totalGuardWeight); - } - if (middleWeights.containsKey(fingerprint)) { - relay.setMiddleProbability(middleWeights.get(fingerprint) - / totalMiddleWeight); - } - if (exitWeights.containsKey(fingerprint)) { - relay.setExitProbability(exitWeights.get(fingerprint) - / totalExitWeight); - } - } - } - - public void writeOutDetails() { - this.updateRelayDetailsFiles(); - this.updateBridgeDetailsFiles(); - } - - private static String escapeJSON(String s) { - return StringEscapeUtils.escapeJavaScript(s).replaceAll("\\\\'", "'"); - } - - private static String unescapeJSON(String s) { - return StringEscapeUtils.unescapeJavaScript(s.replaceAll("'", "\\'")); - } - - private void updateRelayDetailsFiles() { - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - for (Map.Entry<String, NodeStatus> relay : this.relays.entrySet()) { - String fingerprint = relay.getKey(); - - /* Generate network-status-specific part. */ - NodeStatus entry = relay.getValue(); - String nickname = entry.getNickname(); - String address = entry.getAddress(); - List<String> orAddresses = new ArrayList<String>(); - orAddresses.add(address + ":" + entry.getOrPort()); - orAddresses.addAll(entry.getOrAddressesAndPorts()); - StringBuilder orAddressesAndPortsBuilder = new StringBuilder(); - int addressesWritten = 0; - for (String orAddress : orAddresses) { - orAddressesAndPortsBuilder.append( - (addressesWritten++ > 0 ? "," : "") + "\"" - + orAddress.toLowerCase() + "\""); - } - String lastSeen = dateTimeFormat.format(entry.getLastSeenMillis()); - String firstSeen = dateTimeFormat.format( - entry.getFirstSeenMillis()); - String lastChangedOrAddress = dateTimeFormat.format( - entry.getLastChangedOrAddress()); - String running = entry.getRunning() ? "true" : "false"; - int dirPort = entry.getDirPort(); - String countryCode = entry.getCountryCode(); - String latitude = entry.getLatitude(); - String longitude = entry.getLongitude(); - String countryName = entry.getCountryName(); - String regionName = entry.getRegionName(); - String cityName = entry.getCityName(); - String aSNumber = entry.getASNumber(); - String aSName = entry.getASName(); - long consensusWeight = entry.getConsensusWeight(); - String hostName = entry.getHostName(); - double advertisedBandwidthFraction = - entry.getAdvertisedBandwidthFraction(); - double consensusWeightFraction = entry.getConsensusWeightFraction(); - double guardProbability = entry.getGuardProbability(); - double middleProbability = entry.getMiddleProbability(); - double exitProbability = entry.getExitProbability(); - String defaultPolicy = entry.getDefaultPolicy(); - String portList = entry.getPortList(); - StringBuilder sb = new StringBuilder(); - sb.append("{\"version\":1,\n" - + "\"nickname\":\"" + nickname + "\",\n" - + "\"fingerprint\":\"" + fingerprint + "\",\n" - + "\"or_addresses\":[" + orAddressesAndPortsBuilder.toString() - + "]"); - if (dirPort != 0) { - sb.append(",\n\"dir_address\":\"" + address + ":" + dirPort - + "\""); - } - sb.append(",\n\"last_seen\":\"" + lastSeen + "\""); - sb.append(",\n\"first_seen\":\"" + firstSeen + "\""); - sb.append(",\n\"last_changed_address_or_port\":\"" - + lastChangedOrAddress + "\""); - sb.append(",\n\"running\":" + running); - SortedSet<String> relayFlags = entry.getRelayFlags(); - if (!relayFlags.isEmpty()) { - sb.append(",\n\"flags\":["); - int written = 0; - for (String relayFlag : relayFlags) { - sb.append((written++ > 0 ? "," : "") + "\"" + relayFlag + "\""); - } - sb.append("]"); - } - if (countryCode != null) { - sb.append(",\n\"country\":\"" + countryCode + "\""); - } - if (latitude != null) { - sb.append(",\n\"latitude\":" + latitude); - } - if (longitude != null) { - sb.append(",\n\"longitude\":" + longitude); - } - if (countryName != null) { - sb.append(",\n\"country_name\":\"" - + escapeJSON(countryName) + "\""); - } - if (regionName != null) { - sb.append(",\n\"region_name\":\"" - + escapeJSON(regionName) + "\""); - } - if (cityName != null) { - sb.append(",\n\"city_name\":\"" - + escapeJSON(cityName) + "\""); - } - if (aSNumber != null) { - sb.append(",\n\"as_number\":\"" - + escapeJSON(aSNumber) + "\""); - } - if (aSName != null) { - sb.append(",\n\"as_name\":\"" - + escapeJSON(aSName) + "\""); - } - if (consensusWeight >= 0L) { - sb.append(",\n\"consensus_weight\":" - + String.valueOf(consensusWeight)); - } - if (hostName != null) { - sb.append(",\n\"host_name\":\"" - + escapeJSON(hostName) + "\""); - } - if (advertisedBandwidthFraction >= 0.0) { - sb.append(String.format( - ",\n\"advertised_bandwidth_fraction\":%.9f", - advertisedBandwidthFraction)); - } - if (consensusWeightFraction >= 0.0) { - sb.append(String.format(",\n\"consensus_weight_fraction\":%.9f", - consensusWeightFraction)); - } - if (guardProbability >= 0.0) { - sb.append(String.format(",\n\"guard_probability\":%.9f", - guardProbability)); - } - if (middleProbability >= 0.0) { - sb.append(String.format(",\n\"middle_probability\":%.9f", - middleProbability)); - } - if (exitProbability >= 0.0) { - sb.append(String.format(",\n\"exit_probability\":%.9f", - exitProbability)); - } - if (defaultPolicy != null && (defaultPolicy.equals("accept") || - defaultPolicy.equals("reject")) && portList != null) { - sb.append(",\n\"exit_policy_summary\":{\"" + defaultPolicy - + "\":["); - int portsWritten = 0; - for (String portOrPortRange : portList.split(",")) { - sb.append((portsWritten++ > 0 ? "," : "") - + "\"" + portOrPortRange + "\""); - } - sb.append("]}"); - } - - /* Add exit addresses if at least one of them is distinct from the - * onion-routing addresses. */ - if (exitListEntries.containsKey(fingerprint)) { - for (ExitListEntry exitListEntry : - exitListEntries.get(fingerprint)) { - entry.addExitAddress(exitListEntry.getExitAddress()); - } - } - if (!entry.getExitAddresses().isEmpty()) { - sb.append(",\n\"exit_addresses\":["); - int written = 0; - for (String exitAddress : entry.getExitAddresses()) { - sb.append((written++ > 0 ? "," : "") + "\"" - + exitAddress.toLowerCase() + "\""); - } - sb.append("]"); - } - - /* Append descriptor-specific part from details status file, and - * update contact in node status. */ - DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, false, fingerprint); - if (detailsStatus != null && - detailsStatus.documentString.length() > 0) { - sb.append(",\n" + detailsStatus.documentString); - String contact = null; - Scanner s = new Scanner(detailsStatus.documentString); - while (s.hasNextLine()) { - String line = s.nextLine(); - if (!line.startsWith("\"contact\":")) { - continue; - } - int start = "\"contact\":\"".length(), end = line.length() - 1; - if (line.endsWith(",")) { - end--; - } - contact = unescapeJSON(line.substring(start, end)); - break; - } - s.close(); - entry.setContact(contact); - } - - /* Finish details string. */ - sb.append("\n}\n"); - - /* Write details file to disk. */ - DetailsDocument detailsDocument = new DetailsDocument(); - detailsDocument.documentString = sb.toString(); - this.documentStore.store(detailsDocument, fingerprint); - } - } - - private void updateBridgeDetailsFiles() { - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - for (Map.Entry<String, NodeStatus> bridge : this.bridges.entrySet()) { - String fingerprint = bridge.getKey(); - - /* Generate network-status-specific part. */ - NodeStatus entry = bridge.getValue(); - String nickname = entry.getNickname(); - String lastSeen = dateTimeFormat.format(entry.getLastSeenMillis()); - String firstSeen = dateTimeFormat.format( - entry.getFirstSeenMillis()); - String running = entry.getRunning() ? "true" : "false"; - String address = entry.getAddress(); - List<String> orAddresses = new ArrayList<String>(); - orAddresses.add(address + ":" + entry.getOrPort()); - orAddresses.addAll(entry.getOrAddressesAndPorts()); - StringBuilder orAddressesAndPortsBuilder = new StringBuilder(); - int addressesWritten = 0; - for (String orAddress : orAddresses) { - orAddressesAndPortsBuilder.append( - (addressesWritten++ > 0 ? "," : "") + "\"" - + orAddress.toLowerCase() + "\""); - } - StringBuilder sb = new StringBuilder(); - sb.append("{\"version\":1,\n" - + "\"nickname\":\"" + nickname + "\",\n" - + "\"hashed_fingerprint\":\"" + fingerprint + "\",\n" - + "\"or_addresses\":[" + orAddressesAndPortsBuilder.toString() - + "],\n\"last_seen\":\"" + lastSeen + "\",\n\"first_seen\":\"" - + firstSeen + "\",\n\"running\":" + running); - - SortedSet<String> relayFlags = entry.getRelayFlags(); - if (!relayFlags.isEmpty()) { - sb.append(",\n\"flags\":["); - int written = 0; - for (String relayFlag : relayFlags) { - sb.append((written++ > 0 ? "," : "") + "\"" + relayFlag + "\""); - } - sb.append("]"); - } - - /* Append descriptor-specific part from details status file. */ - DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, false, fingerprint); - if (detailsStatus != null && - detailsStatus.documentString.length() > 0) { - sb.append(",\n" + detailsStatus.documentString); - } - - /* Finish details string. */ - sb.append("\n}\n"); - - /* Write details file to disk. */ - DetailsDocument detailsDocument = new DetailsDocument(); - detailsDocument.documentString = sb.toString(); - this.documentStore.store(detailsDocument, fingerprint); - } - } -} - diff --git a/src/org/torproject/onionoo/Main.java b/src/org/torproject/onionoo/Main.java index a03a9fe..a62918b 100644 --- a/src/org/torproject/onionoo/Main.java +++ b/src/org/torproject/onionoo/Main.java @@ -3,7 +3,6 @@ package org.torproject.onionoo; import java.io.File; -import java.util.SortedMap; /* Update search data and status data files. */ public class Main { @@ -30,10 +29,8 @@ public class Main { Logger.printStatusTime("Initialized Geoip lookup service"); ReverseDomainNameResolver rdnr = new ReverseDomainNameResolver(); Logger.printStatusTime("Initialized reverse domain name resolver"); - NodeDataWriter ndw = new NodeDataWriter(dso, ls, ds); + NodeDataWriter ndw = new NodeDataWriter(dso, rdnr, ls, ds); Logger.printStatusTime("Initialized node data writer"); - DetailsDataWriter ddw = new DetailsDataWriter(dso, rdnr, ds); - Logger.printStatusTime("Initialized details data writer"); BandwidthDataWriter bdw = new BandwidthDataWriter(dso, ds); Logger.printStatusTime("Initialized bandwidth data writer"); WeightsDataWriter wdw = new WeightsDataWriter(dso, ds); @@ -60,32 +57,23 @@ public class Main { Logger.printStatus("Updating internal node list."); ndw.readStatusSummary(); Logger.printStatusTime("Read status summary"); + ndw.setCurrentNodes(); + Logger.printStatusTime("Set current node fingerprints"); + ndw.startReverseDomainNameLookups(); + Logger.printStatusTime("Started reverse domain name lookups"); ndw.lookUpCitiesAndASes(); Logger.printStatusTime("Looked up cities and ASes"); ndw.setRunningBits(); Logger.printStatusTime("Set running bits"); + ndw.calculatePathSelectionProbabilities(); + Logger.printStatusTime("Calculated path selection probabilities"); + ndw.finishReverseDomainNameLookups(); + Logger.printStatusTime("Finished reverse domain name lookups"); ndw.writeStatusSummary(); Logger.printStatusTime("Wrote status summary"); ndw.writeOutSummary(); Logger.printStatusTime("Wrote out summary"); - SortedMap<String, NodeStatus> currentNodes = ndw.getCurrentNodes(); - SortedMap<String, Integer> lastBandwidthWeights = - ndw.getLastBandwidthWeights(); - - Logger.printStatus("Updating detail data."); - // TODO Instead of using ndw's currentNodes and lastBandwidthWeights, - // parse statuses once again, keeping separate parse history. Allows - // us to run ndw and ddw in parallel in the future. Alternatively, - // merge ndw and ddw, because they're doing similar things anyway. - ddw.setCurrentNodes(currentNodes); - Logger.printStatusTime("Set current node fingerprints"); - ddw.startReverseDomainNameLookups(); - Logger.printStatusTime("Started reverse domain name lookups"); - ddw.calculatePathSelectionProbabilities(lastBandwidthWeights); - Logger.printStatusTime("Calculated path selection probabilities"); - ddw.finishReverseDomainNameLookups(); - Logger.printStatusTime("Finished reverse domain name lookups"); - ddw.writeOutDetails(); + ndw.writeOutDetails(); Logger.printStatusTime("Wrote detail data files"); Logger.printStatus("Updating bandwidth data."); diff --git a/src/org/torproject/onionoo/NodeDataWriter.java b/src/org/torproject/onionoo/NodeDataWriter.java index 56071f4..480f18b 100644 --- a/src/org/torproject/onionoo/NodeDataWriter.java +++ b/src/org/torproject/onionoo/NodeDataWriter.java @@ -2,24 +2,43 @@ * See LICENSE for licensing information */ package org.torproject.onionoo; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Scanner; +import java.util.Set; import java.util.SortedMap; import java.util.SortedSet; +import java.util.TimeZone; import java.util.TreeMap; import java.util.TreeSet; +import org.apache.commons.lang.StringEscapeUtils; import org.torproject.descriptor.BridgeNetworkStatus; +import org.torproject.descriptor.BridgePoolAssignment; import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.ExitList; +import org.torproject.descriptor.ExitListEntry; import org.torproject.descriptor.NetworkStatusEntry; import org.torproject.descriptor.RelayNetworkStatusConsensus; +import org.torproject.descriptor.ServerDescriptor; import org.torproject.onionoo.LookupService.LookupResult; -/* Store relays and bridges that have been running in the past seven - * days. */ +/* Write updated summary and details data files to disk. + * + * The parts of details files coming from server descriptors always come + * from the last known descriptor of a relay or bridge, not from the + * descriptor that was last referenced in a network status. */ public class NodeDataWriter implements DescriptorListener { private DescriptorSource descriptorSource; + private ReverseDomainNameResolver reverseDomainNameResolver; + private LookupService lookupService; private DocumentStore documentStore; @@ -27,6 +46,10 @@ public class NodeDataWriter implements DescriptorListener { private SortedMap<String, NodeStatus> knownNodes = new TreeMap<String, NodeStatus>(); + private SortedMap<String, NodeStatus> relays; + + private SortedMap<String, NodeStatus> bridges; + private long relaysLastValidAfterMillis = -1L; private long bridgesLastPublishedMillis = -1L; @@ -36,8 +59,10 @@ public class NodeDataWriter implements DescriptorListener { private int relayConsensusesProcessed = 0, bridgeStatusesProcessed = 0; public NodeDataWriter(DescriptorSource descriptorSource, + ReverseDomainNameResolver reverseDomainNameResolver, LookupService lookupService, DocumentStore documentStore) { this.descriptorSource = descriptorSource; + this.reverseDomainNameResolver = reverseDomainNameResolver; this.lookupService = lookupService; this.documentStore = documentStore; this.registerDescriptorListeners(); @@ -47,15 +72,31 @@ public class NodeDataWriter implements DescriptorListener { this.descriptorSource.registerListener(this, DescriptorType.RELAY_CONSENSUSES); this.descriptorSource.registerListener(this, + DescriptorType.RELAY_SERVER_DESCRIPTORS); + this.descriptorSource.registerListener(this, DescriptorType.BRIDGE_STATUSES); + this.descriptorSource.registerListener(this, + DescriptorType.BRIDGE_SERVER_DESCRIPTORS); + this.descriptorSource.registerListener(this, + DescriptorType.BRIDGE_POOL_ASSIGNMENTS); + this.descriptorSource.registerListener(this, + DescriptorType.EXIT_LISTS); } public void processDescriptor(Descriptor descriptor, boolean relay) { if (descriptor instanceof RelayNetworkStatusConsensus) { updateRelayNetworkStatusConsensus( (RelayNetworkStatusConsensus) descriptor); + } else if (descriptor instanceof ServerDescriptor && relay) { + this.processRelayServerDescriptor((ServerDescriptor) descriptor); } else if (descriptor instanceof BridgeNetworkStatus) { updateBridgeNetworkStatus((BridgeNetworkStatus) descriptor); + } else if (descriptor instanceof ServerDescriptor && !relay) { + this.processBridgeServerDescriptor((ServerDescriptor) descriptor); + } else if (descriptor instanceof BridgePoolAssignment) { + this.processBridgePoolAssignment((BridgePoolAssignment) descriptor); + } else if (descriptor instanceof ExitList) { + this.processExitList((ExitList) descriptor); } } @@ -218,8 +259,608 @@ public class NodeDataWriter implements DescriptorListener { return currentNodes; } - public SortedMap<String, Integer> getLastBandwidthWeights() { - return this.lastBandwidthWeights; + private void processRelayServerDescriptor( + ServerDescriptor descriptor) { + String fingerprint = descriptor.getFingerprint(); + DetailsStatus detailsStatus = this.documentStore.retrieve( + DetailsStatus.class, false, fingerprint); + SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setLenient(false); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + String publishedDateTime = + dateTimeFormat.format(descriptor.getPublishedMillis()); + if (detailsStatus != null) { + String detailsString = detailsStatus.documentString; + String descPublishedLine = "\"desc_published\":\"" + + publishedDateTime + "\","; + Scanner s = new Scanner(detailsString); + while (s.hasNextLine()) { + String line = s.nextLine(); + if (line.startsWith("\"desc_published\":\"")) { + if (descPublishedLine.compareTo(line) < 0) { + return; + } else { + break; + } + } + } + s.close(); + } + StringBuilder sb = new StringBuilder(); + String lastRestartedString = dateTimeFormat.format( + descriptor.getPublishedMillis() - descriptor.getUptime() * 1000L); + int bandwidthRate = descriptor.getBandwidthRate(); + int bandwidthBurst = descriptor.getBandwidthBurst(); + int observedBandwidth = descriptor.getBandwidthObserved(); + int advertisedBandwidth = Math.min(bandwidthRate, + Math.min(bandwidthBurst, observedBandwidth)); + sb.append("\"desc_published\":\"" + publishedDateTime + "\",\n" + + "\"last_restarted\":\"" + lastRestartedString + "\",\n" + + "\"bandwidth_rate\":" + bandwidthRate + ",\n" + + "\"bandwidth_burst\":" + bandwidthBurst + ",\n" + + "\"observed_bandwidth\":" + observedBandwidth + ",\n" + + "\"advertised_bandwidth\":" + advertisedBandwidth + ",\n" + + "\"exit_policy\":["); + int written = 0; + for (String exitPolicyLine : descriptor.getExitPolicyLines()) { + sb.append((written++ > 0 ? "," : "") + "\n \"" + exitPolicyLine + + "\""); + } + sb.append("\n]"); + if (descriptor.getContact() != null) { + sb.append(",\n\"contact\":\"" + + escapeJSON(descriptor.getContact()) + "\""); + } + if (descriptor.getPlatform() != null) { + sb.append(",\n\"platform\":\"" + + escapeJSON(descriptor.getPlatform()) + "\""); + } + if (descriptor.getFamilyEntries() != null) { + sb.append(",\n\"family\":["); + written = 0; + for (String familyEntry : descriptor.getFamilyEntries()) { + sb.append((written++ > 0 ? "," : "") + "\n \"" + familyEntry + + "\""); + } + sb.append("\n]"); + } + detailsStatus = new DetailsStatus(); + detailsStatus.documentString = sb.toString(); + this.documentStore.store(detailsStatus, fingerprint); + } + + private void processBridgeServerDescriptor( + ServerDescriptor descriptor) { + String fingerprint = descriptor.getFingerprint(); + DetailsStatus detailsStatus = this.documentStore.retrieve( + DetailsStatus.class, false, fingerprint); + SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setLenient(false); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + String publishedDateTime = + dateTimeFormat.format(descriptor.getPublishedMillis()); + String poolAssignmentLine = null; + if (detailsStatus != null) { + String detailsString = detailsStatus.documentString; + String descPublishedLine = "\"desc_published\":\"" + + publishedDateTime + "\","; + Scanner s = new Scanner(detailsString); + while (s.hasNextLine()) { + String line = s.nextLine(); + if (line.startsWith("\"pool_assignment\":")) { + poolAssignmentLine = line; + } else if (line.startsWith("\"desc_published\":") && + descPublishedLine.compareTo(line) < 0) { + return; + } + } + s.close(); + } + StringBuilder sb = new StringBuilder(); + String lastRestartedString = dateTimeFormat.format( + descriptor.getPublishedMillis() - descriptor.getUptime() * 1000L); + int advertisedBandwidth = Math.min(descriptor.getBandwidthRate(), + Math.min(descriptor.getBandwidthBurst(), + descriptor.getBandwidthObserved())); + sb.append("\"desc_published\":\"" + publishedDateTime + "\",\n" + + "\"last_restarted\":\"" + lastRestartedString + "\",\n" + + "\"advertised_bandwidth\":" + advertisedBandwidth + ",\n" + + "\"platform\":\"" + escapeJSON(descriptor.getPlatform()) + + "\""); + if (poolAssignmentLine != null) { + sb.append(",\n" + poolAssignmentLine); + } + detailsStatus = new DetailsStatus(); + detailsStatus.documentString = sb.toString(); + this.documentStore.store(detailsStatus, fingerprint); + } + + private void processBridgePoolAssignment( + BridgePoolAssignment bridgePoolAssignment) { + for (Map.Entry<String, String> e : + bridgePoolAssignment.getEntries().entrySet()) { + String fingerprint = e.getKey(); + String details = e.getValue(); + StringBuilder sb = new StringBuilder(); + DetailsStatus detailsStatus = this.documentStore.retrieve( + DetailsStatus.class, false, fingerprint); + if (detailsStatus != null) { + String detailsString = detailsStatus.documentString; + Scanner s = new Scanner(detailsString); + int linesWritten = 0; + boolean endsWithComma = false; + while (s.hasNextLine()) { + String line = s.nextLine(); + if (!line.startsWith("\"pool_assignment\":")) { + sb.append((linesWritten++ > 0 ? "\n" : "") + line); + endsWithComma = line.endsWith(","); + } + } + s.close(); + if (sb.length() > 0) { + sb.append((endsWithComma ? "" : ",") + "\n"); + } + } + sb.append("\"pool_assignment\":\"" + details + "\""); + detailsStatus = new DetailsStatus(); + detailsStatus.documentString = sb.toString(); + this.documentStore.store(detailsStatus, fingerprint); + } + } + + private long now = System.currentTimeMillis(); + + private Map<String, Set<ExitListEntry>> exitListEntries = + new HashMap<String, Set<ExitListEntry>>(); + + private void processExitList(ExitList exitList) { + for (ExitListEntry exitListEntry : exitList.getExitListEntries()) { + if (exitListEntry.getScanMillis() < + this.now - 24L * 60L * 60L * 1000L) { + continue; + } + String fingerprint = exitListEntry.getFingerprint(); + if (!this.exitListEntries.containsKey(fingerprint)) { + this.exitListEntries.put(fingerprint, + new HashSet<ExitListEntry>()); + } + this.exitListEntries.get(fingerprint).add(exitListEntry); + } + } + + public void setCurrentNodes() { + SortedMap<String, NodeStatus> currentNodes = this.getCurrentNodes(); + this.relays = new TreeMap<String, NodeStatus>(); + this.bridges = new TreeMap<String, NodeStatus>(); + for (Map.Entry<String, NodeStatus> e : currentNodes.entrySet()) { + if (e.getValue().isRelay()) { + this.relays.put(e.getKey(), e.getValue()); + } else { + this.bridges.put(e.getKey(), e.getValue()); + } + } + } + + public void startReverseDomainNameLookups() { + Map<String, Long> addressLastLookupTimes = + new HashMap<String, Long>(); + for (NodeStatus relay : relays.values()) { + addressLastLookupTimes.put(relay.getAddress(), + relay.getLastRdnsLookup()); + } + this.reverseDomainNameResolver.setAddresses(addressLastLookupTimes); + this.reverseDomainNameResolver.startReverseDomainNameLookups(); + } + + public void finishReverseDomainNameLookups() { + this.reverseDomainNameResolver.finishReverseDomainNameLookups(); + Map<String, String> lookupResults = + this.reverseDomainNameResolver.getLookupResults(); + long startedRdnsLookups = + this.reverseDomainNameResolver.getLookupStartMillis(); + for (NodeStatus relay : relays.values()) { + if (lookupResults.containsKey(relay.getAddress())) { + relay.setHostName(lookupResults.get(relay.getAddress())); + relay.setLastRdnsLookup(startedRdnsLookups); + } + } + } + + public void calculatePathSelectionProbabilities() { + boolean consensusContainsBandwidthWeights = false; + double wgg = 0.0, wgd = 0.0, wmg = 0.0, wmm = 0.0, wme = 0.0, + wmd = 0.0, wee = 0.0, wed = 0.0; + if (this.lastBandwidthWeights != null) { + SortedSet<String> weightKeys = new TreeSet<String>(Arrays.asList( + "Wgg,Wgd,Wmg,Wmm,Wme,Wmd,Wee,Wed".split(","))); + weightKeys.removeAll(this.lastBandwidthWeights.keySet()); + if (weightKeys.isEmpty()) { + consensusContainsBandwidthWeights = true; + wgg = ((double) this.lastBandwidthWeights.get("Wgg")) / 10000.0; + wgd = ((double) this.lastBandwidthWeights.get("Wgd")) / 10000.0; + wmg = ((double) this.lastBandwidthWeights.get("Wmg")) / 10000.0; + wmm = ((double) this.lastBandwidthWeights.get("Wmm")) / 10000.0; + wme = ((double) this.lastBandwidthWeights.get("Wme")) / 10000.0; + wmd = ((double) this.lastBandwidthWeights.get("Wmd")) / 10000.0; + wee = ((double) this.lastBandwidthWeights.get("Wee")) / 10000.0; + wed = ((double) this.lastBandwidthWeights.get("Wed")) / 10000.0; + } + } else { + System.err.println("Could not determine most recent Wxx parameter " + + "values, probably because we didn't parse a consensus in " + + "this execution. All relays' guard/middle/exit weights are " + + "going to be 0.0."); + } + SortedMap<String, Double> + advertisedBandwidths = new TreeMap<String, Double>(), + consensusWeights = new TreeMap<String, Double>(), + guardWeights = new TreeMap<String, Double>(), + middleWeights = new TreeMap<String, Double>(), + exitWeights = new TreeMap<String, Double>(); + double totalAdvertisedBandwidth = 0.0; + double totalConsensusWeight = 0.0; + double totalGuardWeight = 0.0; + double totalMiddleWeight = 0.0; + double totalExitWeight = 0.0; + for (Map.Entry<String, NodeStatus> e : this.relays.entrySet()) { + String fingerprint = e.getKey(); + NodeStatus relay = e.getValue(); + if (!relay.getRunning()) { + continue; + } + boolean isExit = relay.getRelayFlags().contains("Exit") && + !relay.getRelayFlags().contains("BadExit"); + boolean isGuard = relay.getRelayFlags().contains("Guard"); + DetailsStatus detailsStatus = this.documentStore.retrieve( + DetailsStatus.class, false, fingerprint); + if (detailsStatus != null) { + double advertisedBandwidth = -1.0; + String detailsString = detailsStatus.documentString; + Scanner s = new Scanner(detailsString); + while (s.hasNextLine()) { + String line = s.nextLine(); + if (!line.startsWith("\"advertised_bandwidth\":")) { + continue; + } + try { + advertisedBandwidth = (double) Integer.parseInt( + line.split(":")[1].replaceAll(",", "")); + } catch (NumberFormatException ex) { + /* Handle below. */ + } + break; + } + s.close(); + if (advertisedBandwidth >= 0.0) { + advertisedBandwidths.put(fingerprint, advertisedBandwidth); + totalAdvertisedBandwidth += advertisedBandwidth; + } + } + double consensusWeight = (double) relay.getConsensusWeight(); + consensusWeights.put(fingerprint, consensusWeight); + totalConsensusWeight += consensusWeight; + if (consensusContainsBandwidthWeights) { + double guardWeight = consensusWeight, + middleWeight = consensusWeight, + exitWeight = consensusWeight; + if (isGuard && isExit) { + guardWeight *= wgd; + middleWeight *= wmd; + exitWeight *= wed; + } else if (isGuard) { + guardWeight *= wgg; + middleWeight *= wmg; + exitWeight = 0.0; + } else if (isExit) { + guardWeight = 0.0; + middleWeight *= wme; + exitWeight *= wee; + } else { + guardWeight = 0.0; + middleWeight *= wmm; + exitWeight = 0.0; + } + guardWeights.put(fingerprint, guardWeight); + middleWeights.put(fingerprint, middleWeight); + exitWeights.put(fingerprint, exitWeight); + totalGuardWeight += guardWeight; + totalMiddleWeight += middleWeight; + totalExitWeight += exitWeight; + } + } + for (Map.Entry<String, NodeStatus> e : this.relays.entrySet()) { + String fingerprint = e.getKey(); + NodeStatus relay = e.getValue(); + if (advertisedBandwidths.containsKey(fingerprint)) { + relay.setAdvertisedBandwidthFraction(advertisedBandwidths.get( + fingerprint) / totalAdvertisedBandwidth); + } + if (consensusWeights.containsKey(fingerprint)) { + relay.setConsensusWeightFraction(consensusWeights.get(fingerprint) + / totalConsensusWeight); + } + if (guardWeights.containsKey(fingerprint)) { + relay.setGuardProbability(guardWeights.get(fingerprint) + / totalGuardWeight); + } + if (middleWeights.containsKey(fingerprint)) { + relay.setMiddleProbability(middleWeights.get(fingerprint) + / totalMiddleWeight); + } + if (exitWeights.containsKey(fingerprint)) { + relay.setExitProbability(exitWeights.get(fingerprint) + / totalExitWeight); + } + } + } + + public void writeOutDetails() { + this.updateRelayDetailsFiles(); + this.updateBridgeDetailsFiles(); + } + + private static String escapeJSON(String s) { + return StringEscapeUtils.escapeJavaScript(s).replaceAll("\\\\'", "'"); + } + + private static String unescapeJSON(String s) { + return StringEscapeUtils.unescapeJavaScript(s.replaceAll("'", "\\'")); + } + + private void updateRelayDetailsFiles() { + SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + for (Map.Entry<String, NodeStatus> relay : this.relays.entrySet()) { + String fingerprint = relay.getKey(); + + /* Generate network-status-specific part. */ + NodeStatus entry = relay.getValue(); + String nickname = entry.getNickname(); + String address = entry.getAddress(); + List<String> orAddresses = new ArrayList<String>(); + orAddresses.add(address + ":" + entry.getOrPort()); + orAddresses.addAll(entry.getOrAddressesAndPorts()); + StringBuilder orAddressesAndPortsBuilder = new StringBuilder(); + int addressesWritten = 0; + for (String orAddress : orAddresses) { + orAddressesAndPortsBuilder.append( + (addressesWritten++ > 0 ? "," : "") + "\"" + + orAddress.toLowerCase() + "\""); + } + String lastSeen = dateTimeFormat.format(entry.getLastSeenMillis()); + String firstSeen = dateTimeFormat.format( + entry.getFirstSeenMillis()); + String lastChangedOrAddress = dateTimeFormat.format( + entry.getLastChangedOrAddress()); + String running = entry.getRunning() ? "true" : "false"; + int dirPort = entry.getDirPort(); + String countryCode = entry.getCountryCode(); + String latitude = entry.getLatitude(); + String longitude = entry.getLongitude(); + String countryName = entry.getCountryName(); + String regionName = entry.getRegionName(); + String cityName = entry.getCityName(); + String aSNumber = entry.getASNumber(); + String aSName = entry.getASName(); + long consensusWeight = entry.getConsensusWeight(); + String hostName = entry.getHostName(); + double advertisedBandwidthFraction = + entry.getAdvertisedBandwidthFraction(); + double consensusWeightFraction = entry.getConsensusWeightFraction(); + double guardProbability = entry.getGuardProbability(); + double middleProbability = entry.getMiddleProbability(); + double exitProbability = entry.getExitProbability(); + String defaultPolicy = entry.getDefaultPolicy(); + String portList = entry.getPortList(); + StringBuilder sb = new StringBuilder(); + sb.append("{\"version\":1,\n" + + "\"nickname\":\"" + nickname + "\",\n" + + "\"fingerprint\":\"" + fingerprint + "\",\n" + + "\"or_addresses\":[" + orAddressesAndPortsBuilder.toString() + + "]"); + if (dirPort != 0) { + sb.append(",\n\"dir_address\":\"" + address + ":" + dirPort + + "\""); + } + sb.append(",\n\"last_seen\":\"" + lastSeen + "\""); + sb.append(",\n\"first_seen\":\"" + firstSeen + "\""); + sb.append(",\n\"last_changed_address_or_port\":\"" + + lastChangedOrAddress + "\""); + sb.append(",\n\"running\":" + running); + SortedSet<String> relayFlags = entry.getRelayFlags(); + if (!relayFlags.isEmpty()) { + sb.append(",\n\"flags\":["); + int written = 0; + for (String relayFlag : relayFlags) { + sb.append((written++ > 0 ? "," : "") + "\"" + relayFlag + "\""); + } + sb.append("]"); + } + if (countryCode != null) { + sb.append(",\n\"country\":\"" + countryCode + "\""); + } + if (latitude != null) { + sb.append(",\n\"latitude\":" + latitude); + } + if (longitude != null) { + sb.append(",\n\"longitude\":" + longitude); + } + if (countryName != null) { + sb.append(",\n\"country_name\":\"" + + escapeJSON(countryName) + "\""); + } + if (regionName != null) { + sb.append(",\n\"region_name\":\"" + + escapeJSON(regionName) + "\""); + } + if (cityName != null) { + sb.append(",\n\"city_name\":\"" + + escapeJSON(cityName) + "\""); + } + if (aSNumber != null) { + sb.append(",\n\"as_number\":\"" + + escapeJSON(aSNumber) + "\""); + } + if (aSName != null) { + sb.append(",\n\"as_name\":\"" + + escapeJSON(aSName) + "\""); + } + if (consensusWeight >= 0L) { + sb.append(",\n\"consensus_weight\":" + + String.valueOf(consensusWeight)); + } + if (hostName != null) { + sb.append(",\n\"host_name\":\"" + + escapeJSON(hostName) + "\""); + } + if (advertisedBandwidthFraction >= 0.0) { + sb.append(String.format( + ",\n\"advertised_bandwidth_fraction\":%.9f", + advertisedBandwidthFraction)); + } + if (consensusWeightFraction >= 0.0) { + sb.append(String.format(",\n\"consensus_weight_fraction\":%.9f", + consensusWeightFraction)); + } + if (guardProbability >= 0.0) { + sb.append(String.format(",\n\"guard_probability\":%.9f", + guardProbability)); + } + if (middleProbability >= 0.0) { + sb.append(String.format(",\n\"middle_probability\":%.9f", + middleProbability)); + } + if (exitProbability >= 0.0) { + sb.append(String.format(",\n\"exit_probability\":%.9f", + exitProbability)); + } + if (defaultPolicy != null && (defaultPolicy.equals("accept") || + defaultPolicy.equals("reject")) && portList != null) { + sb.append(",\n\"exit_policy_summary\":{\"" + defaultPolicy + + "\":["); + int portsWritten = 0; + for (String portOrPortRange : portList.split(",")) { + sb.append((portsWritten++ > 0 ? "," : "") + + "\"" + portOrPortRange + "\""); + } + sb.append("]}"); + } + + /* Add exit addresses if at least one of them is distinct from the + * onion-routing addresses. */ + if (exitListEntries.containsKey(fingerprint)) { + for (ExitListEntry exitListEntry : + exitListEntries.get(fingerprint)) { + entry.addExitAddress(exitListEntry.getExitAddress()); + } + } + if (!entry.getExitAddresses().isEmpty()) { + sb.append(",\n\"exit_addresses\":["); + int written = 0; + for (String exitAddress : entry.getExitAddresses()) { + sb.append((written++ > 0 ? "," : "") + "\"" + + exitAddress.toLowerCase() + "\""); + } + sb.append("]"); + } + + /* Append descriptor-specific part from details status file, and + * update contact in node status. */ + DetailsStatus detailsStatus = this.documentStore.retrieve( + DetailsStatus.class, false, fingerprint); + if (detailsStatus != null && + detailsStatus.documentString.length() > 0) { + sb.append(",\n" + detailsStatus.documentString); + String contact = null; + Scanner s = new Scanner(detailsStatus.documentString); + while (s.hasNextLine()) { + String line = s.nextLine(); + if (!line.startsWith("\"contact\":")) { + continue; + } + int start = "\"contact\":\"".length(), end = line.length() - 1; + if (line.endsWith(",")) { + end--; + } + contact = unescapeJSON(line.substring(start, end)); + break; + } + s.close(); + entry.setContact(contact); + } + + /* Finish details string. */ + sb.append("\n}\n"); + + /* Write details file to disk. */ + DetailsDocument detailsDocument = new DetailsDocument(); + detailsDocument.documentString = sb.toString(); + this.documentStore.store(detailsDocument, fingerprint); + } + } + + private void updateBridgeDetailsFiles() { + SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + for (Map.Entry<String, NodeStatus> bridge : this.bridges.entrySet()) { + String fingerprint = bridge.getKey(); + + /* Generate network-status-specific part. */ + NodeStatus entry = bridge.getValue(); + String nickname = entry.getNickname(); + String lastSeen = dateTimeFormat.format(entry.getLastSeenMillis()); + String firstSeen = dateTimeFormat.format( + entry.getFirstSeenMillis()); + String running = entry.getRunning() ? "true" : "false"; + String address = entry.getAddress(); + List<String> orAddresses = new ArrayList<String>(); + orAddresses.add(address + ":" + entry.getOrPort()); + orAddresses.addAll(entry.getOrAddressesAndPorts()); + StringBuilder orAddressesAndPortsBuilder = new StringBuilder(); + int addressesWritten = 0; + for (String orAddress : orAddresses) { + orAddressesAndPortsBuilder.append( + (addressesWritten++ > 0 ? "," : "") + "\"" + + orAddress.toLowerCase() + "\""); + } + StringBuilder sb = new StringBuilder(); + sb.append("{\"version\":1,\n" + + "\"nickname\":\"" + nickname + "\",\n" + + "\"hashed_fingerprint\":\"" + fingerprint + "\",\n" + + "\"or_addresses\":[" + orAddressesAndPortsBuilder.toString() + + "],\n\"last_seen\":\"" + lastSeen + "\",\n\"first_seen\":\"" + + firstSeen + "\",\n\"running\":" + running); + + SortedSet<String> relayFlags = entry.getRelayFlags(); + if (!relayFlags.isEmpty()) { + sb.append(",\n\"flags\":["); + int written = 0; + for (String relayFlag : relayFlags) { + sb.append((written++ > 0 ? "," : "") + "\"" + relayFlag + "\""); + } + sb.append("]"); + } + + /* Append descriptor-specific part from details status file. */ + DetailsStatus detailsStatus = this.documentStore.retrieve( + DetailsStatus.class, false, fingerprint); + if (detailsStatus != null && + detailsStatus.documentString.length() > 0) { + sb.append(",\n" + detailsStatus.documentString); + } + + /* Finish details string. */ + sb.append("\n}\n"); + + /* Write details file to disk. */ + DetailsDocument detailsDocument = new DetailsDocument(); + detailsDocument.documentString = sb.toString(); + this.documentStore.store(detailsDocument, fingerprint); + } } public String getStatsString() {

1 0

[onionoo/master] Add note how to improve tests.
by karsten＠torproject.org 28 Nov '13

28 Nov '13

commit 60f4cb605c038094cb30d9034d49adb24ae3c76c Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Nov 28 09:21:26 2013 +0100 Add note how to improve tests. --- test/org/torproject/onionoo/ResourceServletTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/org/torproject/onionoo/ResourceServletTest.java b/test/org/torproject/onionoo/ResourceServletTest.java index 55cf53c..51ea1b5 100644 --- a/test/org/torproject/onionoo/ResourceServletTest.java +++ b/test/org/torproject/onionoo/ResourceServletTest.java @@ -25,6 +25,9 @@ import org.torproject.onionoo.ResourceServlet.HttpServletResponseWrapper; import com.google.gson.Gson; +/* TODO This test class could (should?) be split into ResponseBuilderTest + * which tests ResponseBuilder and a much shorter ResourceServletTest + * which tests servlet specifics. */ public class ResourceServletTest { private SortedMap<String, String> relays, bridges;

1 0

[onionoo/master] Write our own dummy DocumentStore for testing.
by karsten＠torproject.org 28 Nov '13

28 Nov '13

commit 1cfbf760352d73ef9c558ac1c25954067897fd19 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Nov 27 16:19:49 2013 +0100 Write our own dummy DocumentStore for testing. Previously, we wrote out/summary and out/update files to a temp directory. Avoiding this step makes tests faster and prepares for more sophisticated tests. --- src/org/torproject/onionoo/ResourceServlet.java | 11 +-- .../org/torproject/onionoo/DummyDocumentStore.java | 77 ++++++++++++++++++++ .../torproject/onionoo/ResourceServletTest.java | 46 ++++-------- 3 files changed, 98 insertions(+), 36 deletions(-) diff --git a/src/org/torproject/onionoo/ResourceServlet.java b/src/org/torproject/onionoo/ResourceServlet.java index 2869d5e..165bc01 100644 --- a/src/org/torproject/onionoo/ResourceServlet.java +++ b/src/org/torproject/onionoo/ResourceServlet.java @@ -31,17 +31,18 @@ public class ResourceServlet extends HttpServlet { config.getInitParameter("maintenance") != null && config.getInitParameter("maintenance").equals("1"); File outDir = new File(config.getInitParameter("outDir")); - this.init(maintenanceMode, outDir, new Time()); + Time time = new Time(); + DocumentStore documentStore = new DocumentStore(outDir, time); + this.init(maintenanceMode, documentStore, time); } /* Called (indirectly) by servlet container and (directly) by test * class. */ - protected void init(boolean maintenanceMode, File outDir, - Time time) { + protected void init(boolean maintenanceMode, + DocumentStore documentStore, Time time) { this.maintenanceMode = maintenanceMode; if (!maintenanceMode) { - ResponseBuilder.initialize(new DocumentStore(outDir, time), - time); + ResponseBuilder.initialize(documentStore, time); } } diff --git a/test/org/torproject/onionoo/DummyDocumentStore.java b/test/org/torproject/onionoo/DummyDocumentStore.java new file mode 100644 index 0000000..0fce5d9 --- /dev/null +++ b/test/org/torproject/onionoo/DummyDocumentStore.java @@ -0,0 +1,77 @@ +package org.torproject.onionoo; + +import java.io.File; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; + +public class DummyDocumentStore extends DocumentStore { + + private long lastModified; + + public DummyDocumentStore(long lastModified, Time time) { + super((File) null, time); + this.lastModified = lastModified; + } + + private SortedMap<String, NodeStatus> nodeStatuses = + new TreeMap<String, NodeStatus>(); + void addNodeStatus(String nodeStatusString) { + NodeStatus nodeStatus = NodeStatus.fromString(nodeStatusString); + this.nodeStatuses.put(nodeStatus.getFingerprint(), nodeStatus); + } + + public void flushDocumentCache() { + throw new RuntimeException("Not implemented."); + } + + public String getStatsString() { + throw new RuntimeException("Not implemented."); + } + + public <T extends Document> SortedSet<String> list( + Class<T> documentType, boolean includeArchive) { + if (documentType.equals(NodeStatus.class)) { + return new TreeSet<String>(this.nodeStatuses.keySet()); + } + throw new RuntimeException("Not implemented."); + } + + public <T extends Document> boolean remove(Class<T> documentType) { + throw new RuntimeException("Not implemented."); + } + + public <T extends Document> boolean remove(Class<T> documentType, + String fingerprint) { + throw new RuntimeException("Not implemented."); + } + + public <T extends Document> T retrieve(Class<T> documentType, + boolean parse) { + if (documentType.equals(UpdateStatus.class)) { + UpdateStatus updateStatus = new UpdateStatus(); + updateStatus.documentString = String.valueOf(this.lastModified); + return documentType.cast(updateStatus); + } + throw new RuntimeException("Not implemented."); + } + + public <T extends Document> T retrieve(Class<T> documentType, + boolean parse, String fingerprint) { + if (documentType.equals(NodeStatus.class)) { + return documentType.cast(this.nodeStatuses.get(fingerprint)); + } + throw new RuntimeException("Not implemented."); + } + + public <T extends Document> boolean store(T document) { + throw new RuntimeException("Not implemented."); + } + + public <T extends Document> boolean store(T document, + String fingerprint) { + throw new RuntimeException("Not implemented."); + } +} + diff --git a/test/org/torproject/onionoo/ResourceServletTest.java b/test/org/torproject/onionoo/ResourceServletTest.java index ca7cd69..55cf53c 100644 --- a/test/org/torproject/onionoo/ResourceServletTest.java +++ b/test/org/torproject/onionoo/ResourceServletTest.java @@ -8,9 +8,6 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; @@ -22,9 +19,7 @@ import java.util.SortedMap; import java.util.TreeMap; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.TemporaryFolder; import org.torproject.onionoo.ResourceServlet.HttpServletRequestWrapper; import org.torproject.onionoo.ResourceServlet.HttpServletResponseWrapper; @@ -34,6 +29,8 @@ public class ResourceServletTest { private SortedMap<String, String> relays, bridges; + private DummyDocumentStore documentStore; + // 2013-04-24 12:22:22 private static long lastModified = 1366806142000L; @@ -155,7 +152,7 @@ public class ResourceServletTest { private void runTest(String requestURI, Map<String, String[]> parameterMap) { try { - this.writeSummaryFile(); + this.createDummyDocumentStore(); this.makeRequest(requestURI, parameterMap); this.parseResponse(); } catch (IOException e) { @@ -163,28 +160,25 @@ public class ResourceServletTest { } } - /* TODO Instead of writing out/summary and out/update to a temp - * directory, we could also write our own DocumentStore instance. */ - private void writeSummaryFile() throws IOException { - File summaryFile = new File(this.tempOutDir, "summary"); - BufferedWriter bw = new BufferedWriter(new FileWriter(summaryFile)); - for (String relay : this.relays.values()) { - bw.write(relay + "\n"); + private void createDummyDocumentStore() { + /* TODO Incrementing static lastModified is necessary for + * ResponseBuilder to read state from the newly created DocumentStore. + * Otherwise, ResponseBuilder would use data from the previous test + * run. This is bad design and should be fixed. */ + this.documentStore = new DummyDocumentStore(lastModified++, + this.testingTime); + for (String relay : relays.values()) { + documentStore.addNodeStatus(relay); } - for (String bridge : this.bridges.values()) { - bw.write(bridge + "\n"); + for (String bridge : bridges.values()) { + documentStore.addNodeStatus(bridge); } - bw.close(); - File updateFile = new File(this.tempOutDir, "update"); - bw = new BufferedWriter(new FileWriter(updateFile)); - bw.write(String.valueOf(lastModified++)); - bw.close(); } private void makeRequest(String requestURI, Map<String, String[]> parameterMap) throws IOException { ResourceServlet rs = new ResourceServlet(); - rs.init(maintenanceMode, this.tempOutDir, this.testingTime); + rs.init(maintenanceMode, this.documentStore, this.testingTime); this.request = new TestingHttpServletRequestWrapper(requestURI, parameterMap); this.response = new TestingHttpServletResponseWrapper(); @@ -261,16 +255,6 @@ public class ResourceServletTest { return parameters; } - @Rule - public TemporaryFolder tempFolder = new TemporaryFolder(); - - private File tempOutDir; - - @Before - public void createTempOutDir() throws IOException { - this.tempOutDir = this.tempFolder.newFolder("out"); - } - private static class SummaryDocument { private String relays_published; private RelaySummary[] relays;

1 0

r26439: {website} make the 'what to do with your mail' sentence even more obvi (website/trunk/getinvolved/en)
by Roger Dingledine 28 Nov '13

28 Nov '13

Author: arma Date: 2013-11-28 09:55:21 +0000 (Thu, 28 Nov 2013) New Revision: 26439 Modified: website/trunk/getinvolved/en/tshirt.wml Log: make the 'what to do with your mail' sentence even more obvious bug report by paul wouters Modified: website/trunk/getinvolved/en/tshirt.wml =================================================================== --- website/trunk/getinvolved/en/tshirt.wml 2013-11-26 22:18:51 UTC (rev 26438) +++ website/trunk/getinvolved/en/tshirt.wml 2013-11-28 09:55:21 UTC (rev 26439) @@ -38,7 +38,11 @@ system watches all of the relays. If you have a valid email address (lightly obfuscated is acceptable) in your ContactInfo field, we'll automatically email you when your relay earns you a free - t-shirt. If you get such a mail, or you've done something else cool, + t-shirt. + </p> + + <p> + If you get such a mail, or you've done something else cool, email us at donations(a)torproject.org with the details. </p>

1 0

[torbutton/master] Bug #10078: Updated code to clear SSL identifier cache in ESR24
by mikeperry＠torproject.org 28 Nov '13

28 Nov '13

commit 47b733b16b6eb9394c857fda00ff3830b8c2585d Author: Georg Koppen <g.koppen(a)jondos.de> Date: Wed Nov 27 21:03:27 2013 -0800 Bug #10078: Updated code to clear SSL identifier cache in ESR24 Also clean up comments and whitespace. --- src/chrome/content/torbutton.js | 47 ++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/src/chrome/content/torbutton.js b/src/chrome/content/torbutton.js index 0f9ade3..fcdc0d8 100644 --- a/src/chrome/content/torbutton.js +++ b/src/chrome/content/torbutton.js @@ -1437,7 +1437,7 @@ function torbutton_do_new_identity() { } catch(e) { torbutton_log(3, "Exception on wifi token clear: "+e); } - + torbutton_log(3, "New Identity: Closing tabs and clearing searchbox"); torbutton_close_on_toggle(true, true); @@ -1454,7 +1454,7 @@ function torbutton_do_new_identity() { torbutton_log(3, "New Identity: Emitting Private Browsing Session clear event"); obsSvc.notifyObservers(null, "browser:purge-session-history", ""); - + torbutton_log(3, "New Identity: Clearing HTTP Auth"); if(m_tb_prefs.getBoolPref('extensions.torbutton.clear_http_auth')) { @@ -1462,32 +1462,33 @@ function torbutton_do_new_identity() { getService(Components.interfaces.nsIHttpAuthManager); auth.clearAll(); } - + torbutton_log(3, "New Identity: Clearing Crypto Tokens"); try { - var secMgr = Cc["@mozilla.org/security/crypto;1"]. - getService(Ci.nsIDOMCrypto); - secMgr.logout(); - torbutton_log(3, "nsIDOMCrypto logout succeeded"); + // This clears the SSL identifier cache. + // See https://bugzilla.mozilla.org/show_bug.cgi?id=448747. But bug 683262 + // removes nsIDOMCrypto.logout(). We need to resort to our previous fallback + // method: switching a proper preference that triggers clearing the SSL + // identifier cache. + // See: https://mxr.mozilla.org/comm-esr24/source/mozilla/security/manager/ssl/src/… for the ones being available. + // secruity.enable_md5_signatures seems to be a good choice as it is still + // available on trunk. + m_tb_prefs.setBoolPref("security.enable_md5_signatures", !m_tb_prefs. + getBoolPref("security.enable_md5_signatures")); + m_tb_prefs.setBoolPref("security.enable_md5_signatures", !m_tb_prefs. + getBoolPref("security.enable_md5_signatures")); } catch(e) { - torbutton_log(4, "Failed to use nsIDOMCrypto to clear SSL Session ids. Falling back to old method. Error: "+e); - - // This clears the SSL Identifier Cache. - // See https://bugzilla.mozilla.org/show_bug.cgi?id=448747 and - // http://mxr.mozilla.org/security/source/security/manager/ssl/src/nsNSSCompon… - m_tb_prefs.setBoolPref("security.enable_ssl2", - !m_tb_prefs.getBoolPref("security.enable_ssl2")); - m_tb_prefs.setBoolPref("security.enable_ssl2", - !m_tb_prefs.getBoolPref("security.enable_ssl2")); + torbutton_log(4, "Failed to clear SSL session ids: "+e); } // This clears the OCSP cache. // // nsNSSComponent::Observe() watches security.OCSP.enabled, which calls - // setOCSPOptions(), which if set to 0, calls CERT_DisableOCSPChecking(), + // setValidationOptions(), which in turn calls setNonPkixOcspEnabled() which, + // if security.OCSP.enabled is set to 0, calls CERT_DisableOCSPChecking(), // which calls CERT_ClearOCSPCache(). - // See: http://mxr.mozilla.org/security/source/security/manager/ssl/src/nsNSSCompon… + // See: https://mxr.mozilla.org/comm-esr24/source/mozilla/security/manager/ssl/src/… var ocsp = m_tb_prefs.getIntPref("security.OCSP.enabled"); m_tb_prefs.setIntPref("security.OCSP.enabled", 0); m_tb_prefs.setIntPref("security.OCSP.enabled", ocsp); @@ -1507,7 +1508,7 @@ function torbutton_do_new_identity() { var tabs = m_tb_prefs.getIntPref("browser.sessionstore.max_tabs_undo"); m_tb_prefs.setIntPref("browser.sessionstore.max_tabs_undo", 0); m_tb_prefs.setIntPref("browser.sessionstore.max_tabs_undo", tabs); - + torbutton_log(3, "New Identity: Clearing Image Cache"); torbutton_clear_image_caches(); @@ -1541,7 +1542,7 @@ function torbutton_do_new_identity() { torbutton_log(5, "Exception on cache clearing: "+e); window.alert("Torbutton: Unexpected error during cache clearing: "+e); } - + torbutton_log(3, "New Identity: Clearing Cookies and DOM Storage"); if (m_tb_prefs.getBoolPref('extensions.torbutton.cookie_protections')) { @@ -1554,12 +1555,12 @@ function torbutton_do_new_identity() { } else { torbutton_clear_cookies(); } - + torbutton_log(3, "New Identity: Closing open connections"); // Clear keep-alive obsSvc.notifyObservers(this, "net:prune-all-connections", null); - + torbutton_log(3, "New Identity: Clearing Content Preferences"); // XXX: This may not clear zoom site-specific @@ -1598,7 +1599,7 @@ function torbutton_do_new_identity() { window.alert(warning); } } - + torbutton_log(3, "New Identity: Opening a new browser window"); // Open a new window with the TBB check homepage

1 0

[tor-browser/tor-browser-24.1.1esr-1] fixup! Build determinism fixes
by mikeperry＠torproject.org 28 Nov '13

28 Nov '13

commit b5ef2283fcf43695903af350058a98bd53f8b695 Author: Georg Koppen <g.koppen(a)jondos.de> Date: Fri Nov 22 16:12:27 2013 +0100 fixup! Build determinism fixes bug 10159: making the manifest files deterministic by sorting their entries before writing them --- config/JarMaker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/JarMaker.py b/config/JarMaker.py index 96d0207..4005fd8 100644 --- a/config/JarMaker.py +++ b/config/JarMaker.py @@ -189,7 +189,7 @@ class JarMaker(object): continue myregister[l] = None mf.seek(0) - for k in myregister.iterkeys(): + for k in sorted(myregister.iterkeys()): mf.write(k + os.linesep) mf.close() finally:

1 0

[tor-browser/tor-browser-24.1.1esr-1] Disable uninstall helper.exe on Win32.
by mikeperry＠torproject.org 28 Nov '13

28 Nov '13

commit 25d490116f43e91c27b3f5701c6e5df04df300a6 Author: Georg Koppen <georg(a)getfoxyproxy.org> Date: Wed Nov 27 20:39:43 2013 -0800 Disable uninstall helper.exe on Win32. Fixes 'make package' breakage on Windows builds. See https://trac.torproject.org/projects/tor/ticket/9837 --- browser/installer/package-manifest.in | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/browser/installer/package-manifest.in b/browser/installer/package-manifest.in index 6408606..87e2314 100644 --- a/browser/installer/package-manifest.in +++ b/browser/installer/package-manifest.in @@ -42,9 +42,9 @@ @BINPATH@/hyphenation/* @BINPATH@/browser/@PREF_DIR@/firefox-l10n.js @BINPATH@/browser/searchplugins/* -#ifdef XP_WIN32 -@BINPATH@/uninstall/helper.exe -#endif +;#ifdef XP_WIN32 +;@BINPATH@/uninstall/helper.exe +;#endif #ifdef MOZ_UPDATER @BINPATH@/update.locale @BINPATH@/updater.ini

1 0

[tor-launcher/master] Add support for Instantbird
by mikeperry＠torproject.org 28 Nov '13

28 Nov '13

commit 0dbee8e000a01062496ad4ce61f6efc1f830f7ff Author: Sukhbir Singh <sukhbir(a)torproject.org> Date: Wed Nov 27 20:29:26 2013 -0800 Add support for Instantbird --- src/components/tl-process.js | 5 +++-- src/install.rdf | 10 +++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/components/tl-process.js b/src/components/tl-process.js index a1ec60b..d1166a2 100644 --- a/src/components/tl-process.js +++ b/src/components/tl-process.js @@ -31,6 +31,7 @@ TorProcessService.prototype = kServiceName : "Tor Launcher Process Service", kClassID: Components.ID("{FE7B4CAF-BCF4-4848-8BFF-EFA66C9AFDA1}"), kThunderbirdID: "{3550f703-e582-4d05-9a08-453d09bdfdc6}", + kInstantbirdID: "{33cb9019-c295-46dd-be21-8c4936574bee}", kTorLauncherExtPath: "tor-launcher(a)torproject.org", // This could vary. kPrefPromptAtStartup: "extensions.torlauncher.prompt_at_startup", @@ -517,9 +518,9 @@ TorProcessService.prototype = var topDir; var appInfo = Cc["@mozilla.org/xre/app-info;1"] .getService(Ci.nsIXULAppInfo); - if (appInfo.ID == this.kThunderbirdID) + if (appInfo.ID == this.kThunderbirdID || appInfo.ID == this.kInstantbirdID) { - // For Thunderbird, paths are relative to this extension's folder. + // For Thunderbird and Instantbird, paths are relative to this extension's folder. topDir = Cc["@mozilla.org/file/directory_service;1"] .getService(Ci.nsIProperties).get("ProfD", Ci.nsIFile); topDir.append("extensions"); diff --git a/src/install.rdf b/src/install.rdf index 7788ddb..74ee5a4 100644 --- a/src/install.rdf +++ b/src/install.rdf @@ -25,7 +25,15 @@ <Description> <em:id>{3550f703-e582-4d05-9a08-453d09bdfdc6}</em:id> <em:minVersion>10.0</em:minVersion> - <em:maxVersion>23.*</em:maxVersion> + <em:maxVersion>24.*</em:maxVersion> + </Description> + </em:targetApplication> +  + <em:targetApplication> + <Description> + <em:id>{33cb9019-c295-46dd-be21-8c4936574bee}</em:id> + <em:minVersion>1.4</em:minVersion> + <em:maxVersion>1000</em:maxVersion> </Description> </em:targetApplication>

1 0

[torbutton/master] Merge remote-tracking branch 'brade/bug9570'
by mikeperry＠torproject.org 28 Nov '13

28 Nov '13

commit ddd79e67026e8bb64131462dae2ccb6d7d557c4e Merge: 27b9c7d 3a0ab36 Author: Mike Perry <mikeperry-git(a)torproject.org> Date: Wed Nov 27 19:35:16 2013 -0800 Merge remote-tracking branch 'brade/bug9570' makexpi.sh | 4 +- src/chrome/content/torbutton.js | 89 +- src/components/certDialogsOverride.js | 170 --- src/components/ignore-history.js | 368 ------ src/components/nsSessionStore2.diff | 80 -- src/components/nsSessionStore2.js | 2233 -------------------------------- src/components/nsSessionStore3.diff | 111 -- src/components/nsSessionStore3.js | 2246 --------------------------------- 8 files changed, 73 insertions(+), 5228 deletions(-)

1 0