[tor-commits] [metrics-lib/master] Introduce a new ExitList.Entry type.
karsten at torproject.org
karsten at torproject.org
Tue Dec 15 09:51:26 UTC 2015
commit b50e961a63a40a4c0d18129d42be351a47f5248e
Author: iwakeh <iwakeh at users.ourproject.org>
Date: Sat Dec 12 15:00:00 2015 +0000
Introduce a new ExitList.Entry type.
Patch for #17821
---
CHANGELOG.md | 5 +
src/org/torproject/descriptor/ExitList.java | 26 ++++
src/org/torproject/descriptor/ExitListEntry.java | 4 +-
.../descriptor/impl/ExitListEntryImpl.java | 115 ++++++++++++-----
.../torproject/descriptor/impl/ExitListImpl.java | 80 ++++++++----
.../descriptor/impl/ExitListImplTest.java | 131 ++++++++++++++++++++
6 files changed, 300 insertions(+), 61 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 03c2940..a56f9f4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,11 @@
- Support parsing of .xz-compressed tarballs using Apache Commons
Compress and XZ for Java. Applications only need to add XZ for
Java as dependency if they want to parse .xz-compressed tarballs.
+ - Introduce a new ExitList.Entry type for exit list entries instead
+ of the ExitListEntry type which is now deprecated. The main
+ difference between the two is that ExitList.Entry can hold more
+ than one exit address and scan time which were previously parsed
+ as multiple ExitListEntry instances.
# Changes in version 1.0.0 - 2015-12-05
diff --git a/src/org/torproject/descriptor/ExitList.java b/src/org/torproject/descriptor/ExitList.java
index 09d7c25..c813a6b 100644
--- a/src/org/torproject/descriptor/ExitList.java
+++ b/src/org/torproject/descriptor/ExitList.java
@@ -2,15 +2,41 @@
* See LICENSE for licensing information */
package org.torproject.descriptor;
+import java.util.Map;
import java.util.Set;
/* Exit list containing all known exit scan results at a given time. */
public interface ExitList extends Descriptor {
+ public final static String EOL = "\n";
+
+ /* Exit list entry containing results from a single exit scan. */
+ public interface Entry {
+
+ /* Return the scanned relay's fingerprint. */
+ public String getFingerprint();
+
+ /* Return the publication time of the scanned relay's last known
+ * descriptor. */
+ public long getPublishedMillis();
+
+ /* Return the publication time of the network status that this scan
+ * was based on. */
+ public long getLastStatusMillis();
+
+ /* Return the IP addresses that were determined in the scan. */
+ public Map<String, Long> getExitAddresses();
+ }
+
/* Return the download time of the exit list. */
public long getDownloadedMillis();
/* Return the unordered set of exit scan results. */
+ /* Use getEntries instead. */
+ @Deprecated
public Set<ExitListEntry> getExitListEntries();
+
+ /* Return the unordered set of exit scan results. */
+ public Set<ExitList.Entry> getEntries();
}
diff --git a/src/org/torproject/descriptor/ExitListEntry.java b/src/org/torproject/descriptor/ExitListEntry.java
index 201a172..7b69483 100644
--- a/src/org/torproject/descriptor/ExitListEntry.java
+++ b/src/org/torproject/descriptor/ExitListEntry.java
@@ -3,7 +3,9 @@
package org.torproject.descriptor;
/* Exit list entry containing results from a single exit scan. */
-public interface ExitListEntry {
+/* Use org.torproject.descriptor.ExitList.Entry instead. */
+ at Deprecated
+public interface ExitListEntry extends ExitList.Entry {
/* Return the scanned relay's fingerprint. */
public String getFingerprint();
diff --git a/src/org/torproject/descriptor/impl/ExitListEntryImpl.java b/src/org/torproject/descriptor/impl/ExitListEntryImpl.java
index a03e373..e899bcf 100644
--- a/src/org/torproject/descriptor/impl/ExitListEntryImpl.java
+++ b/src/org/torproject/descriptor/impl/ExitListEntryImpl.java
@@ -3,15 +3,19 @@
package org.torproject.descriptor.impl;
import org.torproject.descriptor.DescriptorParseException;
+import org.torproject.descriptor.ExitList;
+
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Scanner;
import java.util.SortedSet;
import java.util.TreeSet;
import org.torproject.descriptor.ExitListEntry;
-public class ExitListEntryImpl implements ExitListEntry {
+public class ExitListEntryImpl implements ExitListEntry, ExitList.Entry {
private byte[] exitListEntryBytes;
public byte[] getExitListEntryBytes() {
@@ -26,6 +30,31 @@ public class ExitListEntryImpl implements ExitListEntry {
return lines;
}
+ @Deprecated
+ private ExitListEntryImpl(String fingerprint, long publishedMillis,
+ long lastStatusMillis, String exitAddress, long scanMillis) {
+ this.fingerprint = fingerprint;
+ this.publishedMillis = publishedMillis;
+ this.lastStatusMillis = lastStatusMillis;
+ this.exitAddresses.put(exitAddress, scanMillis);
+ }
+
+ @Deprecated
+ List<ExitListEntry> oldEntries() {
+ List<ExitListEntry> result = new ArrayList<>();
+ if (this.exitAddresses.size() > 1) {
+ for (Map.Entry<String, Long> entry :
+ this.exitAddresses.entrySet()) {
+ result.add(new ExitListEntryImpl(this.fingerprint,
+ this.publishedMillis, this.lastStatusMillis, entry.getKey(),
+ entry.getValue()));
+ }
+ } else {
+ result.add(this);
+ }
+ return result;
+ }
+
protected ExitListEntryImpl(byte[] exitListEntryBytes,
boolean failUnrecognizedDescriptorLines)
throws DescriptorParseException {
@@ -37,56 +66,63 @@ public class ExitListEntryImpl implements ExitListEntry {
this.checkAndClearKeywords();
}
- private SortedSet<String> exactlyOnceKeywords;
+ private SortedSet<String> keywordCountingSet;
private void initializeKeywords() {
- this.exactlyOnceKeywords = new TreeSet<String>();
- this.exactlyOnceKeywords.add("ExitNode");
- this.exactlyOnceKeywords.add("Published");
- this.exactlyOnceKeywords.add("LastStatus");
- this.exactlyOnceKeywords.add("ExitAddress");
+ this.keywordCountingSet = new TreeSet<String>();
+ this.keywordCountingSet.add("ExitNode");
+ this.keywordCountingSet.add("Published");
+ this.keywordCountingSet.add("LastStatus");
+ this.keywordCountingSet.add("ExitAddress");
}
private void parsedExactlyOnceKeyword(String keyword)
throws DescriptorParseException {
- if (!this.exactlyOnceKeywords.contains(keyword)) {
+ if (!this.keywordCountingSet.contains(keyword)) {
throw new DescriptorParseException("Duplicate '" + keyword
+ "' line in exit list entry.");
}
- this.exactlyOnceKeywords.remove(keyword);
+ this.keywordCountingSet.remove(keyword);
}
private void checkAndClearKeywords() throws DescriptorParseException {
- for (String missingKeyword : this.exactlyOnceKeywords) {
+ for (String missingKeyword : this.keywordCountingSet) {
throw new DescriptorParseException("Missing '" + missingKeyword
+ "' line in exit list entry.");
}
- this.exactlyOnceKeywords = null;
+ this.keywordCountingSet = null;
}
private void parseExitListEntryBytes()
throws DescriptorParseException {
Scanner s = new Scanner(new String(this.exitListEntryBytes)).
- useDelimiter("\n");
+ useDelimiter(ExitList.EOL);
while (s.hasNext()) {
String line = s.next();
String[] parts = line.split(" ");
String keyword = parts[0];
- if (keyword.equals("ExitNode")) {
- this.parseExitNodeLine(line, parts);
- } else if (keyword.equals("Published")) {
- this.parsePublishedLine(line, parts);
- } else if (keyword.equals("LastStatus")) {
- this.parseLastStatusLine(line, parts);
- } else if (keyword.equals("ExitAddress")) {
- this.parseExitAddressLine(line, parts);
- } else if (this.failUnrecognizedDescriptorLines) {
- throw new DescriptorParseException("Unrecognized line '" + line
- + "' in exit list entry.");
- } else {
- if (this.unrecognizedLines == null) {
- this.unrecognizedLines = new ArrayList<String>();
- }
- this.unrecognizedLines.add(line);
+ switch (keyword) {
+ case "ExitNode":
+ this.parseExitNodeLine(line, parts);
+ break;
+ case "Published":
+ this.parsePublishedLine(line, parts);
+ break;
+ case "LastStatus":
+ this.parseLastStatusLine(line, parts);
+ break;
+ case "ExitAddress":
+ this.parseExitAddressLine(line, parts);
+ break;
+ default:
+ if (this.failUnrecognizedDescriptorLines) {
+ throw new DescriptorParseException("Unrecognized line '"
+ + line + "' in exit list entry.");
+ } else {
+ if (this.unrecognizedLines == null) {
+ this.unrecognizedLines = new ArrayList<>();
+ }
+ this.unrecognizedLines.add(line);
+ }
}
}
}
@@ -130,10 +166,9 @@ public class ExitListEntryImpl implements ExitListEntry {
throw new DescriptorParseException("Invalid line '" + line + "' in "
+ "exit list entry.");
}
- this.parsedExactlyOnceKeyword(parts[0]);
- this.exitAddress = ParseHelper.parseIpv4Address(line, parts[1]);
- this.scanMillis = ParseHelper.parseTimestampAtIndex(line, parts,
- 2, 3);
+ this.keywordCountingSet.remove(parts[0]);
+ this.exitAddresses.put(ParseHelper.parseIpv4Address(line, parts[1]),
+ ParseHelper.parseTimestampAtIndex(line, parts, 2, 3));
}
private String fingerprint;
@@ -153,12 +188,26 @@ public class ExitListEntryImpl implements ExitListEntry {
private String exitAddress;
public String getExitAddress() {
+ if (null == exitAddress) {
+ Map.Entry<String, Long> randomEntry =
+ this.exitAddresses.entrySet().iterator().next();
+ this.exitAddress = randomEntry.getKey();
+ this.scanMillis = randomEntry.getValue();
+ }
return this.exitAddress;
}
+ private Map<String, Long> exitAddresses = new HashMap<>();
+ public Map<String, Long> getExitAddresses(){
+ return new HashMap<>(this.exitAddresses);
+ }
+
private long scanMillis;
public long getScanMillis() {
- return this.scanMillis;
+ if (null == exitAddress) {
+ getExitAddress();
+ }
+ return scanMillis;
}
}
diff --git a/src/org/torproject/descriptor/impl/ExitListImpl.java b/src/org/torproject/descriptor/impl/ExitListImpl.java
index 53dc112..730217e 100644
--- a/src/org/torproject/descriptor/impl/ExitListImpl.java
+++ b/src/org/torproject/descriptor/impl/ExitListImpl.java
@@ -15,7 +15,6 @@ import java.util.TimeZone;
import org.torproject.descriptor.ExitList;
import org.torproject.descriptor.ExitListEntry;
-/* TODO Add test class. */
public class ExitListImpl extends DescriptorImpl implements ExitList {
protected ExitListImpl(byte[] rawDescriptorBytes, String fileName,
@@ -52,36 +51,57 @@ public class ExitListImpl extends DescriptorImpl implements ExitList {
throw new DescriptorParseException("Descriptor is empty.");
}
String descriptorString = new String(rawDescriptorBytes);
- Scanner s = new Scanner(descriptorString).useDelimiter("\n");
+ Scanner s = new Scanner(descriptorString).useDelimiter(EOL);
StringBuilder sb = new StringBuilder();
+ boolean firstEntry = true;
while (s.hasNext()) {
String line = s.next();
+ if (line.startsWith("@")) { /* Skip annotation. */
+ if (!s.hasNext()) {
+ throw new DescriptorParseException("Descriptor is empty.");
+ } else {
+ line = s.next();
+ }
+ }
String[] parts = line.split(" ");
String keyword = parts[0];
- if (keyword.equals("Downloaded")) {
- this.downloadedMillis = ParseHelper.parseTimestampAtIndex(line,
- parts, 1, 2);
- } else if (keyword.equals("ExitNode")) {
- sb = new StringBuilder();
- sb.append(line + "\n");
- } else if (keyword.equals("Published")) {
- sb.append(line + "\n");
- } else if (keyword.equals("LastStatus")) {
- sb.append(line + "\n");
- } else if (keyword.equals("ExitAddress")) {
- String exitListEntryString = sb.toString() + line + "\n";
- byte[] exitListEntryBytes = exitListEntryString.getBytes();
- this.parseExitListEntry(exitListEntryBytes);
- } else if (this.failUnrecognizedDescriptorLines) {
- throw new DescriptorParseException("Unrecognized line '" + line
- + "' in exit list.");
- } else {
- if (this.unrecognizedLines == null) {
- this.unrecognizedLines = new ArrayList<String>();
- }
- this.unrecognizedLines.add(line);
+ switch (keyword) {
+ case "Downloaded":
+ this.downloadedMillis = ParseHelper.parseTimestampAtIndex(line,
+ parts, 1, 2);
+ break;
+ case "ExitNode":
+ if (!firstEntry) {
+ this.parseExitListEntry(sb.toString().getBytes());
+ } else {
+ firstEntry = false;
+ }
+ sb = new StringBuilder();
+ sb.append(line).append(ExitList.EOL);
+ break;
+ case "Published":
+ sb.append(line).append(ExitList.EOL);
+ break;
+ case "LastStatus":
+ sb.append(line).append(ExitList.EOL);
+ break;
+ case "ExitAddress":
+ sb.append(line).append(ExitList.EOL);
+ break;
+ default:
+ if (this.failUnrecognizedDescriptorLines) {
+ throw new DescriptorParseException("Unrecognized line '"
+ + line + "' in exit list.");
+ } else {
+ if (this.unrecognizedLines == null) {
+ this.unrecognizedLines = new ArrayList<String>();
+ }
+ this.unrecognizedLines.add(line);
+ }
}
}
+ /* Parse the last entry. */
+ this.parseExitListEntry(sb.toString().getBytes());
}
protected void parseExitListEntry(byte[] exitListEntryBytes)
@@ -89,6 +109,7 @@ public class ExitListImpl extends DescriptorImpl implements ExitList {
ExitListEntryImpl exitListEntry = new ExitListEntryImpl(
exitListEntryBytes, this.failUnrecognizedDescriptorLines);
this.exitListEntries.add(exitListEntry);
+ this.oldExitListEntries.addAll(exitListEntry.oldEntries());
List<String> unrecognizedExitListEntryLines = exitListEntry.
getAndClearUnrecognizedLines();
if (unrecognizedExitListEntryLines != null) {
@@ -104,10 +125,15 @@ public class ExitListImpl extends DescriptorImpl implements ExitList {
return this.downloadedMillis;
}
- private Set<ExitListEntry> exitListEntries =
- new HashSet<ExitListEntry>();
+ private Set<ExitListEntry> oldExitListEntries = new HashSet<>();
+ @Deprecated
public Set<ExitListEntry> getExitListEntries() {
- return new HashSet<ExitListEntry>(this.exitListEntries);
+ return new HashSet<>(this.oldExitListEntries);
+ }
+
+ private Set<ExitList.Entry> exitListEntries = new HashSet<>();
+ public Set<ExitList.Entry> getEntries() {
+ return new HashSet<ExitList.Entry>(this.exitListEntries);
}
}
diff --git a/test/org/torproject/descriptor/impl/ExitListImplTest.java b/test/org/torproject/descriptor/impl/ExitListImplTest.java
new file mode 100644
index 0000000..a563857
--- /dev/null
+++ b/test/org/torproject/descriptor/impl/ExitListImplTest.java
@@ -0,0 +1,131 @@
+/* Copyright 2015 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.descriptor.impl;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.junit.Test;
+import org.torproject.descriptor.DescriptorParseException;
+import org.torproject.descriptor.ExitListEntry;
+
+public class ExitListImplTest {
+
+ @Test()
+ public void testAnnotatedInput() throws Exception {
+ ExitListImpl result = new ExitListImpl((tordnselAnnotation + input)
+ .getBytes("US-ASCII"), fileName, false);
+ assertEquals("Expected one annotation.", 1,
+ result.getAnnotations().size());
+ assertEquals(tordnselAnnotation.substring(0, 18),
+ result.getAnnotations().get(0));
+ assertEquals(1441065722000L, result.getDownloadedMillis());
+ assertTrue("Unrecognized lines: " + result.getUnrecognizedLines(),
+ result.getUnrecognizedLines().isEmpty());
+ assertEquals("Found: " + result.getExitListEntries(), 7,
+ result.getExitListEntries().size());
+ assertEquals("Found: " + result.getEntries(), 5,
+ result.getEntries().size());
+ }
+
+ @Test()
+ public void testMultipleOldExitAddresses() throws Exception {
+ ExitListImpl result = new ExitListImpl(
+ (tordnselAnnotation + multiExitAddressInput)
+ .getBytes("US-ASCII"), fileName, false);
+ assertTrue("Unrecognized lines: " + result.getUnrecognizedLines(),
+ result.getUnrecognizedLines().isEmpty());
+ assertEquals("Found: " + result.getExitListEntries(),
+ 3, result.getExitListEntries().size());
+ Map<String, Long> testMap = new HashMap();
+ testMap.put("81.7.17.171", 1441044592000L);
+ testMap.put("81.7.17.172", 1441044652000L);
+ testMap.put("81.7.17.173", 1441044712000L);
+ for (ExitListEntry ele : result.getExitListEntries()) {
+ Map<String, Long> map = ele.getExitAddresses();
+ assertEquals("Found: " + map, 1, map.size());
+ Map.Entry<String, Long> ea = map.entrySet().iterator().next();
+ assertTrue("Map: " + testMap,
+ testMap.keySet().contains(ea.getKey()));
+ assertTrue("Map: " + testMap + " exitaddress: " + ea,
+ testMap.values().contains(ea.getValue()));
+ testMap.remove(ea.getKey());
+ }
+ assertTrue("Map: " + testMap, testMap.isEmpty());
+ }
+
+ @Test()
+ public void testMultipleExitAddresses() throws Exception {
+ ExitListImpl result = new ExitListImpl(
+ (tordnselAnnotation + multiExitAddressInput)
+ .getBytes("US-ASCII"), fileName, false);
+ assertTrue("Unrecognized lines: " + result.getUnrecognizedLines(),
+ result.getUnrecognizedLines().isEmpty());
+ Map<String, Long> map = result.getEntries()
+ .iterator().next().getExitAddresses();
+ assertEquals("Found: " + map, 3, map.size());
+ assertTrue("Map: " + map, map.containsKey("81.7.17.171"));
+ assertTrue("Map: " + map, map.containsKey("81.7.17.172"));
+ assertTrue("Map: " + map, map.containsKey("81.7.17.173"));
+ }
+
+ @Test(expected = DescriptorParseException.class)
+ public void testInsufficientInput0() throws Exception {
+ new ExitListImpl((tordnselAnnotation + insufficientInput[0])
+ .getBytes("US-ASCII"), fileName, false);
+ }
+
+ @Test(expected = DescriptorParseException.class)
+ public void testInsufficientInput1() throws Exception {
+ new ExitListImpl((tordnselAnnotation + insufficientInput[1])
+ .getBytes("US-ASCII"), fileName, false);
+ }
+
+ private static final String tordnselAnnotation = "@type tordnsel 1.0\n";
+ private static final String fileName = "2015-09-01-00-02-02";
+ private static final String[] insufficientInput = new String[] {
+ "Downloaded 2015-09-01 00:02:02\n"
+ + "ExitNode 0011BD2485AD45D984EC4159C88FC066E5E3300E\n"
+ + "Published 2015-08-31 16:17:30\n"
+ + "LastStatus 2015-08-31 17:03:18\n",
+ "Downloaded 2015-09-01 00:02:02\n"
+ + "ExitNode 0011BD2485AD45D984EC4159C88FC066E5E3300E\n"
+ + "LastStatus 2015-08-31 17:03:18\n"
+ + "ExitAddress 81.7.17.172 2015-08-31 18:10:52\n" };
+
+ private static final String multiExitAddressInput =
+ "Downloaded 2015-09-01 00:02:02\n"
+ + "ExitNode 0011BD2485AD45D984EC4159C88FC066E5E3300E\n"
+ + "Published 2015-08-31 16:17:30\n"
+ + "LastStatus 2015-08-31 17:03:18\n"
+ + "ExitAddress 81.7.17.171 2015-08-31 18:09:52\n"
+ + "ExitAddress 81.7.17.172 2015-08-31 18:10:52\n"
+ + "ExitAddress 81.7.17.173 2015-08-31 18:11:52\n";
+ private static final String input = "Downloaded 2015-09-01 00:02:02\n"
+ + "ExitNode 0011BD2485AD45D984EC4159C88FC066E5E3300E\n"
+ + "Published 2015-08-31 16:17:30\n"
+ + "LastStatus 2015-08-31 17:03:18\n"
+ + "ExitAddress 162.247.72.201 2015-08-31 17:09:23\n"
+ + "ExitNode 0098C475875ABC4AA864738B1D1079F711C38287\n"
+ + "Published 2015-08-31 13:59:24\n"
+ + "LastStatus 2015-08-31 15:03:20\n"
+ + "ExitAddress 162.248.160.151 2015-08-31 15:07:27\n"
+ + "ExitNode 00C4B4731658D3B4987132A3F77100CFCB190D97\n"
+ + "Published 2015-08-31 17:47:52\n"
+ + "LastStatus 2015-08-31 18:03:17\n"
+ + "ExitAddress 81.7.17.171 2015-08-31 18:09:52\n"
+ + "ExitAddress 81.7.17.172 2015-08-31 18:10:52\n"
+ + "ExitAddress 81.7.17.173 2015-08-31 18:11:52\n"
+ + "ExitNode 00F2D93EBAF2F51D6EE4DCB0F37D91D72F824B16\n"
+ + "Published 2015-08-31 14:39:05\n"
+ + "LastStatus 2015-08-31 16:02:18\n"
+ + "ExitAddress 23.239.18.57 2015-08-31 16:06:07\n"
+ + "ExitNode 011B1D1E876B2C835D01FB9D407F2E00B28077F6\n"
+ + "Published 2015-08-31 05:14:35\n"
+ + "LastStatus 2015-08-31 06:03:29\n"
+ + "ExitAddress 104.131.51.150 2015-08-31 06:04:07\n";
+}
+
More information about the tor-commits
mailing list