commit b50e961a63a40a4c0d18129d42be351a47f5248e Author: iwakeh <iwakeh@users.ourproject.org> Date: Sat Dec 12 15:00:00 2015 +0000 Introduce a new ExitList.Entry type. Patch for #17821 --- CHANGELOG.md | 5 + src/org/torproject/descriptor/ExitList.java | 26 ++++ src/org/torproject/descriptor/ExitListEntry.java | 4 +- .../descriptor/impl/ExitListEntryImpl.java | 115 ++++++++++++----- .../torproject/descriptor/impl/ExitListImpl.java | 80 ++++++++---- .../descriptor/impl/ExitListImplTest.java | 131 ++++++++++++++++++++ 6 files changed, 300 insertions(+), 61 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03c2940..a56f9f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ - Support parsing of .xz-compressed tarballs using Apache Commons Compress and XZ for Java. Applications only need to add XZ for Java as dependency if they want to parse .xz-compressed tarballs. + - Introduce a new ExitList.Entry type for exit list entries instead + of the ExitListEntry type which is now deprecated. The main + difference between the two is that ExitList.Entry can hold more + than one exit address and scan time which were previously parsed + as multiple ExitListEntry instances. # Changes in version 1.0.0 - 2015-12-05 diff --git a/src/org/torproject/descriptor/ExitList.java b/src/org/torproject/descriptor/ExitList.java index 09d7c25..c813a6b 100644 --- a/src/org/torproject/descriptor/ExitList.java +++ b/src/org/torproject/descriptor/ExitList.java @@ -2,15 +2,41 @@ * See LICENSE for licensing information */ package org.torproject.descriptor; +import java.util.Map; import java.util.Set; /* Exit list containing all known exit scan results at a given time. */ public interface ExitList extends Descriptor { + public final static String EOL = "\n"; + + /* Exit list entry containing results from a single exit scan. */ + public interface Entry { + + /* Return the scanned relay's fingerprint. */ + public String getFingerprint(); + + /* Return the publication time of the scanned relay's last known + * descriptor. */ + public long getPublishedMillis(); + + /* Return the publication time of the network status that this scan + * was based on. */ + public long getLastStatusMillis(); + + /* Return the IP addresses that were determined in the scan. */ + public Map<String, Long> getExitAddresses(); + } + /* Return the download time of the exit list. */ public long getDownloadedMillis(); /* Return the unordered set of exit scan results. */ + /* Use getEntries instead. */ + @Deprecated public Set<ExitListEntry> getExitListEntries(); + + /* Return the unordered set of exit scan results. */ + public Set<ExitList.Entry> getEntries(); } diff --git a/src/org/torproject/descriptor/ExitListEntry.java b/src/org/torproject/descriptor/ExitListEntry.java index 201a172..7b69483 100644 --- a/src/org/torproject/descriptor/ExitListEntry.java +++ b/src/org/torproject/descriptor/ExitListEntry.java @@ -3,7 +3,9 @@ package org.torproject.descriptor; /* Exit list entry containing results from a single exit scan. */ -public interface ExitListEntry { +/* Use org.torproject.descriptor.ExitList.Entry instead. */ +@Deprecated +public interface ExitListEntry extends ExitList.Entry { /* Return the scanned relay's fingerprint. */ public String getFingerprint(); diff --git a/src/org/torproject/descriptor/impl/ExitListEntryImpl.java b/src/org/torproject/descriptor/impl/ExitListEntryImpl.java index a03e373..e899bcf 100644 --- a/src/org/torproject/descriptor/impl/ExitListEntryImpl.java +++ b/src/org/torproject/descriptor/impl/ExitListEntryImpl.java @@ -3,15 +3,19 @@ package org.torproject.descriptor.impl; import org.torproject.descriptor.DescriptorParseException; +import org.torproject.descriptor.ExitList; + import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Scanner; import java.util.SortedSet; import java.util.TreeSet; import org.torproject.descriptor.ExitListEntry; -public class ExitListEntryImpl implements ExitListEntry { +public class ExitListEntryImpl implements ExitListEntry, ExitList.Entry { private byte[] exitListEntryBytes; public byte[] getExitListEntryBytes() { @@ -26,6 +30,31 @@ public class ExitListEntryImpl implements ExitListEntry { return lines; } + @Deprecated + private ExitListEntryImpl(String fingerprint, long publishedMillis, + long lastStatusMillis, String exitAddress, long scanMillis) { + this.fingerprint = fingerprint; + this.publishedMillis = publishedMillis; + this.lastStatusMillis = lastStatusMillis; + this.exitAddresses.put(exitAddress, scanMillis); + } + + @Deprecated + List<ExitListEntry> oldEntries() { + List<ExitListEntry> result = new ArrayList<>(); + if (this.exitAddresses.size() > 1) { + for (Map.Entry<String, Long> entry : + this.exitAddresses.entrySet()) { + result.add(new ExitListEntryImpl(this.fingerprint, + this.publishedMillis, this.lastStatusMillis, entry.getKey(), + entry.getValue())); + } + } else { + result.add(this); + } + return result; + } + protected ExitListEntryImpl(byte[] exitListEntryBytes, boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { @@ -37,56 +66,63 @@ public class ExitListEntryImpl implements ExitListEntry { this.checkAndClearKeywords(); } - private SortedSet<String> exactlyOnceKeywords; + private SortedSet<String> keywordCountingSet; private void initializeKeywords() { - this.exactlyOnceKeywords = new TreeSet<String>(); - this.exactlyOnceKeywords.add("ExitNode"); - this.exactlyOnceKeywords.add("Published"); - this.exactlyOnceKeywords.add("LastStatus"); - this.exactlyOnceKeywords.add("ExitAddress"); + this.keywordCountingSet = new TreeSet<String>(); + this.keywordCountingSet.add("ExitNode"); + this.keywordCountingSet.add("Published"); + this.keywordCountingSet.add("LastStatus"); + this.keywordCountingSet.add("ExitAddress"); } private void parsedExactlyOnceKeyword(String keyword) throws DescriptorParseException { - if (!this.exactlyOnceKeywords.contains(keyword)) { + if (!this.keywordCountingSet.contains(keyword)) { throw new DescriptorParseException("Duplicate '" + keyword + "' line in exit list entry."); } - this.exactlyOnceKeywords.remove(keyword); + this.keywordCountingSet.remove(keyword); } private void checkAndClearKeywords() throws DescriptorParseException { - for (String missingKeyword : this.exactlyOnceKeywords) { + for (String missingKeyword : this.keywordCountingSet) { throw new DescriptorParseException("Missing '" + missingKeyword + "' line in exit list entry."); } - this.exactlyOnceKeywords = null; + this.keywordCountingSet = null; } private void parseExitListEntryBytes() throws DescriptorParseException { Scanner s = new Scanner(new String(this.exitListEntryBytes)). - useDelimiter("\n"); + useDelimiter(ExitList.EOL); while (s.hasNext()) { String line = s.next(); String[] parts = line.split(" "); String keyword = parts[0]; - if (keyword.equals("ExitNode")) { - this.parseExitNodeLine(line, parts); - } else if (keyword.equals("Published")) { - this.parsePublishedLine(line, parts); - } else if (keyword.equals("LastStatus")) { - this.parseLastStatusLine(line, parts); - } else if (keyword.equals("ExitAddress")) { - this.parseExitAddressLine(line, parts); - } else if (this.failUnrecognizedDescriptorLines) { - throw new DescriptorParseException("Unrecognized line '" + line - + "' in exit list entry."); - } else { - if (this.unrecognizedLines == null) { - this.unrecognizedLines = new ArrayList<String>(); - } - this.unrecognizedLines.add(line); + switch (keyword) { + case "ExitNode": + this.parseExitNodeLine(line, parts); + break; + case "Published": + this.parsePublishedLine(line, parts); + break; + case "LastStatus": + this.parseLastStatusLine(line, parts); + break; + case "ExitAddress": + this.parseExitAddressLine(line, parts); + break; + default: + if (this.failUnrecognizedDescriptorLines) { + throw new DescriptorParseException("Unrecognized line '" + + line + "' in exit list entry."); + } else { + if (this.unrecognizedLines == null) { + this.unrecognizedLines = new ArrayList<>(); + } + this.unrecognizedLines.add(line); + } } } } @@ -130,10 +166,9 @@ public class ExitListEntryImpl implements ExitListEntry { throw new DescriptorParseException("Invalid line '" + line + "' in " + "exit list entry."); } - this.parsedExactlyOnceKeyword(parts[0]); - this.exitAddress = ParseHelper.parseIpv4Address(line, parts[1]); - this.scanMillis = ParseHelper.parseTimestampAtIndex(line, parts, - 2, 3); + this.keywordCountingSet.remove(parts[0]); + this.exitAddresses.put(ParseHelper.parseIpv4Address(line, parts[1]), + ParseHelper.parseTimestampAtIndex(line, parts, 2, 3)); } private String fingerprint; @@ -153,12 +188,26 @@ public class ExitListEntryImpl implements ExitListEntry { private String exitAddress; public String getExitAddress() { + if (null == exitAddress) { + Map.Entry<String, Long> randomEntry = + this.exitAddresses.entrySet().iterator().next(); + this.exitAddress = randomEntry.getKey(); + this.scanMillis = randomEntry.getValue(); + } return this.exitAddress; } + private Map<String, Long> exitAddresses = new HashMap<>(); + public Map<String, Long> getExitAddresses(){ + return new HashMap<>(this.exitAddresses); + } + private long scanMillis; public long getScanMillis() { - return this.scanMillis; + if (null == exitAddress) { + getExitAddress(); + } + return scanMillis; } } diff --git a/src/org/torproject/descriptor/impl/ExitListImpl.java b/src/org/torproject/descriptor/impl/ExitListImpl.java index 53dc112..730217e 100644 --- a/src/org/torproject/descriptor/impl/ExitListImpl.java +++ b/src/org/torproject/descriptor/impl/ExitListImpl.java @@ -15,7 +15,6 @@ import java.util.TimeZone; import org.torproject.descriptor.ExitList; import org.torproject.descriptor.ExitListEntry; -/* TODO Add test class. */ public class ExitListImpl extends DescriptorImpl implements ExitList { protected ExitListImpl(byte[] rawDescriptorBytes, String fileName, @@ -52,36 +51,57 @@ public class ExitListImpl extends DescriptorImpl implements ExitList { throw new DescriptorParseException("Descriptor is empty."); } String descriptorString = new String(rawDescriptorBytes); - Scanner s = new Scanner(descriptorString).useDelimiter("\n"); + Scanner s = new Scanner(descriptorString).useDelimiter(EOL); StringBuilder sb = new StringBuilder(); + boolean firstEntry = true; while (s.hasNext()) { String line = s.next(); + if (line.startsWith("@")) { /* Skip annotation. */ + if (!s.hasNext()) { + throw new DescriptorParseException("Descriptor is empty."); + } else { + line = s.next(); + } + } String[] parts = line.split(" "); String keyword = parts[0]; - if (keyword.equals("Downloaded")) { - this.downloadedMillis = ParseHelper.parseTimestampAtIndex(line, - parts, 1, 2); - } else if (keyword.equals("ExitNode")) { - sb = new StringBuilder(); - sb.append(line + "\n"); - } else if (keyword.equals("Published")) { - sb.append(line + "\n"); - } else if (keyword.equals("LastStatus")) { - sb.append(line + "\n"); - } else if (keyword.equals("ExitAddress")) { - String exitListEntryString = sb.toString() + line + "\n"; - byte[] exitListEntryBytes = exitListEntryString.getBytes(); - this.parseExitListEntry(exitListEntryBytes); - } else if (this.failUnrecognizedDescriptorLines) { - throw new DescriptorParseException("Unrecognized line '" + line - + "' in exit list."); - } else { - if (this.unrecognizedLines == null) { - this.unrecognizedLines = new ArrayList<String>(); - } - this.unrecognizedLines.add(line); + switch (keyword) { + case "Downloaded": + this.downloadedMillis = ParseHelper.parseTimestampAtIndex(line, + parts, 1, 2); + break; + case "ExitNode": + if (!firstEntry) { + this.parseExitListEntry(sb.toString().getBytes()); + } else { + firstEntry = false; + } + sb = new StringBuilder(); + sb.append(line).append(ExitList.EOL); + break; + case "Published": + sb.append(line).append(ExitList.EOL); + break; + case "LastStatus": + sb.append(line).append(ExitList.EOL); + break; + case "ExitAddress": + sb.append(line).append(ExitList.EOL); + break; + default: + if (this.failUnrecognizedDescriptorLines) { + throw new DescriptorParseException("Unrecognized line '" + + line + "' in exit list."); + } else { + if (this.unrecognizedLines == null) { + this.unrecognizedLines = new ArrayList<String>(); + } + this.unrecognizedLines.add(line); + } } } + /* Parse the last entry. */ + this.parseExitListEntry(sb.toString().getBytes()); } protected void parseExitListEntry(byte[] exitListEntryBytes) @@ -89,6 +109,7 @@ public class ExitListImpl extends DescriptorImpl implements ExitList { ExitListEntryImpl exitListEntry = new ExitListEntryImpl( exitListEntryBytes, this.failUnrecognizedDescriptorLines); this.exitListEntries.add(exitListEntry); + this.oldExitListEntries.addAll(exitListEntry.oldEntries()); List<String> unrecognizedExitListEntryLines = exitListEntry. getAndClearUnrecognizedLines(); if (unrecognizedExitListEntryLines != null) { @@ -104,10 +125,15 @@ public class ExitListImpl extends DescriptorImpl implements ExitList { return this.downloadedMillis; } - private Set<ExitListEntry> exitListEntries = - new HashSet<ExitListEntry>(); + private Set<ExitListEntry> oldExitListEntries = new HashSet<>(); + @Deprecated public Set<ExitListEntry> getExitListEntries() { - return new HashSet<ExitListEntry>(this.exitListEntries); + return new HashSet<>(this.oldExitListEntries); + } + + private Set<ExitList.Entry> exitListEntries = new HashSet<>(); + public Set<ExitList.Entry> getEntries() { + return new HashSet<ExitList.Entry>(this.exitListEntries); } } diff --git a/test/org/torproject/descriptor/impl/ExitListImplTest.java b/test/org/torproject/descriptor/impl/ExitListImplTest.java new file mode 100644 index 0000000..a563857 --- /dev/null +++ b/test/org/torproject/descriptor/impl/ExitListImplTest.java @@ -0,0 +1,131 @@ +/* Copyright 2015 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.descriptor.impl; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.Test; +import org.torproject.descriptor.DescriptorParseException; +import org.torproject.descriptor.ExitListEntry; + +public class ExitListImplTest { + + @Test() + public void testAnnotatedInput() throws Exception { + ExitListImpl result = new ExitListImpl((tordnselAnnotation + input) + .getBytes("US-ASCII"), fileName, false); + assertEquals("Expected one annotation.", 1, + result.getAnnotations().size()); + assertEquals(tordnselAnnotation.substring(0, 18), + result.getAnnotations().get(0)); + assertEquals(1441065722000L, result.getDownloadedMillis()); + assertTrue("Unrecognized lines: " + result.getUnrecognizedLines(), + result.getUnrecognizedLines().isEmpty()); + assertEquals("Found: " + result.getExitListEntries(), 7, + result.getExitListEntries().size()); + assertEquals("Found: " + result.getEntries(), 5, + result.getEntries().size()); + } + + @Test() + public void testMultipleOldExitAddresses() throws Exception { + ExitListImpl result = new ExitListImpl( + (tordnselAnnotation + multiExitAddressInput) + .getBytes("US-ASCII"), fileName, false); + assertTrue("Unrecognized lines: " + result.getUnrecognizedLines(), + result.getUnrecognizedLines().isEmpty()); + assertEquals("Found: " + result.getExitListEntries(), + 3, result.getExitListEntries().size()); + Map<String, Long> testMap = new HashMap(); + testMap.put("81.7.17.171", 1441044592000L); + testMap.put("81.7.17.172", 1441044652000L); + testMap.put("81.7.17.173", 1441044712000L); + for (ExitListEntry ele : result.getExitListEntries()) { + Map<String, Long> map = ele.getExitAddresses(); + assertEquals("Found: " + map, 1, map.size()); + Map.Entry<String, Long> ea = map.entrySet().iterator().next(); + assertTrue("Map: " + testMap, + testMap.keySet().contains(ea.getKey())); + assertTrue("Map: " + testMap + " exitaddress: " + ea, + testMap.values().contains(ea.getValue())); + testMap.remove(ea.getKey()); + } + assertTrue("Map: " + testMap, testMap.isEmpty()); + } + + @Test() + public void testMultipleExitAddresses() throws Exception { + ExitListImpl result = new ExitListImpl( + (tordnselAnnotation + multiExitAddressInput) + .getBytes("US-ASCII"), fileName, false); + assertTrue("Unrecognized lines: " + result.getUnrecognizedLines(), + result.getUnrecognizedLines().isEmpty()); + Map<String, Long> map = result.getEntries() + .iterator().next().getExitAddresses(); + assertEquals("Found: " + map, 3, map.size()); + assertTrue("Map: " + map, map.containsKey("81.7.17.171")); + assertTrue("Map: " + map, map.containsKey("81.7.17.172")); + assertTrue("Map: " + map, map.containsKey("81.7.17.173")); + } + + @Test(expected = DescriptorParseException.class) + public void testInsufficientInput0() throws Exception { + new ExitListImpl((tordnselAnnotation + insufficientInput[0]) + .getBytes("US-ASCII"), fileName, false); + } + + @Test(expected = DescriptorParseException.class) + public void testInsufficientInput1() throws Exception { + new ExitListImpl((tordnselAnnotation + insufficientInput[1]) + .getBytes("US-ASCII"), fileName, false); + } + + private static final String tordnselAnnotation = "@type tordnsel 1.0\n"; + private static final String fileName = "2015-09-01-00-02-02"; + private static final String[] insufficientInput = new String[] { + "Downloaded 2015-09-01 00:02:02\n" + + "ExitNode 0011BD2485AD45D984EC4159C88FC066E5E3300E\n" + + "Published 2015-08-31 16:17:30\n" + + "LastStatus 2015-08-31 17:03:18\n", + "Downloaded 2015-09-01 00:02:02\n" + + "ExitNode 0011BD2485AD45D984EC4159C88FC066E5E3300E\n" + + "LastStatus 2015-08-31 17:03:18\n" + + "ExitAddress 81.7.17.172 2015-08-31 18:10:52\n" }; + + private static final String multiExitAddressInput = + "Downloaded 2015-09-01 00:02:02\n" + + "ExitNode 0011BD2485AD45D984EC4159C88FC066E5E3300E\n" + + "Published 2015-08-31 16:17:30\n" + + "LastStatus 2015-08-31 17:03:18\n" + + "ExitAddress 81.7.17.171 2015-08-31 18:09:52\n" + + "ExitAddress 81.7.17.172 2015-08-31 18:10:52\n" + + "ExitAddress 81.7.17.173 2015-08-31 18:11:52\n"; + private static final String input = "Downloaded 2015-09-01 00:02:02\n" + + "ExitNode 0011BD2485AD45D984EC4159C88FC066E5E3300E\n" + + "Published 2015-08-31 16:17:30\n" + + "LastStatus 2015-08-31 17:03:18\n" + + "ExitAddress 162.247.72.201 2015-08-31 17:09:23\n" + + "ExitNode 0098C475875ABC4AA864738B1D1079F711C38287\n" + + "Published 2015-08-31 13:59:24\n" + + "LastStatus 2015-08-31 15:03:20\n" + + "ExitAddress 162.248.160.151 2015-08-31 15:07:27\n" + + "ExitNode 00C4B4731658D3B4987132A3F77100CFCB190D97\n" + + "Published 2015-08-31 17:47:52\n" + + "LastStatus 2015-08-31 18:03:17\n" + + "ExitAddress 81.7.17.171 2015-08-31 18:09:52\n" + + "ExitAddress 81.7.17.172 2015-08-31 18:10:52\n" + + "ExitAddress 81.7.17.173 2015-08-31 18:11:52\n" + + "ExitNode 00F2D93EBAF2F51D6EE4DCB0F37D91D72F824B16\n" + + "Published 2015-08-31 14:39:05\n" + + "LastStatus 2015-08-31 16:02:18\n" + + "ExitAddress 23.239.18.57 2015-08-31 16:06:07\n" + + "ExitNode 011B1D1E876B2C835D01FB9D407F2E00B28077F6\n" + + "Published 2015-08-31 05:14:35\n" + + "LastStatus 2015-08-31 06:03:29\n" + + "ExitAddress 104.131.51.150 2015-08-31 06:04:07\n"; +} +