December 2011 - tor-commits - lists.torproject.org

[doctor/master] metrics-lib now returns null for no version recommendations.
by karsten＠torproject.org 13 Dec '11

13 Dec '11

commit f48bd96573471df5fada559e14accf3e60ec8892 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Dec 13 12:48:18 2011 +0100 metrics-lib now returns null for no version recommendations. --- src/org/torproject/doctor/Checker.java | 4 ++-- .../torproject/doctor/MetricsWebsiteReport.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/org/torproject/doctor/Checker.java b/src/org/torproject/doctor/Checker.java index 795a730..ee4e29e 100755 --- a/src/org/torproject/doctor/Checker.java +++ b/src/org/torproject/doctor/Checker.java @@ -208,7 +208,7 @@ public class Checker { SortedSet<String> unrecommendedClientVersions = new TreeSet<String>(), unrecommendedServerVersions = new TreeSet<String>(); for (RelayNetworkStatusVote vote : this.downloadedVotes) { - if (!vote.getRecommendedClientVersions().isEmpty() && + if (vote.getRecommendedClientVersions() != null && !downloadedConsensus.getRecommendedClientVersions().equals( vote.getRecommendedClientVersions())) { StringBuilder message = new StringBuilder(); @@ -218,7 +218,7 @@ public class Checker { } unrecommendedClientVersions.add(message.toString()); } - if (!vote.getRecommendedServerVersions().isEmpty() && + if (vote.getRecommendedServerVersions() != null && !downloadedConsensus.getRecommendedServerVersions().equals( vote.getRecommendedServerVersions())) { StringBuilder message = new StringBuilder(); diff --git a/src/org/torproject/doctor/MetricsWebsiteReport.java b/src/org/torproject/doctor/MetricsWebsiteReport.java index f4552e0..8d0bb0b 100755 --- a/src/org/torproject/doctor/MetricsWebsiteReport.java +++ b/src/org/torproject/doctor/MetricsWebsiteReport.java @@ -331,7 +331,7 @@ public class MetricsWebsiteReport { for (RelayNetworkStatusVote vote : this.downloadedVotes.values()) { SortedSet<String> voteRecommendedClientVersions = vote.getRecommendedClientVersions(); - if (!voteRecommendedClientVersions.isEmpty()) { + if (voteRecommendedClientVersions != null) { if (downloadedConsensus.getRecommendedClientVersions().equals( voteRecommendedClientVersions)) { this.bw.write(" <tr>\n" @@ -359,7 +359,7 @@ public class MetricsWebsiteReport { } SortedSet<String> voteRecommendedServerVersions = vote.getRecommendedServerVersions(); - if (!voteRecommendedServerVersions.isEmpty()) { + if (voteRecommendedServerVersions != null) { if (downloadedConsensus.getRecommendedServerVersions().equals( voteRecommendedServerVersions)) { this.bw.write(" <tr>\n"

1 0

[metrics-lib/master] Distinguish between not recommending anything and recommending nothing.
by karsten＠torproject.org 13 Dec '11

13 Dec '11

commit d17812139aa1642883b932a0716b39e94141d180 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Dec 13 12:47:24 2011 +0100 Distinguish between not recommending anything and recommending nothing. --- .../descriptor/RelayNetworkStatusConsensus.java | 6 ++++-- .../descriptor/RelayNetworkStatusVote.java | 6 ++++-- .../impl/RelayNetworkStatusConsensusImpl.java | 16 ++++++++-------- .../impl/RelayNetworkStatusVoteImpl.java | 16 ++++++++-------- 4 files changed, 24 insertions(+), 20 deletions(-) diff --git a/src/org/torproject/descriptor/RelayNetworkStatusConsensus.java b/src/org/torproject/descriptor/RelayNetworkStatusConsensus.java index 452a0ba..17beeb5 100755 --- a/src/org/torproject/descriptor/RelayNetworkStatusConsensus.java +++ b/src/org/torproject/descriptor/RelayNetworkStatusConsensus.java @@ -27,10 +27,12 @@ public interface RelayNetworkStatusConsensus extends Descriptor { /* Return a list of the voting-delay times in seconds. */ public List<Long> getVotingDelay(); - /* Return cecommended server versions. */ + /* Return recommended server versions or null if the consensus doesn't + * contain recommended server versions. */ public SortedSet<String> getRecommendedServerVersions(); - /* Return recommended client versions. */ + /* Return recommended client versions or null if the consensus doesn't + * contain recommended client versions. */ public SortedSet<String> getRecommendedClientVersions(); /* Return known relay flags. */ diff --git a/src/org/torproject/descriptor/RelayNetworkStatusVote.java b/src/org/torproject/descriptor/RelayNetworkStatusVote.java index 7d85a0c..f591c10 100755 --- a/src/org/torproject/descriptor/RelayNetworkStatusVote.java +++ b/src/org/torproject/descriptor/RelayNetworkStatusVote.java @@ -30,10 +30,12 @@ public interface RelayNetworkStatusVote extends Descriptor { /* Return a list of the voting-delay times in seconds. */ public List<Long> getVotingDelay(); - /* Return cecommended server versions. */ + /* Return recommended server versions or null if the authority doesn't + * recommend server versions. */ public SortedSet<String> getRecommendedServerVersions(); - /* Return recommended client versions. */ + /* Return recommended client versions or null if the authority doesn't + * recommend server versions. */ public SortedSet<String> getRecommendedClientVersions(); /* Return known relay flags. */ diff --git a/src/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java b/src/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java index 3d2b0a2..7cbe94c 100755 --- a/src/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java +++ b/src/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java @@ -94,10 +94,10 @@ public class RelayNetworkStatusConsensusImpl this.votingDelay.add(Long.parseLong(votingDelayString)); } } else if (line.startsWith("client-versions ")) { - this.recommendedClientVersions.addAll( + this.recommendedClientVersions = new TreeSet<String>( Arrays.asList(line.split(" ")[1].split(","))); } else if (line.startsWith("server-versions ")) { - this.recommendedServerVersions.addAll( + this.recommendedServerVersions = new TreeSet<String>( Arrays.asList(line.split(" ")[1].split(","))); } else if (line.startsWith("known-flags ")) { for (String flag : line.substring("known-flags ".length()). @@ -257,16 +257,16 @@ public class RelayNetworkStatusConsensusImpl return new ArrayList<Long>(this.votingDelay); } - private SortedSet<String> recommendedClientVersions = - new TreeSet<String>(); + private SortedSet<String> recommendedClientVersions; public SortedSet<String> getRecommendedClientVersions() { - return new TreeSet<String>(this.recommendedClientVersions); + return this.recommendedClientVersions == null ? null : + new TreeSet<String>(this.recommendedClientVersions); } - private SortedSet<String> recommendedServerVersions = - new TreeSet<String>(); + private SortedSet<String> recommendedServerVersions; public SortedSet<String> getRecommendedServerVersions() { - return new TreeSet<String>(this.recommendedServerVersions); + return this.recommendedServerVersions == null ? null : + new TreeSet<String>(this.recommendedServerVersions); } private SortedSet<String> knownFlags = new TreeSet<String>(); diff --git a/src/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java b/src/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java index fa5b4d5..bf2668b 100755 --- a/src/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java +++ b/src/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java @@ -100,10 +100,10 @@ public class RelayNetworkStatusVoteImpl this.votingDelay.add(Long.parseLong(votingDelayString)); } } else if (line.startsWith("client-versions ")) { - this.recommendedClientVersions.addAll( + this.recommendedClientVersions = new TreeSet<String>( Arrays.asList(line.split(" ")[1].split(","))); } else if (line.startsWith("server-versions ")) { - this.recommendedServerVersions.addAll( + this.recommendedServerVersions = new TreeSet<String>( Arrays.asList(line.split(" ")[1].split(","))); } else if (line.startsWith("known-flags ")) { for (String flag : line.substring("known-flags ".length()). @@ -233,16 +233,16 @@ public class RelayNetworkStatusVoteImpl return new ArrayList<Long>(this.votingDelay); } - private SortedSet<String> recommendedClientVersions = - new TreeSet<String>(); + private SortedSet<String> recommendedClientVersions; public SortedSet<String> getRecommendedClientVersions() { - return new TreeSet<String>(this.recommendedClientVersions); + return this.recommendedClientVersions == null ? null : + new TreeSet<String>(this.recommendedClientVersions); } - private SortedSet<String> recommendedServerVersions = - new TreeSet<String>(); + private SortedSet<String> recommendedServerVersions; public SortedSet<String> getRecommendedServerVersions() { - return new TreeSet<String>(this.recommendedServerVersions); + return this.recommendedServerVersions == null ? null : + new TreeSet<String>(this.recommendedServerVersions); } private SortedSet<String> knownFlags = new TreeSet<String>();

1 0

[doctor/master] Don't warn if authorities don't recommend any versions.
by karsten＠torproject.org 13 Dec '11

13 Dec '11

commit b018b58523a6041b3bbd2dee9d7d2d876bb9c491 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Dec 13 12:05:47 2011 +0100 Don't warn if authorities don't recommend any versions. Spotted by Robert. --- src/org/torproject/doctor/Checker.java | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/org/torproject/doctor/Checker.java b/src/org/torproject/doctor/Checker.java index ee4e29e..795a730 100755 --- a/src/org/torproject/doctor/Checker.java +++ b/src/org/torproject/doctor/Checker.java @@ -208,7 +208,7 @@ public class Checker { SortedSet<String> unrecommendedClientVersions = new TreeSet<String>(), unrecommendedServerVersions = new TreeSet<String>(); for (RelayNetworkStatusVote vote : this.downloadedVotes) { - if (vote.getRecommendedClientVersions() != null && + if (!vote.getRecommendedClientVersions().isEmpty() && !downloadedConsensus.getRecommendedClientVersions().equals( vote.getRecommendedClientVersions())) { StringBuilder message = new StringBuilder(); @@ -218,7 +218,7 @@ public class Checker { } unrecommendedClientVersions.add(message.toString()); } - if (vote.getRecommendedServerVersions() != null && + if (!vote.getRecommendedServerVersions().isEmpty() && !downloadedConsensus.getRecommendedServerVersions().equals( vote.getRecommendedServerVersions())) { StringBuilder message = new StringBuilder();

1 0

[metrics-web/master] Add a few <hr>'s.
by karsten＠torproject.org 13 Dec '11

13 Dec '11

commit ddd2b6ecad5b0dea5f9629eb48d4fbd4a6c854f1 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Dec 13 11:55:56 2011 +0100 Add a few <hr>'s. --- web/WEB-INF/users.jsp | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diff --git a/web/WEB-INF/users.jsp b/web/WEB-INF/users.jsp index bdd01b4..b3dfa31 100644 --- a/web/WEB-INF/users.jsp +++ b/web/WEB-INF/users.jsp @@ -62,6 +62,7 @@ based on the requests seen by a few dozen directory mirrors.</p> </p> </div> </form> +<hr> <a name="direct-users-table"></a> <p><b>Top-10 countries by directly connecting users:</b></p> <table> @@ -92,6 +93,7 @@ based on the requests seen by a few dozen directory mirrors.</p> </p> </div> </form> +<hr> <a name="censorship-events"></a> <p><b>Top-10 countries by possible censorship events (<a href="papers/detector-2011-09-09.pdf">BETA</a>):</b></p> @@ -125,6 +127,7 @@ based on the requests seen by a few dozen directory mirrors.</p> </p> </div> </form> +<hr> <p><a href="csv/direct-users.csv">CSV</a> file containing daily directly connecting users by country.</p> <p><a href="csv/monthly-users-peak.csv">CSV</a> file containing peak daily @@ -170,6 +173,7 @@ by a few hundred bridges.</p> </p> </div> </form> +<hr> <p><a href="csv/bridge-users.csv">CSV</a> file containing all data.</p> <p><a href="csv/monthly-users-peak.csv">CSV</a> file containing peak daily Tor users (direct and bridge) per month by country.</p>

1 0

[doctor/master] Add a missing word.
by karsten＠torproject.org 13 Dec '11

13 Dec '11

commit 1619c2cf49ddbe485511376b2483f7c0bb249f4c Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Dec 13 11:04:39 2011 +0100 Add a missing word. --- .../torproject/doctor/MetricsWebsiteReport.java | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/org/torproject/doctor/MetricsWebsiteReport.java b/src/org/torproject/doctor/MetricsWebsiteReport.java index 1529461..f4552e0 100755 --- a/src/org/torproject/doctor/MetricsWebsiteReport.java +++ b/src/org/torproject/doctor/MetricsWebsiteReport.java @@ -163,8 +163,8 @@ public class MetricsWebsiteReport { this.bw.write(dateTimeFormat.format( this.downloadedConsensus.getValidAfterMillis())); } - this.bw.write(". <i>Note that it takes up to 15 to learn about new " - + "consensus and votes and process them.</i></p>\n"); + this.bw.write(". <i>Note that it takes up to 15 minutes to learn " + + "about new consensus and votes and process them.</i></p>\n"); } /* Write the lists of known flags. */

1 0

[doctor/master] Rely on metrics-lib to parse consensuses and votes.
by karsten＠torproject.org 13 Dec '11

13 Dec '11

commit ad912387e433d29726eefcaca301967f4ce50427 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Dec 13 10:18:07 2011 +0100 Rely on metrics-lib to parse consensuses and votes. --- src/org/torproject/doctor/Checker.java | 91 ++++++--- src/org/torproject/doctor/Download.java | 46 ----- src/org/torproject/doctor/DownloadStatistics.java | 25 ++- src/org/torproject/doctor/Downloader.java | 62 ++----- src/org/torproject/doctor/Main.java | 52 ++---- .../torproject/doctor/MetricsWebsiteReport.java | 139 ++++++++------ src/org/torproject/doctor/Parser.java | 180 ----------------- src/org/torproject/doctor/Report.java | 27 --- src/org/torproject/doctor/Status.java | 205 -------------------- src/org/torproject/doctor/StatusEntry.java | 48 ----- src/org/torproject/doctor/StatusFileReport.java | 17 +-- 11 files changed, 194 insertions(+), 698 deletions(-) diff --git a/src/org/torproject/doctor/Checker.java b/src/org/torproject/doctor/Checker.java index 4d6493f..ee4e29e 100755 --- a/src/org/torproject/doctor/Checker.java +++ b/src/org/torproject/doctor/Checker.java @@ -5,6 +5,7 @@ package org.torproject.doctor; import java.io.*; import java.text.*; import java.util.*; +import org.torproject.descriptor.*; /* Check a given consensus and votes for irregularities and write results * to a warnings map consisting of warning type and details. */ @@ -26,12 +27,15 @@ public class Checker { } /* Downloaded consensus and corresponding votes for processing. */ - private SortedMap<String, Status> downloadedConsensuses; - private Status downloadedConsensus; - private SortedSet<Status> downloadedVotes; + private SortedMap<String, RelayNetworkStatusConsensus> + downloadedConsensuses = new TreeMap<String, + RelayNetworkStatusConsensus>(); + private RelayNetworkStatusConsensus downloadedConsensus; + private List<RelayNetworkStatusVote> downloadedVotes = + new ArrayList<RelayNetworkStatusVote>(); public void processDownloadedConsensuses( - SortedMap<String, Status> downloadedConsensuses) { - this.downloadedConsensuses = downloadedConsensuses; + List<DescriptorRequest> downloads) { + this.storeDownloads(downloads); this.findMostRecentConsensus(); this.checkMissingConsensuses(); this.checkAllConsensusesFresh(); @@ -51,10 +55,29 @@ public class Checker { } } + /* Store consensuses and votes in a way that we can process them more + * easily. */ + private void storeDownloads(List<DescriptorRequest> downloads) { + for (DescriptorRequest request : downloads) { + for (Descriptor descriptor : request.getDescriptors()) { + if (descriptor instanceof RelayNetworkStatusConsensus) { + this.downloadedConsensuses.put(request.getDirectoryNickname(), + (RelayNetworkStatusConsensus) descriptor); + } else if (descriptor instanceof RelayNetworkStatusVote) { + this.downloadedVotes.add((RelayNetworkStatusVote) descriptor); + } else { + System.err.println("Did not expect a descriptor of type " + + descriptor.getClass() + ". Ignoring."); + } + } + } + } + /* Find most recent consensus and corresponding votes. */ private void findMostRecentConsensus() { long mostRecentValidAfterMillis = -1L; - for (Status downloadedConsensus : downloadedConsensuses.values()) { + for (RelayNetworkStatusConsensus downloadedConsensus : + downloadedConsensuses.values()) { if (downloadedConsensus.getValidAfterMillis() > mostRecentValidAfterMillis) { this.downloadedConsensus = downloadedConsensus; @@ -62,9 +85,6 @@ public class Checker { downloadedConsensus.getValidAfterMillis(); } } - if (this.downloadedConsensus != null) { - this.downloadedVotes = this.downloadedConsensus.getVotes(); - } } /* Check if any directory authority didn't tell us a consensus. */ @@ -87,9 +107,10 @@ public class Checker { private void checkAllConsensusesFresh() { long fresh = System.currentTimeMillis() - 60L * 60L * 1000L; SortedSet<String> nonFresh = new TreeSet<String>(); - for (Map.Entry<String, Status> e : downloadedConsensuses.entrySet()) { + for (Map.Entry<String, RelayNetworkStatusConsensus> e : + downloadedConsensuses.entrySet()) { String nickname = e.getKey(); - Status downloadedConsensus = e.getValue(); + RelayNetworkStatusConsensus downloadedConsensus = e.getValue(); if (downloadedConsensus.getValidAfterMillis() < fresh) { nonFresh.add(nickname); } @@ -107,14 +128,16 @@ public class Checker { /* Check if all downloaded consensuses contain the same set of votes. */ private void checkContainedVotes() { Set<String> allVotes = new HashSet<String>(); - for (Status consensus : downloadedConsensuses.values()) { - allVotes.addAll(consensus.getContainedVotes()); + for (RelayNetworkStatusConsensus consensus : + downloadedConsensuses.values()) { + allVotes.addAll(consensus.getDirSourceEntries().keySet()); } SortedSet<String> missingVotes = new TreeSet<String>(); - for (Map.Entry<String, Status> e : downloadedConsensuses.entrySet()) { + for (Map.Entry<String, RelayNetworkStatusConsensus> e : + downloadedConsensuses.entrySet()) { String nickname = e.getKey(); - Status downloadedConsensus = e.getValue(); - if (!downloadedConsensus.getContainedVotes().containsAll( + RelayNetworkStatusConsensus downloadedConsensus = e.getValue(); + if (!downloadedConsensus.getDirSourceEntries().keySet().containsAll( allVotes)) { missingVotes.add(nickname); } @@ -133,11 +156,13 @@ public class Checker { * authorities. */ private void checkConsensusSignatures() { SortedSet<String> missingSignatures = new TreeSet<String>(); - for (Map.Entry<String, Status> e : downloadedConsensuses.entrySet()) { + for (Map.Entry<String, RelayNetworkStatusConsensus> e : + downloadedConsensuses.entrySet()) { String nickname = e.getKey(); - Status downloadedConsensus = e.getValue(); - if (!downloadedConsensus.getContainedSignatures().containsAll( - downloadedConsensus.getContainedVotes())) { + RelayNetworkStatusConsensus downloadedConsensus = e.getValue(); + if (!downloadedConsensus.getDirectorySignatures().keySet(). + containsAll(downloadedConsensus.getDirSourceEntries(). + keySet())) { missingSignatures.add(nickname); } } @@ -152,7 +177,8 @@ public class Checker { } /* Check if the most recent consensus is older than 1 hour. */ - private boolean isConsensusFresh(Status consensus) { + private boolean isConsensusFresh( + RelayNetworkStatusConsensus consensus) { return (consensus.getValidAfterMillis() >= System.currentTimeMillis() - 60L * 60L * 1000L); } @@ -160,9 +186,9 @@ public class Checker { /* Check supported consensus methods of all votes. */ private void checkConsensusMethods() { SortedSet<String> dirs = new TreeSet<String>(); - for (Status vote : this.downloadedVotes) { + for (RelayNetworkStatusVote vote : this.downloadedVotes) { if (!vote.getConsensusMethods().contains( - this.downloadedConsensus.getConsensusMethods().last())) { + this.downloadedConsensus.getConsensusMethod())) { dirs.add(vote.getNickname()); } } @@ -181,7 +207,7 @@ public class Checker { private void checkRecommendedVersions() { SortedSet<String> unrecommendedClientVersions = new TreeSet<String>(), unrecommendedServerVersions = new TreeSet<String>(); - for (Status vote : this.downloadedVotes) { + for (RelayNetworkStatusVote vote : this.downloadedVotes) { if (vote.getRecommendedClientVersions() != null && !downloadedConsensus.getRecommendedClientVersions().equals( vote.getRecommendedClientVersions())) { @@ -231,7 +257,7 @@ public class Checker { + "cbtmintimeout,cbtinitialtimeout,perconnbwburst,perconnbwrate"). split(","))); SortedSet<String> conflicts = new TreeSet<String>(); - for (Status vote : this.downloadedVotes) { + for (RelayNetworkStatusVote vote : this.downloadedVotes) { Map<String, String> voteConsensusParams = vote.getConsensusParams(); boolean conflictOrInvalid = false; @@ -272,7 +298,7 @@ public class Checker { SortedMap<String, String> expiringCertificates = new TreeMap<String, String>(); long now = System.currentTimeMillis(); - for (Status vote : this.downloadedVotes) { + for (RelayNetworkStatusVote vote : this.downloadedVotes) { long voteDirKeyExpiresMillis = vote.getDirKeyExpiresMillis(); if (voteDirKeyExpiresMillis - 14L * 24L * 60L * 60L * 1000L < now) { expiringCertificates.put(vote.getNickname(), @@ -299,7 +325,7 @@ public class Checker { + "tor26,urras").split(","))); SortedSet<String> missingVotes = new TreeSet<String>(knownAuthorities); - for (Status vote : this.downloadedVotes) { + for (RelayNetworkStatusVote vote : this.downloadedVotes) { missingVotes.remove(vote.getNickname()); } if (!missingVotes.isEmpty()) { @@ -316,8 +342,15 @@ public class Checker { private void checkBandwidthScanners() { SortedSet<String> missingBandwidthScanners = new TreeSet<String>( Arrays.asList("ides,urras,moria1,gabelmoo,maatuska".split(","))); - for (Status vote : this.downloadedVotes) { - if (vote.getBandwidthWeights() > 0) { + for (RelayNetworkStatusVote vote : this.downloadedVotes) { + boolean containsMeasuredBandwidths = false; + for (NetworkStatusEntry entry : vote.getStatusEntries().values()) { + if (entry.getBandwidth().contains("Measured=")) { + containsMeasuredBandwidths = true; + break; + } + } + if (containsMeasuredBandwidths) { missingBandwidthScanners.remove(vote.getNickname()); } } diff --git a/src/org/torproject/doctor/Download.java b/src/org/torproject/doctor/Download.java deleted file mode 100644 index 5c84277..0000000 --- a/src/org/torproject/doctor/Download.java +++ /dev/null @@ -1,46 +0,0 @@ -/* Copyright 2011 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.doctor; - -public class Download { - - /* Nickname of the authority from which this download was made. */ - private String authority; - public String getAuthority() { - return this.authority; - } - - /* Request URL. */ - private String url; - public String getUrl() { - return this.url; - } - - /* Unparsed response string. */ - private String responseString; - public String getResponseString() { - return this.responseString; - } - - /* Request start timestamp. */ - private long requestStartMillis; - public long getRequestStartMillis() { - return this.requestStartMillis; - } - - /* Fetch time in millis. */ - private long fetchTime; - public long getFetchTime() { - return this.fetchTime; - } - - public Download(String authority, String url, String responseString, - long requestStartMillis, long fetchTime) { - this.authority = authority; - this.url = url; - this.responseString = responseString; - this.requestStartMillis = requestStartMillis; - this.fetchTime = fetchTime; - } -} - diff --git a/src/org/torproject/doctor/DownloadStatistics.java b/src/org/torproject/doctor/DownloadStatistics.java index 9eeffde..9df0333 100644 --- a/src/org/torproject/doctor/DownloadStatistics.java +++ b/src/org/torproject/doctor/DownloadStatistics.java @@ -4,21 +4,26 @@ package org.torproject.doctor; import java.io.*; import java.util.*; +import org.torproject.descriptor.*; public class DownloadStatistics { - public void memorizeFetchTimes(List<Download> downloadedConsensuses) { + public void memorizeFetchTimes(List<DescriptorRequest> downloads) { try { BufferedWriter bw = new BufferedWriter(new FileWriter( this.statisticsFile, true)); - for (Download downloadedConsensus : downloadedConsensuses) { - String authority = downloadedConsensus.getAuthority(); - long requestStartMillis = - downloadedConsensus.getRequestStartMillis(); - long fetchTimeMillis = downloadedConsensus.getFetchTime(); - String line = authority + "," - + String.valueOf(requestStartMillis) + "," - + String.valueOf(fetchTimeMillis); - bw.write(line + "\n"); + for (DescriptorRequest request : downloads) { + for (Descriptor descriptor : request.getDescriptors()) { + if (descriptor instanceof RelayNetworkStatusConsensus) { + String authority = request.getDirectoryNickname(); + long requestStartMillis = request.getRequestStart(); + long fetchTimeMillis = request.getRequestEnd() + - request.getRequestStart(); + String line = authority + "," + + String.valueOf(requestStartMillis) + "," + + String.valueOf(fetchTimeMillis); + bw.write(line + "\n"); + } + } } bw.close(); } catch (IOException e) { diff --git a/src/org/torproject/doctor/Downloader.java b/src/org/torproject/doctor/Downloader.java index a82ac8d..317b3b6 100755 --- a/src/org/torproject/doctor/Downloader.java +++ b/src/org/torproject/doctor/Downloader.java @@ -13,8 +13,8 @@ import org.torproject.descriptor.*; * votes. */ public class Downloader { - /* Download a new consensus and corresponding votes. */ - public void downloadFromAuthorities() { + /* Download the current consensus and corresponding votes. */ + public List<DescriptorRequest> downloadFromAuthorities() { RelayDescriptorDownloader downloader = DescriptorSourceFactory.createRelayDescriptorDownloader(); @@ -33,58 +33,24 @@ public class Downloader { downloader.setRequestTimeout(60L * 1000L); + List<DescriptorRequest> allRequests = + new ArrayList<DescriptorRequest>(); Iterator<DescriptorRequest> descriptorRequests = downloader.downloadDescriptors(); while (descriptorRequests.hasNext()) { - DescriptorRequest request = descriptorRequests.next(); - String authority = request.getDirectoryNickname(); - String requestUrl = request.getRequestUrl(); - long requestStart = request.getRequestStart(); - long fetchTime = request.getRequestEnd() - - request.getRequestStart(); - if (request.globalTimeoutHasExpired()) { - System.err.println("Global timeout has expired. Exiting."); - System.exit(1); - } else if (!request.requestTimeoutHasExpired()) { - if (request.getDescriptors().isEmpty()) { - /* No response. We'll realize later on if we're missing a - * consensus or vote. */ - continue; - } else if (request.getDescriptors().size() > 1) { - System.out.println("Response contains more than 1 " - + "descriptor. Considering only the first."); - } - Descriptor downloadedDescriptor = request.getDescriptors().get(0); - String response = new String(request.getDescriptors().get(0). - getRawDescriptorBytes()); - Download download = new Download(authority, requestUrl, response, - requestStart, fetchTime); - if (downloadedDescriptor instanceof - RelayNetworkStatusConsensus) { - this.downloadedConsensuses.add(download); - } else if (downloadedDescriptor instanceof - RelayNetworkStatusVote) { - this.downloadedVotes.add(download); - } else { - System.err.println("Did not expect a descriptor of type " - + downloadedDescriptor.getClass() + ". Ignoring."); - } + try { + allRequests.add(descriptorRequests.next()); + } catch (NoSuchElementException e) { + /* TODO In theory, this exception shouldn't be thrown. */ + System.err.println("Internal error: next() doesn't provide an " + + "element even though hasNext() returned true. Got " + + allRequests.size() + " elements so far. Stopping to " + + "request further elements."); + break; } } - } - - /* Return the previously downloaded (unparsed) consensus string by - * authority nickname. */ - private List<Download> downloadedConsensuses = - new ArrayList<Download>(); - public List<Download> getConsensuses() { - return this.downloadedConsensuses; - } - /* Return the previously downloaded (unparsed) vote strings. */ - private List<Download> downloadedVotes = new ArrayList<Download>(); - public List<Download> getVotes() { - return this.downloadedVotes; + return allRequests; } } diff --git a/src/org/torproject/doctor/Main.java b/src/org/torproject/doctor/Main.java index 0de6298..40973b4 100755 --- a/src/org/torproject/doctor/Main.java +++ b/src/org/torproject/doctor/Main.java @@ -3,53 +3,37 @@ package org.torproject.doctor; import java.util.*; +import org.torproject.descriptor.*; /* Coordinate the process of downloading consensus and votes to check * Tor's consensus health. */ public class Main { public static void main(String[] args) { - /* Initialize reports. */ - List<Report> reports = new ArrayList<Report>(); - reports.add(new MetricsWebsiteReport( - "website/consensus-health.html")); - reports.add(new StatusFileReport()); - /* Download consensus and corresponding votes from the directory * authorities. */ Downloader downloader = new Downloader(); - downloader.downloadFromAuthorities(); - List<Download> downloadedConsensuses = downloader.getConsensuses(); - List<Download> downloadedVotes = downloader.getVotes(); - - /* Write fetch times for requesting consensuses to disk and prepare - * statistics about fetch times in the last 7 days. */ - DownloadStatistics fetchStatistics = new DownloadStatistics(); - fetchStatistics.memorizeFetchTimes(downloadedConsensuses); - fetchStatistics.prepareStatistics(); + List<DescriptorRequest> downloads = + downloader.downloadFromAuthorities(); - /* Parse consensus and votes. */ - Parser parser = new Parser(); - SortedMap<String, Status> parsedDownloadedConsensuses = parser.parse( - downloadedConsensuses, downloadedVotes); - - /* Check consensus and votes for possible problems. */ + /* Check consensus and votes for possible problems and write warnings + * to status files. */ + StatusFileReport statusFile = new StatusFileReport(); Checker checker = new Checker(); - checker.processDownloadedConsensuses(parsedDownloadedConsensuses); + checker.processDownloadedConsensuses(downloads); SortedMap<Warning, String> warnings = checker.getWarnings(); + statusFile.processWarnings(warnings); + statusFile.writeReport(); - /* Pass warnings, consensuses, and votes to the reports, and finish - * writing them. */ - for (Report report : reports) { - report.processWarnings(warnings); - report.processDownloadedConsensuses(parsedDownloadedConsensuses); - report.includeFetchStatistics(fetchStatistics); - report.writeReport(); - } - - /* Terminate the program including any download threads that may still - * be running. */ - System.exit(0); + /* Write a complete consensus-health report to an HTML file. */ + MetricsWebsiteReport website = + new MetricsWebsiteReport("website/consensus-health.html"); + website.processDownloadedConsensuses(downloads); + DownloadStatistics fetchStatistics = new DownloadStatistics(); + fetchStatistics.memorizeFetchTimes(downloads); + fetchStatistics.prepareStatistics(); + website.includeFetchStatistics(fetchStatistics); + website.writeReport(); } } diff --git a/src/org/torproject/doctor/MetricsWebsiteReport.java b/src/org/torproject/doctor/MetricsWebsiteReport.java index 9ae897a..1529461 100755 --- a/src/org/torproject/doctor/MetricsWebsiteReport.java +++ b/src/org/torproject/doctor/MetricsWebsiteReport.java @@ -5,10 +5,11 @@ package org.torproject.doctor; import java.io.*; import java.text.*; import java.util.*; +import org.torproject.descriptor.*; /* Transform the most recent consensus and corresponding votes into an * HTML page showing possible irregularities. */ -public class MetricsWebsiteReport implements Report { +public class MetricsWebsiteReport { /* Date-time format to format timestamps. */ private static SimpleDateFormat dateTimeFormat; @@ -25,30 +26,35 @@ public class MetricsWebsiteReport implements Report { this.htmlOutputFile = new File(htmlOutputFilename); } - /* Process warnings. */ - public void processWarnings(SortedMap<Warning, String> warnings) { - /* We could use these warnings instead of running all checks - * ourselves. But we're not doing that yet. */ - } - /* Store the downloaded consensus and corresponding votes for later * processing. */ - private Status downloadedConsensus; - private SortedSet<Status> downloadedVotes; + private RelayNetworkStatusConsensus downloadedConsensus; + private SortedMap<String, RelayNetworkStatusVote> downloadedVotes = + new TreeMap<String, RelayNetworkStatusVote>(); public void processDownloadedConsensuses( - SortedMap<String, Status> downloadedConsensuses) { + List<DescriptorRequest> downloads) { long mostRecentValidAfterMillis = -1L; - for (Status downloadedConsensus : downloadedConsensuses.values()) { - if (downloadedConsensus.getValidAfterMillis() > - mostRecentValidAfterMillis) { - this.downloadedConsensus = downloadedConsensus; - mostRecentValidAfterMillis = - downloadedConsensus.getValidAfterMillis(); + for (DescriptorRequest request : downloads) { + for (Descriptor descriptor : request.getDescriptors()) { + if (descriptor instanceof RelayNetworkStatusConsensus) { + RelayNetworkStatusConsensus downloadedConsensus = + (RelayNetworkStatusConsensus) descriptor; + if (downloadedConsensus.getValidAfterMillis() > + mostRecentValidAfterMillis) { + this.downloadedConsensus = downloadedConsensus; + mostRecentValidAfterMillis = + downloadedConsensus.getValidAfterMillis(); + } + } else if (descriptor instanceof RelayNetworkStatusVote) { + RelayNetworkStatusVote vote = + (RelayNetworkStatusVote) descriptor; + this.downloadedVotes.put(vote.getNickname(), vote); + } else { + System.err.println("Did not expect a descriptor of type " + + descriptor.getClass() + ". Ignoring."); + } } } - if (this.downloadedConsensus != null) { - this.downloadedVotes = this.downloadedConsensus.getVotes(); - } } /* Store the DownloadStatistics reference to request download statistics @@ -177,7 +183,7 @@ public class MetricsWebsiteReport implements Report { if (this.downloadedVotes.size() < 1) { this.bw.write(" <tr><td>(No votes.)</td><td></td></tr>\n"); } else { - for (Status vote : this.downloadedVotes) { + for (RelayNetworkStatusVote vote : this.downloadedVotes.values()) { this.bw.write(" <tr>\n" + " <td>" + vote.getNickname() + "</td>\n" + " <td>known-flags"); @@ -218,24 +224,36 @@ public class MetricsWebsiteReport implements Report { this.bw.write(" <tr><td>(No votes.)</td><td></td><td></td>" + "</tr>\n"); } else { - for (Status vote : this.downloadedVotes) { + for (RelayNetworkStatusVote vote : this.downloadedVotes.values()) { + int runningRelays = 0; + for (NetworkStatusEntry entry : + vote.getStatusEntries().values()) { + if (entry.getFlags().contains("Running")) { + runningRelays++; + } + } this.bw.write(" <tr>\n" + " <td>" + vote.getNickname() + "</td>\n" + " <td>" + vote.getStatusEntries().size() + " total</td>\n" - + " <td>" + vote.getRunningRelays() - + " Running</td>\n" + + " <td>" + runningRelays + " Running</td>\n" + " </tr>\n"); } } + int runningRelays = 0; + for (NetworkStatusEntry entry : + this.downloadedConsensus.getStatusEntries().values()) { + if (entry.getFlags().contains("Running")) { + runningRelays++; + } + } this.bw.write(" <tr>\n" + " <td><font color=\"blue\">consensus</font>" + "</td>\n" + " <td><font color=\"blue\">" + this.downloadedConsensus.getStatusEntries().size() + " total</font></td>\n" - + " <td><font color=\"blue\">" - + this.downloadedConsensus.getRunningRelays() + + " <td><font color=\"blue\">" + runningRelays + " Running</font></td>\n" + " </tr>\n" + " </table>\n"); @@ -258,11 +276,10 @@ public class MetricsWebsiteReport implements Report { if (this.downloadedVotes.size() < 1) { this.bw.write(" <tr><td>(No votes.)</td><td></td></tr>\n"); } else { - for (Status vote : this.downloadedVotes) { - SortedSet<Integer> consensusMethods = - vote.getConsensusMethods(); + for (RelayNetworkStatusVote vote : this.downloadedVotes.values()) { + List<Integer> consensusMethods = vote.getConsensusMethods(); if (consensusMethods.contains( - this.downloadedConsensus.getConsensusMethods().last())) { + this.downloadedConsensus.getConsensusMethod())) { this.bw.write(" <tr>\n" + " <td>" + vote.getNickname() + "</td>\n" + " <td>consensus-methods"); @@ -289,7 +306,7 @@ public class MetricsWebsiteReport implements Report { + " <td><font color=\"blue\">consensus</font>" + "</td>\n" + " <td><font color=\"blue\">consensus-method " - + this.downloadedConsensus.getConsensusMethods().last() + + this.downloadedConsensus.getConsensusMethod() + "</font></td>\n" + " </tr>\n" + " </table>\n"); @@ -311,10 +328,10 @@ public class MetricsWebsiteReport implements Report { if (this.downloadedVotes.size() < 1) { this.bw.write(" <tr><td>(No votes.)</td><td></td></tr>\n"); } else { - for (Status vote : this.downloadedVotes) { + for (RelayNetworkStatusVote vote : this.downloadedVotes.values()) { SortedSet<String> voteRecommendedClientVersions = vote.getRecommendedClientVersions(); - if (voteRecommendedClientVersions != null) { + if (!voteRecommendedClientVersions.isEmpty()) { if (downloadedConsensus.getRecommendedClientVersions().equals( voteRecommendedClientVersions)) { this.bw.write(" <tr>\n" @@ -342,7 +359,7 @@ public class MetricsWebsiteReport implements Report { } SortedSet<String> voteRecommendedServerVersions = vote.getRecommendedServerVersions(); - if (voteRecommendedServerVersions != null) { + if (!voteRecommendedServerVersions.isEmpty()) { if (downloadedConsensus.getRecommendedServerVersions().equals( voteRecommendedServerVersions)) { this.bw.write(" <tr>\n" @@ -415,7 +432,7 @@ public class MetricsWebsiteReport implements Report { + "cbtmintimeout,cbtinitialtimeout").split(","))); Map<String, String> consensusConsensusParams = downloadedConsensus.getConsensusParams(); - for (Status vote : this.downloadedVotes) { + for (RelayNetworkStatusVote vote : this.downloadedVotes.values()) { Map<String, String> voteConsensusParams = vote.getConsensusParams(); boolean conflictOrInvalid = false; @@ -485,7 +502,7 @@ public class MetricsWebsiteReport implements Report { if (this.downloadedVotes.size() < 1) { this.bw.write(" <tr><td>(No votes.)</td><td></td></tr>\n"); } else { - for (Status vote : this.downloadedVotes) { + for (RelayNetworkStatusVote vote : this.downloadedVotes.values()) { long voteDirKeyExpiresMillis = vote.getDirKeyExpiresMillis(); if (voteDirKeyExpiresMillis - 14L * 24L * 60L * 60L * 1000L < System.currentTimeMillis()) { @@ -530,11 +547,17 @@ public class MetricsWebsiteReport implements Report { if (this.downloadedVotes.size() < 1) { this.bw.write(" <tr><td>(No votes.)</td><td></td></tr>\n"); } else { - for (Status vote : this.downloadedVotes) { - if (vote.getBandwidthWeights() > 0) { + for (RelayNetworkStatusVote vote : this.downloadedVotes.values()) { + int bandwidthWeights = 0; + for (NetworkStatusEntry entry : vote.getStatusEntries().values()) { + if (entry.getBandwidth().contains("Measured=")) { + bandwidthWeights++; + } + } + if (bandwidthWeights > 0) { this.bw.write(" <tr>\n" + " <td>" + vote.getNickname() + "</td>\n" - + " <td>" + vote.getBandwidthWeights() + + " <td>" + bandwidthWeights + " Measured values in w lines</td>\n" + " </tr>\n"); } @@ -550,8 +573,14 @@ public class MetricsWebsiteReport implements Report { + " <h3><a href=\"#authorityversions\" class=\"anchor\">" + "Authority versions</a></h3>\n" + " <br>\n"); - Map<String, String> authorityVersions = - this.downloadedConsensus.getAuthorityVersions(); + SortedMap<String, String> authorityVersions = + new TreeMap<String, String>(); + for (NetworkStatusEntry entry : + this.downloadedConsensus.getStatusEntries().values()) { + if (entry.getFlags().contains("Authority")) { + authorityVersions.put(entry.getNickname(), entry.getVersion()); + } + } if (authorityVersions.size() < 1) { this.bw.write(" <p>(No relays with Authority flag found.)" + "</p>\n"); @@ -671,13 +700,14 @@ public class MetricsWebsiteReport implements Report { } this.bw.write(" </colgroup>\n"); SortedMap<String, String> allRelays = new TreeMap<String, String>(); - for (Status vote : this.downloadedVotes) { - for (StatusEntry statusEntry : vote.getStatusEntries().values()) { + for (RelayNetworkStatusVote vote : this.downloadedVotes.values()) { + for (NetworkStatusEntry statusEntry : + vote.getStatusEntries().values()) { allRelays.put(statusEntry.getFingerprint(), statusEntry.getNickname()); } } - for (StatusEntry statusEntry : + for (NetworkStatusEntry statusEntry : this.downloadedConsensus.getStatusEntries().values()) { allRelays.put(statusEntry.getFingerprint(), statusEntry.getNickname()); @@ -699,7 +729,7 @@ public class MetricsWebsiteReport implements Report { private void writeRelayFlagsTableHeader() throws IOException { this.bw.write(" <tr><td><br><b>Fingerprint</b></td>" + "<td><br><b>Nickname</b></td>\n"); - for (Status vote : this.downloadedVotes) { + for (RelayNetworkStatusVote vote : this.downloadedVotes.values()) { String shortDirName = vote.getNickname().length() > 6 ? vote.getNickname().substring(0, 5) + "." : vote.getNickname(); @@ -714,8 +744,7 @@ public class MetricsWebsiteReport implements Report { this.bw.write(" <tr>\n"); if (this.downloadedConsensus.containsStatusEntry(fingerprint) && this.downloadedConsensus.getStatusEntry(fingerprint).getFlags(). - contains("Named") && - !Character.isDigit(nickname.charAt(0))) { + contains("Named") && !Character.isDigit(nickname.charAt(0))) { this.bw.write(" <td id=\"" + nickname + "\"><a href=\"relay.html?fingerprint=" + fingerprint + "\" target=\"_blank\">" @@ -727,18 +756,18 @@ public class MetricsWebsiteReport implements Report { } this.bw.write(" <td>" + nickname + "</td>\n"); SortedSet<String> relevantFlags = new TreeSet<String>(); - for (Status vote : this.downloadedVotes) { + for (RelayNetworkStatusVote vote : this.downloadedVotes.values()) { if (vote.containsStatusEntry(fingerprint)) { relevantFlags.addAll(vote.getStatusEntry(fingerprint).getFlags()); } } SortedSet<String> consensusFlags = null; if (this.downloadedConsensus.containsStatusEntry(fingerprint)) { - consensusFlags = this.downloadedConsensus. - getStatusEntry(fingerprint).getFlags(); + consensusFlags = this.downloadedConsensus.getStatusEntries().get( + fingerprint).getFlags(); relevantFlags.addAll(consensusFlags); } - for (Status vote : this.downloadedVotes) { + for (RelayNetworkStatusVote vote : this.downloadedVotes.values()) { if (vote.containsStatusEntry(fingerprint)) { SortedSet<String> flags = vote.getStatusEntry(fingerprint). getFlags(); @@ -818,7 +847,7 @@ public class MetricsWebsiteReport implements Report { + "<td><b>In vote and consensus</b></td>" + "<td><b>Only in consensus</b></td>\n"); Set<String> allFingerprints = new HashSet<String>(); - for (Status vote : this.downloadedVotes) { + for (RelayNetworkStatusVote vote : this.downloadedVotes.values()) { allFingerprints.addAll(vote.getStatusEntries().keySet()); } allFingerprints.addAll(this.downloadedConsensus.getStatusEntries(). @@ -832,9 +861,9 @@ public class MetricsWebsiteReport implements Report { for (String fingerprint : allFingerprints) { SortedSet<String> consensusFlags = this.downloadedConsensus.containsStatusEntry(fingerprint) ? - this.downloadedConsensus.getStatusEntry(fingerprint).getFlags() : - null; - for (Status vote : this.downloadedVotes) { + this.downloadedConsensus.getStatusEntry(fingerprint).getFlags() + : null; + for (RelayNetworkStatusVote vote : this.downloadedVotes.values()) { String dir = vote.getNickname(); if (vote.containsStatusEntry(fingerprint)) { SortedSet<String> flags = vote.getStatusEntry(fingerprint). @@ -868,7 +897,7 @@ public class MetricsWebsiteReport implements Report { } } } - for (Status vote : this.downloadedVotes) { + for (RelayNetworkStatusVote vote : this.downloadedVotes.values()) { String dir = vote.getNickname(); int i = 0; for (String flag : vote.getKnownFlags()) { diff --git a/src/org/torproject/doctor/Parser.java b/src/org/torproject/doctor/Parser.java deleted file mode 100755 index d318f29..0000000 --- a/src/org/torproject/doctor/Parser.java +++ /dev/null @@ -1,180 +0,0 @@ -/* Copyright 2011 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.doctor; - -import java.io.*; -import java.text.*; -import java.util.*; -import org.apache.commons.codec.binary.*; - -/* Parse a network status consensus or vote. */ -public class Parser { - - /* Parse and return a consensus and corresponding votes, or null if - * something goes wrong. */ - public SortedMap<String, Status> parse( - List<Download> downloadedConsensuses, - List<Download> downloadedVotes) { - SortedSet<Status> parsedVotes = new TreeSet<Status>(); - for (Download downloadedVote : downloadedVotes) { - String voteString = downloadedVote.getResponseString(); - long fetchTime = downloadedVote.getFetchTime(); - Status parsedVote = this.parseConsensusOrVote(voteString, fetchTime, - false); - if (parsedVote != null) { - parsedVotes.add(parsedVote); - } - } - SortedMap<String, Status> parsedConsensuses = - new TreeMap<String, Status>(); - for (Download downloadedConsensus : downloadedConsensuses) { - String nickname = downloadedConsensus.getAuthority(); - String consensusString = downloadedConsensus.getResponseString(); - long fetchTime = downloadedConsensus.getFetchTime(); - Status parsedConsensus = this.parseConsensusOrVote(consensusString, - fetchTime, true); - if (parsedConsensus != null) { - for (Status parsedVote : parsedVotes) { - if (parsedConsensus.getValidAfterMillis() == - parsedVote.getValidAfterMillis()) { - parsedConsensus.addVote(parsedVote); - } - } - parsedConsensuses.put(nickname, parsedConsensus); - } - } - return parsedConsensuses; - } - - /* Date-time formats to parse and format timestamps. */ - private static SimpleDateFormat dateTimeFormat; - static { - dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - } - - /* Parse a consensus or vote string into a Status instance. */ - private Status parseConsensusOrVote(String statusString, long fetchTime, - boolean isConsensus) { - if (statusString == null) { - return null; - } - Status status = new Status(); - status.setUnparsedString(statusString); - status.setFetchTime(fetchTime); - try { - BufferedReader br = new BufferedReader(new StringReader( - statusString)); - String line, rLine = null, sLine = null; - int totalRelays = 0, runningRelays = 0, bandwidthWeights = 0; - while ((line = br.readLine()) != null) { - if (line.startsWith("consensus-method ") || - line.startsWith("consensus-methods ")) { - SortedSet<Integer> consensusMethods = new TreeSet<Integer>(); - String[] parts = line.split(" "); - for (int i = 1; i < parts.length; i++) { - consensusMethods.add(Integer.parseInt(parts[i])); - } - status.setConsensusMethods(consensusMethods); - } else if (line.startsWith("valid-after ")) { - try { - status.setValidAfterMillis(dateTimeFormat.parse( - line.substring("valid-after ".length())).getTime()); - } catch (ParseException e) { - System.err.println("Could not parse valid-after timestamp in " - + "line '" + line + "' of a " - + (isConsensus ? "consensus" : "vote") + ". Skipping."); - return null; - } - } else if (line.startsWith("client-versions ")) { - status.setRecommendedClientVersions( - new TreeSet<String>(Arrays.asList( - line.split(" ")[1].split(",")))); - } else if (line.startsWith("server-versions ")) { - status.setRecommendedServerVersions( - new TreeSet<String>(Arrays.asList( - line.split(" ")[1].split(",")))); - } else if (line.startsWith("known-flags ")) { - for (String flag : line.substring("known-flags ".length()). - split(" ")) { - status.addKnownFlag(flag); - } - } else if (line.startsWith("params ")) { - if (line.length() > "params ".length()) { - for (String param : - line.substring("params ".length()).split(" ")) { - String paramName = param.split("=")[0]; - String paramValue = param.split("=")[1]; - status.addConsensusParam(paramName, paramValue); - } - } - } else if (line.startsWith("dir-source ")) { - String nickname = line.split(" ")[1]; - String fingerprint = line.split(" ")[2]; - if (isConsensus) { - status.addContainedVote(fingerprint); - } else { - status.setNickname(line.split(" ")[1]); - status.setFingerprint(line.split(" ")[2]); - } - } else if (line.startsWith("dir-key-expires ")) { - try { - status.setDirKeyExpiresMillis(dateTimeFormat.parse( - line.substring("dir-key-expires ".length())).getTime()); - } catch (ParseException e) { - System.err.println("Could not parse dir-key-expires " - + "timestamp in line '" + line + "' of a " - + (isConsensus ? "consensus" : "vote") + ". Skipping."); - return null; - } - } else if (line.startsWith("r ") || - line.equals("directory-footer")) { - if (rLine != null) { - StatusEntry statusEntry = new StatusEntry(); - statusEntry.setNickname(rLine.split(" ")[1]); - statusEntry.setFingerprint(Hex.encodeHexString( - Base64.decodeBase64(rLine.split(" ")[2] + "=")). - toUpperCase()); - SortedSet<String> flags = new TreeSet<String>(); - if (sLine.length() > 2) { - for (String flag : sLine.substring(2).split(" ")) { - flags.add(flag); - } - } - statusEntry.setFlags(flags); - status.addStatusEntry(statusEntry); - } - if (line.startsWith("r ")) { - rLine = line; - } - } else if (line.startsWith("s ") || line.equals("s")) { - sLine = line; - if (line.contains(" Running")) { - runningRelays++; - } - } else if (line.startsWith("v ") && - sLine.contains(" Authority")) { - String nickname = rLine.split(" ")[1]; - String versionString = line.substring(2); - status.addAuthorityVersion(nickname, versionString); - } else if (line.startsWith("w ") && !isConsensus && - line.contains(" Measured")) { - bandwidthWeights++; - } else if (line.startsWith("directory-signature ") && - isConsensus) { - String fingerprint = line.split(" ")[1]; - status.addContainedSignature(fingerprint); - } - } - br.close(); - status.setRunningRelays(runningRelays); - status.setBandwidthWeights(bandwidthWeights); - } catch (IOException e) { - System.err.println("Caught an IOException while parsing a " - + (isConsensus ? "consensus" : "vote") + " string. Skipping."); - return null; - } - return status; - } -} - diff --git a/src/org/torproject/doctor/Report.java b/src/org/torproject/doctor/Report.java deleted file mode 100755 index 2e508bc..0000000 --- a/src/org/torproject/doctor/Report.java +++ /dev/null @@ -1,27 +0,0 @@ -/* Copyright 2011 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.doctor; - -import java.util.*; - -/* Transform findings from parsing consensuses and votes into a report of - * some form. */ -public interface Report { - - /* Process the downloaded current consensus and corresponding votes to - * find irregularities between them. */ - public abstract void processDownloadedConsensuses( - SortedMap<String, Status> downloadedConsensuses); - - /* Process warnings consisting of warning type and details. */ - public abstract void processWarnings( - SortedMap<Warning, String> warnings); - - /* Include download statistics. */ - public abstract void includeFetchStatistics( - DownloadStatistics statistics); - - /* Finish writing report. */ - public abstract void writeReport(); -} - diff --git a/src/org/torproject/doctor/Status.java b/src/org/torproject/doctor/Status.java deleted file mode 100755 index df41e47..0000000 --- a/src/org/torproject/doctor/Status.java +++ /dev/null @@ -1,205 +0,0 @@ -/* Copyright 2011 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.doctor; - -import java.util.*; - -/* Contains the unparsed string and parsed fields from a network status - * consensus or vote. */ -public class Status implements Comparable<Status> { - - /* Helper methods to implement the Comparable interface; Status - * instances are compared by nickname of the publishing directory - * authorities. */ - public int compareTo(Status o) { - return this.nickname.compareTo(o.nickname); - } - public boolean equals(Object o) { - return (o instanceof Status && - this.nickname.equals(((Status) o).nickname)); - } - - /* Unparsed string that was downloaded or read from disk and that can - * be written to disk. */ - private String unparsedString; - public void setUnparsedString(String unparsedString) { - this.unparsedString = unparsedString; - } - public String getUnparsedString() { - return this.unparsedString; - } - - /* Fetch time in millis. */ - private long fetchTime; - public void setFetchTime(long fetchTime) { - this.fetchTime = fetchTime; - } - public long getFetchTime() { - return this.fetchTime; - } - - /* Votes published at the same time as this consensus; votes don't - * reference any statuses. */ - private SortedSet<Status> votes = new TreeSet<Status>(); - public void addVote(Status vote) { - this.votes.add(vote); - } - public SortedSet<Status> getVotes() { - return this.votes; - } - - /* Fingerprint of the directory authority publishing this vote; left - * empty for consensuses. */ - private String fingerprint; - public void setFingerprint(String fingerprint) { - this.fingerprint = fingerprint; - } - public String getFingerprint() { - return this.fingerprint; - } - - /* Nickname of the directory authority publishing this vote; left empty - * for consensuses. */ - private String nickname; - public void setNickname(String nickname) { - this.nickname= nickname; - } - public String getNickname() { - return this.nickname; - } - - /* Valid-after time in milliseconds. */ - private long validAfterMillis; - public void setValidAfterMillis(long validAfterMillis) { - this.validAfterMillis = validAfterMillis; - } - public long getValidAfterMillis() { - return this.validAfterMillis; - } - - /* Consensus parameters. */ - private SortedMap<String, String> consensusParams = - new TreeMap<String, String>(); - public void addConsensusParam(String paramName, String paramValue) { - this.consensusParams.put(paramName, paramValue); - } - public SortedMap<String, String> getConsensusParams() { - return this.consensusParams; - } - - /* Consensus methods supported by the directory authority sending a vote - * or of the produced consensus. */ - private SortedSet<Integer> consensusMethods; - public void setConsensusMethods(SortedSet<Integer> consensusMethods) { - this.consensusMethods = consensusMethods; - } - public SortedSet<Integer> getConsensusMethods() { - return this.consensusMethods; - } - - /* Recommended server versions. */ - private SortedSet<String> recommendedServerVersions; - public void setRecommendedServerVersions( - SortedSet<String> recommendedServerVersions) { - this.recommendedServerVersions = recommendedServerVersions; - } - public SortedSet<String> getRecommendedServerVersions() { - return this.recommendedServerVersions; - } - - /* Recommended client versions. */ - private SortedSet<String> recommendedClientVersions; - public void setRecommendedClientVersions( - SortedSet<String> recommendedClientVersions) { - this.recommendedClientVersions = recommendedClientVersions; - } - public SortedSet<String> getRecommendedClientVersions() { - return this.recommendedClientVersions; - } - - /* Expiration times of directory signing keys. */ - private long dirKeyExpiresMillis; - public void setDirKeyExpiresMillis(long dirKeyExpiresMillis) { - this.dirKeyExpiresMillis = dirKeyExpiresMillis; - } - public long getDirKeyExpiresMillis() { - return this.dirKeyExpiresMillis; - } - - /* Known flags by the directory authority sending a vote or of the - * produced consensus. */ - private SortedSet<String> knownFlags = new TreeSet<String>(); - public void addKnownFlag(String knownFlag) { - this.knownFlags.add(knownFlag); - } - public SortedSet<String> getKnownFlags() { - return this.knownFlags; - } - - /* Fingerprints of directory authorities of contained votes (only - * relevant for consensuses). */ - private SortedSet<String> containedVotes = new TreeSet<String>(); - public void addContainedVote(String fingerprint) { - this.containedVotes.add(fingerprint); - } - public SortedSet<String> getContainedVotes() { - return this.containedVotes; - } - - /* Fingerprints of directory authorities of contained signatures (only - * relevant for consensuses). */ - private SortedSet<String> containedSignatures = new TreeSet<String>(); - public void addContainedSignature(String fingerprint) { - this.containedSignatures.add(fingerprint); - } - public SortedSet<String> getContainedSignatures() { - return this.containedSignatures; - } - - /* Number of status entries with the Running flag. */ - private int runningRelays; - public void setRunningRelays(int runningRelays) { - this.runningRelays = runningRelays; - } - public int getRunningRelays() { - return this.runningRelays; - } - - /* Number of status entries containing bandwidth weights (only relevant - * in votes). */ - private int bandwidthWeights; - public void setBandwidthWeights(int bandwidthWeights) { - this.bandwidthWeights = bandwidthWeights; - } - public int getBandwidthWeights() { - return this.bandwidthWeights; - } - - /* Status entries contained in this status. */ - private SortedMap<String, StatusEntry> statusEntries = - new TreeMap<String, StatusEntry>(); - public void addStatusEntry(StatusEntry statusEntry) { - this.statusEntries.put(statusEntry.getFingerprint(), statusEntry); - } - public SortedMap<String, StatusEntry> getStatusEntries() { - return this.statusEntries; - } - public boolean containsStatusEntry(String fingerprint) { - return this.statusEntries.containsKey(fingerprint); - } - public StatusEntry getStatusEntry(String fingerprint) { - return this.statusEntries.get(fingerprint); - } - - /* Versions of directory authorities (only set in a consensus). */ - private SortedMap<String, String> authorityVersions = - new TreeMap<String, String>(); - public void addAuthorityVersion(String fingerprint, - String versionString) { - this.authorityVersions.put(fingerprint, versionString); - } - public SortedMap<String, String> getAuthorityVersions() { - return this.authorityVersions; - } -} - diff --git a/src/org/torproject/doctor/StatusEntry.java b/src/org/torproject/doctor/StatusEntry.java deleted file mode 100755 index 18f1149..0000000 --- a/src/org/torproject/doctor/StatusEntry.java +++ /dev/null @@ -1,48 +0,0 @@ -/* Copyright 2011 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.doctor; - -import java.util.*; - -/* Contains the parsed data from a network status entry contained in a - * network status consensus or vote. */ -public class StatusEntry implements Comparable<StatusEntry> { - - /* Helper methods to implement the Comparable interface; StatusEntry - * instances are compared by fingerprint. */ - public int compareTo(StatusEntry o) { - return this.fingerprint.compareTo(o.fingerprint); - } - public boolean equals(Object o) { - return (o instanceof StatusEntry && - this.fingerprint.equals(((StatusEntry) o).fingerprint)); - } - - /* Relay fingerprint. */ - private String fingerprint; - public void setFingerprint(String fingerprint) { - this.fingerprint = fingerprint; - } - public String getFingerprint() { - return this.fingerprint; - } - - /* Relay nickname. */ - private String nickname; - public void setNickname(String nickname) { - this.nickname = nickname; - } - public String getNickname() { - return this.nickname; - } - - /* Relay flags. */ - private SortedSet<String> flags; - public void setFlags(SortedSet<String> flags) { - this.flags = flags; - } - public SortedSet<String> getFlags() { - return this.flags; - } -} - diff --git a/src/org/torproject/doctor/StatusFileReport.java b/src/org/torproject/doctor/StatusFileReport.java index 884dc18..7890005 100755 --- a/src/org/torproject/doctor/StatusFileReport.java +++ b/src/org/torproject/doctor/StatusFileReport.java @@ -8,7 +8,7 @@ import java.util.*; /* Check a given consensus and votes for irregularities and write results * to stdout while rate-limiting warnings based on severity. */ -public class StatusFileReport implements Report { +public class StatusFileReport { /* Date-time format to format timestamps. */ private static SimpleDateFormat dateTimeFormat; @@ -17,27 +17,12 @@ public class StatusFileReport implements Report { dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); } - /* Downloaded consensus and corresponding votes for later - * processing. */ - private SortedMap<String, Status> downloadedConsensuses; - private Status downloadedConsensus; - private SortedSet<Status> downloadedVotes; - public void processDownloadedConsensuses( - SortedMap<String, Status> downloadedConsensuses) { - this.downloadedConsensuses = downloadedConsensuses; - } - /* Warnings obtained from checking the current consensus and votes. */ private SortedMap<Warning, String> warnings; public void processWarnings(SortedMap<Warning, String> warnings) { this.warnings = warnings; } - /* Ignore download statistics for this report. */ - public void includeFetchStatistics(DownloadStatistics statistics) { - /* Do nothing. */ - } - /* Check consensuses and votes for irregularities and write output to * stdout. */ public void writeReport() {

1 0

[metrics-lib/master] Add a work-in-progress note to the README.
by karsten＠torproject.org 13 Dec '11

13 Dec '11

commit 34f4820ac3504c00ab5c09e85376a49fac47031c Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Dec 12 19:15:24 2011 +0100 Add a work-in-progress note to the README. --- README | 10 ++++++++++ 1 files changed, 10 insertions(+), 0 deletions(-) diff --git a/README b/README index 9f235d3..f192501 100755 --- a/README +++ b/README @@ -24,3 +24,13 @@ running, batch-processing applications and not for continuously running applications that rely on learning about changes to an underlying descriptor source. +Work-In-Progress Notice +======================= + +As of December 2011, this library is a work in progress. Its interfaces +may change at any time. If you want to be informed of interface changes +during the initial development phase, ask for your software to be added to +the list below (software name, repository, contact person): + + - DocTor, https://git.torproject.org/doctor.git, karsten +

1 0

[metrics-lib/master] Implement parsing of network status votes.
by karsten＠torproject.org 13 Dec '11

13 Dec '11

commit 276e0ef72aae65f0e69cb55bb4b1d80c537691e9 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Dec 13 10:17:34 2011 +0100 Implement parsing of network status votes. --- .../descriptor/RelayNetworkStatusConsensus.java | 7 + .../descriptor/RelayNetworkStatusVote.java | 80 +++++- .../descriptor/impl/NetworkStatusEntryImpl.java | 7 +- .../impl/RelayNetworkStatusConsensusImpl.java | 6 + .../impl/RelayNetworkStatusVoteImpl.java | 315 +++++++++++++++++++- 5 files changed, 408 insertions(+), 7 deletions(-) diff --git a/src/org/torproject/descriptor/RelayNetworkStatusConsensus.java b/src/org/torproject/descriptor/RelayNetworkStatusConsensus.java index 192c963..452a0ba 100755 --- a/src/org/torproject/descriptor/RelayNetworkStatusConsensus.java +++ b/src/org/torproject/descriptor/RelayNetworkStatusConsensus.java @@ -46,6 +46,13 @@ public interface RelayNetworkStatusConsensus extends Descriptor { /* Return status entries, one for each contained relay. */ public SortedMap<String, NetworkStatusEntry> getStatusEntries(); + /* Return whether a status entry with the given fingerprint exists. */ + public boolean containsStatusEntry(String fingerprint); + + /* Return a status entry by fingerprint or null if no such status entry + * exists. */ + public NetworkStatusEntry getStatusEntry(String fingerprint); + /* Return directory signatures. */ public SortedMap<String, String> getDirectorySignatures(); diff --git a/src/org/torproject/descriptor/RelayNetworkStatusVote.java b/src/org/torproject/descriptor/RelayNetworkStatusVote.java index 62d1307..7d85a0c 100755 --- a/src/org/torproject/descriptor/RelayNetworkStatusVote.java +++ b/src/org/torproject/descriptor/RelayNetworkStatusVote.java @@ -2,11 +2,87 @@ * See LICENSE for licensing information */ package org.torproject.descriptor; +import java.util.List; import java.util.SortedMap; import java.util.SortedSet; -/* Contains the unparsed string and parsed fields from a network status - * vote. */ +/* Contains a network status vote. */ public interface RelayNetworkStatusVote extends Descriptor { + + /* Return the network status version. */ + public int getNetworkStatusVersion(); + + /* Return the consensus method. */ + public List<Integer> getConsensusMethods(); + + /* Return the publication time in milliseconds. */ + public long getPublishedMillis(); + + /* Return the valid-after time in milliseconds. */ + public long getValidAfterMillis(); + + /* Return the fresh-until time in milliseconds. */ + public long getFreshUntilMillis(); + + /* Return the valid-until time in milliseconds. */ + public long getValidUntilMillis(); + + /* Return a list of the voting-delay times in seconds. */ + public List<Long> getVotingDelay(); + + /* Return cecommended server versions. */ + public SortedSet<String> getRecommendedServerVersions(); + + /* Return recommended client versions. */ + public SortedSet<String> getRecommendedClientVersions(); + + /* Return known relay flags. */ + public SortedSet<String> getKnownFlags(); + + /* Return consensus parameters. */ + public SortedMap<String, String> getConsensusParams(); + + /* Return the directory nickname. */ + public String getNickname(); + + /* Return the directory identity. */ + public String getIdentity(); + + /* Return the IP address. */ + public String getAddress(); + + /* Return the DiRPort. */ + public int getDirport(); + + /* Return the ORPort. */ + public int getOrport(); + + /* Return the contact line. */ + public String getContactLine(); + + /* Return the directory key certificate version. */ + public int getDirKeyCertificateVersion(); + + /* Return the directory key publication timestamp. */ + public long getDirKeyPublishedMillis(); + + /* Return the directory key expiration timestamp. */ + public long getDirKeyExpiresMillis(); + + /* Return the signing key digest. */ + public String getSigningKeyDigest(); + + /* Return status entries, one for each contained relay. */ + public SortedMap<String, NetworkStatusEntry> getStatusEntries(); + + /* Return whether a status entry with the given fingerprint exists. */ + public boolean containsStatusEntry(String fingerprint); + + /* Return a status entry by fingerprint or null if no such status entry + * exists. */ + public NetworkStatusEntry getStatusEntry(String fingerprint); + + /* Return directory signatures. */ + public SortedMap<String, String> getDirectorySignatures(); } diff --git a/src/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java b/src/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java index d0a511d..610fb1a 100755 --- a/src/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java +++ b/src/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java @@ -57,12 +57,15 @@ public class NetworkStatusEntryImpl implements NetworkStatusEntry { } else if (line.startsWith("s ")) { this.flags.addAll(Arrays.asList(line.substring("s ".length()). split(" "))); - } else if (line.startsWith("v ")) { - this.version = line.substring("v ".length()); + } else if (line.startsWith("v ") || line.startsWith("opt v")) { + this.version = line.substring( + line.startsWith("v ") ? "v ".length() : "opt v".length()); } else if (line.startsWith("w ")) { this.bandwidth = line.substring("w ".length()); } else if (line.startsWith("p ")) { this.ports = line.substring(2); + } else if (line.startsWith("m ")) { + /* TODO Parse m lines in votes. */ } else { throw new RuntimeException("Unknown line '" + line + "' in " + "status entry."); diff --git a/src/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java b/src/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java index a9579a6..3d2b0a2 100755 --- a/src/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java +++ b/src/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java @@ -291,6 +291,12 @@ public class RelayNetworkStatusConsensusImpl public SortedMap<String, NetworkStatusEntry> getStatusEntries() { return new TreeMap<String, NetworkStatusEntry>(this.statusEntries); } + public boolean containsStatusEntry(String fingerprint) { + return this.statusEntries.containsKey(fingerprint); + } + public NetworkStatusEntry getStatusEntry(String fingerprint) { + return this.statusEntries.get(fingerprint); + } private SortedMap<String, String> directorySignatures = new TreeMap<String, String>(); diff --git a/src/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java b/src/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java index 84a81e2..fa5b4d5 100755 --- a/src/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java +++ b/src/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java @@ -2,14 +2,26 @@ * See LICENSE for licensing information */ package org.torproject.descriptor.impl; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.StringReader; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TimeZone; +import java.util.TreeMap; +import java.util.TreeSet; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.NetworkStatusEntry; import org.torproject.descriptor.RelayNetworkStatusVote; /* Contains a network status vote. */ -/* TODO This class doesn't contain any parsing code yet, and it would be - * sharing a lot of that code with the consensus class. Should there be - * an abstract super class for the two? */ +/* TODO This class is sharing a lot of parsing code with the consensus + * class. Should there be an abstract super class for the two? */ public class RelayNetworkStatusVoteImpl implements RelayNetworkStatusVote { @@ -40,11 +52,308 @@ public class RelayNetworkStatusVoteImpl protected RelayNetworkStatusVoteImpl(byte[] voteBytes) { this.voteBytes = voteBytes; + this.parseVoteBytes(); + this.checkConsistency(); + /* TODO Find a way to handle parse and consistency-check problems. */ + } + + private void parseVoteBytes() { + String line = null; + try { + BufferedReader br = new BufferedReader(new StringReader( + new String(this.voteBytes))); + SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + StringBuilder dirSourceEntryLines = null, statusEntryLines = null; + boolean skipCrypto = false; + while ((line = br.readLine()) != null) { + if (line.startsWith("network-status-version ")) { + this.networkStatusVersion = Integer.parseInt(line.substring( + "network-status-version ".length())); + } else if (line.startsWith("vote-status ")) { + if (!line.equals("vote-status vote")) { + throw new RuntimeException("Line '" + line + "' indicates " + + "that this string is not a vote. Aborting parsing."); + } + } else if (line.startsWith("consensus-methods ")) { + for (String consensusMethodString : line.substring( + "consensus-methods ".length()).split(" ")) { + this.consensusMethods.add(Integer.parseInt( + consensusMethodString)); + } + } else if (line.startsWith("published ")) { + this.publishedMillis = dateTimeFormat.parse( + line.substring("published ".length())).getTime(); + } else if (line.startsWith("valid-after ")) { + this.validAfterMillis = dateTimeFormat.parse( + line.substring("valid-after ".length())).getTime(); + } else if (line.startsWith("fresh-until ")) { + this.freshUntilMillis = dateTimeFormat.parse( + line.substring("fresh-until ".length())).getTime(); + } else if (line.startsWith("valid-until ")) { + this.validUntilMillis = dateTimeFormat.parse( + line.substring("valid-until ".length())).getTime(); + } else if (line.startsWith("voting-delay ")) { + for (String votingDelayString : line.substring( + "voting-delay ".length()).split(" ")) { + this.votingDelay.add(Long.parseLong(votingDelayString)); + } + } else if (line.startsWith("client-versions ")) { + this.recommendedClientVersions.addAll( + Arrays.asList(line.split(" ")[1].split(","))); + } else if (line.startsWith("server-versions ")) { + this.recommendedServerVersions.addAll( + Arrays.asList(line.split(" ")[1].split(","))); + } else if (line.startsWith("known-flags ")) { + for (String flag : line.substring("known-flags ".length()). + split(" ")) { + this.knownFlags.add(flag); + } + } else if (line.startsWith("params ")) { + if (line.length() > "params ".length()) { + for (String param : + line.substring("params ".length()).split(" ")) { + String paramName = param.split("=")[0]; + String paramValue = param.split("=")[1]; + this.consensusParams.put(paramName, paramValue); + } + } + } else if (line.startsWith("dir-source ")) { + String[] parts = line.split(" "); + this.nickname = parts[1]; + this.identity = parts[2]; + this.address = parts[4]; + this.dirPort = Integer.parseInt(parts[5]); + this.orPort = Integer.parseInt(parts[6]); + /* TODO Add code for parsing legacy dir sources. */ + } else if (line.startsWith("contact ")) { + this.contactLine = line.substring("contact ".length()); + } else if (line.startsWith("dir-key-certificate-version ")) { + this.dirKeyCertificateVersion = Integer.parseInt(line.substring( + "dir-key-certificate-version ".length())); + } else if (line.startsWith("fingerprint ")) { + /* Nothing new to learn here. We already know the fingerprint + * from the dir-source line. */ + } else if (line.startsWith("dir-key-published ")) { + this.dirKeyPublishedMillis = dateTimeFormat.parse( + line.substring("dir-key-published ".length())).getTime(); + } else if (line.startsWith("dir-key-expires ")) { + this.dirKeyExpiresMillis = dateTimeFormat.parse( + line.substring("dir-key-expires ".length())).getTime(); + } else if (line.equals("dir-identity-key") || + line.equals("dir-signing-key") || + line.equals("dir-key-crosscert") || + line.equals("dir-key-certification")) { + /* Ignore crypto parts for now. */ + } else if (line.startsWith("r ") || + line.equals("directory-footer")) { + if (statusEntryLines != null) { + NetworkStatusEntryImpl statusEntry = + new NetworkStatusEntryImpl( + statusEntryLines.toString().getBytes()); + this.statusEntries.put(statusEntry.getFingerprint(), + statusEntry); + statusEntryLines = null; + } + if (line.startsWith("r ")) { + statusEntryLines = new StringBuilder(); + statusEntryLines.append(line + "\n"); + } + } else if (line.startsWith("s ") || line.equals("s") || + line.startsWith("opt v ") || line.startsWith("w ") || + line.startsWith("p ") || line.startsWith("m ")) { + statusEntryLines.append(line + "\n"); + } else if (line.startsWith("directory-signature ")) { + String[] parts = line.split(" "); + String identity = parts[1]; + String signingKeyDigest = parts[2]; + this.directorySignatures.put(identity, signingKeyDigest); + } else if (line.startsWith("-----BEGIN")) { + skipCrypto = true; + } else if (line.startsWith("-----END")) { + skipCrypto = false; + } else if (!skipCrypto) { + throw new RuntimeException("Unrecognized line '" + line + "'."); + } + } + } catch (IOException e) { + throw new RuntimeException("Internal error: Ran into an " + + "IOException while parsing a String in memory. Something's " + + "really wrong.", e); + } catch (ParseException e) { + /* TODO Handle me correctly. */ + throw new RuntimeException("Parse error in line '" + line + "'."); + } catch (NumberFormatException e) { + /* TODO Handle me. In theory, we shouldn't catch runtime + * exceptions, but in this case it keeps the parsing code small. */ + } catch (ArrayIndexOutOfBoundsException e) { + /* TODO Handle me. In theory, we shouldn't catch runtime + * exceptions, but in this case it keeps the parsing code small. */ + } } private byte[] voteBytes; public byte[] getRawDescriptorBytes() { return this.voteBytes; } + + private int networkStatusVersion; + public int getNetworkStatusVersion() { + return this.networkStatusVersion; + } + + private List<Integer> consensusMethods = new ArrayList<Integer>(); + public List<Integer> getConsensusMethods() { + return this.consensusMethods; + } + + private long publishedMillis; + public long getPublishedMillis() { + return this.publishedMillis; + } + + private long validAfterMillis; + public long getValidAfterMillis() { + return this.validAfterMillis; + } + + private long freshUntilMillis; + public long getFreshUntilMillis() { + return this.freshUntilMillis; + } + + private long validUntilMillis; + public long getValidUntilMillis() { + return this.validUntilMillis; + } + + private List<Long> votingDelay = new ArrayList<Long>(); + public List<Long> getVotingDelay() { + return new ArrayList<Long>(this.votingDelay); + } + + private SortedSet<String> recommendedClientVersions = + new TreeSet<String>(); + public SortedSet<String> getRecommendedClientVersions() { + return new TreeSet<String>(this.recommendedClientVersions); + } + + private SortedSet<String> recommendedServerVersions = + new TreeSet<String>(); + public SortedSet<String> getRecommendedServerVersions() { + return new TreeSet<String>(this.recommendedServerVersions); + } + + private SortedSet<String> knownFlags = new TreeSet<String>(); + public SortedSet<String> getKnownFlags() { + return new TreeSet<String>(this.knownFlags); + } + + private SortedMap<String, String> consensusParams = + new TreeMap<String, String>(); + public SortedMap<String, String> getConsensusParams() { + return new TreeMap<String, String>(this.consensusParams); + } + + private String nickname; + public String getNickname() { + return this.nickname; + } + + private String identity; + public String getIdentity() { + return this.identity; + } + + private String address; + public String getAddress() { + return this.address; + } + + private int dirPort; + public int getDirport() { + return this.dirPort; + } + + private int orPort; + public int getOrport() { + return this.orPort; + } + + private String contactLine; + public String getContactLine() { + return this.contactLine; + } + + private int dirKeyCertificateVersion; + public int getDirKeyCertificateVersion() { + return this.dirKeyCertificateVersion; + } + + private long dirKeyPublishedMillis; + public long getDirKeyPublishedMillis() { + return this.dirKeyPublishedMillis; + } + + private long dirKeyExpiresMillis; + public long getDirKeyExpiresMillis() { + return this.dirKeyExpiresMillis; + } + + private String signingKeyDigest; + public String getSigningKeyDigest() { + return this.signingKeyDigest; + } + + private SortedMap<String, NetworkStatusEntry> statusEntries = + new TreeMap<String, NetworkStatusEntry>(); + public SortedMap<String, NetworkStatusEntry> getStatusEntries() { + return new TreeMap<String, NetworkStatusEntry>(this.statusEntries); + } + public boolean containsStatusEntry(String fingerprint) { + return this.statusEntries.containsKey(fingerprint); + } + public NetworkStatusEntry getStatusEntry(String fingerprint) { + return this.statusEntries.get(fingerprint); + } + + + private SortedMap<String, String> directorySignatures = + new TreeMap<String, String>(); + public SortedMap<String, String> getDirectorySignatures() { + return new TreeMap<String, String>(this.directorySignatures); + } + + private void checkConsistency() { + if (this.networkStatusVersion == 0) { + throw new RuntimeException("Consensus doesn't contain a " + + "'network-status-version' line."); + } + if (this.validAfterMillis == 0L) { + throw new RuntimeException("Consensus doesn't contain a " + + "'valid-after' line."); + } + if (this.freshUntilMillis == 0L) { + throw new RuntimeException("Consensus doesn't contain a " + + "'fresh-until' line."); + } + if (this.validUntilMillis == 0L) { + throw new RuntimeException("Consensus doesn't contain a " + + "'valid-until' line."); + } + if (this.votingDelay.isEmpty()) { + throw new RuntimeException("Consensus doesn't contain a " + + "'voting-delay' line."); + } + if (this.knownFlags.isEmpty()) { + throw new RuntimeException("Consensus doesn't contain a " + + "'known-flags' line."); + } + if (this.statusEntries.isEmpty()) { + throw new RuntimeException("Consensus doesn't contain any 'r' " + + "lines."); + } + } }

1 0

r25250: {} add roger's latest presentations. (projects/presentations)
by Andrew Lewman 12 Dec '11

12 Dec '11

Author: phobos Date: 2011-12-12 21:09:13 +0000 (Mon, 12 Dec 2011) New Revision: 25250 Added: projects/presentations/slides-pi11.odp projects/presentations/slides-pi11.pdf Log: add roger's latest presentations. Added: projects/presentations/slides-pi11.odp =================================================================== (Binary files differ) Property changes on: projects/presentations/slides-pi11.odp ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: projects/presentations/slides-pi11.pdf =================================================================== (Binary files differ) Property changes on: projects/presentations/slides-pi11.pdf ___________________________________________________________________ Added: svn:mime-type + application/octet-stream

1 0

[tor/master] Don't crash on startup of a dormant relay
by nickm＠torproject.org 12 Dec '11

12 Dec '11

commit d688a40a0e7c1e8417ecdc463821e50cd1762715 Author: Robert Ransom <rransom.8774(a)gmail.com> Date: Mon Dec 12 11:25:55 2011 -0800 Don't crash on startup of a dormant relay If a relay is dormant at startup, it will call init_keys before crypto_set_tls_dh_prime. This is bad. Let's make it not so bad, because someday it *will* happen again. --- changes/bug4702 | 6 ++++++ src/common/crypto.c | 7 +++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/changes/bug4702 b/changes/bug4702 new file mode 100644 index 0000000..dcd3bfc --- /dev/null +++ b/changes/bug4702 @@ -0,0 +1,6 @@ + o Minor bugfixes: + + - Fix an assertion failure when a relay with accounting enabled + starts up while dormant. Fixes bug 4702; bugfix on + 0.2.3.9-alpha. + diff --git a/src/common/crypto.c b/src/common/crypto.c index 2aa3fc9..673fc0c 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -2089,8 +2089,11 @@ init_dh_param(void) dh_param_p = circuit_dh_prime; dh_param_g = generator; - /* Should be already set by config.c. */ - tor_assert(dh_param_p_tls); + /* Ensure that we have TLS DH parameters set up, too, even if we're + going to change them soon. */ + if (!dh_param_p_tls) { + crypto_set_tls_dh_prime(NULL); + } } /** Number of bits to use when choosing the x or y value in a Diffie-Hellman

1 0