[tor-commits] [doctor/master] Rely on metrics-lib to parse consensuses and votes.

commit ad912387e433d29726eefcaca301967f4ce50427 Author: Karsten Loesing karsten.loesing@gmx.net Date: Tue Dec 13 10:18:07 2011 +0100

Rely on metrics-lib to parse consensuses and votes. --- src/org/torproject/doctor/Checker.java | 91 ++++++--- src/org/torproject/doctor/Download.java | 46 ----- src/org/torproject/doctor/DownloadStatistics.java | 25 ++- src/org/torproject/doctor/Downloader.java | 62 ++----- src/org/torproject/doctor/Main.java | 52 ++---- .../torproject/doctor/MetricsWebsiteReport.java | 139 ++++++++------ src/org/torproject/doctor/Parser.java | 180 ----------------- src/org/torproject/doctor/Report.java | 27 --- src/org/torproject/doctor/Status.java | 205 -------------------- src/org/torproject/doctor/StatusEntry.java | 48 ----- src/org/torproject/doctor/StatusFileReport.java | 17 +-- 11 files changed, 194 insertions(+), 698 deletions(-)

diff --git a/src/org/torproject/doctor/Checker.java b/src/org/torproject/doctor/Checker.java index 4d6493f..ee4e29e 100755 --- a/src/org/torproject/doctor/Checker.java +++ b/src/org/torproject/doctor/Checker.java @@ -5,6 +5,7 @@ package org.torproject.doctor; import java.io.*; import java.text.*; import java.util.*; +import org.torproject.descriptor.*;

/* Check a given consensus and votes for irregularities and write results * to a warnings map consisting of warning type and details. */ @@ -26,12 +27,15 @@ public class Checker { }

/* Downloaded consensus and corresponding votes for processing. */ - private SortedMap<String, Status> downloadedConsensuses; - private Status downloadedConsensus; - private SortedSet<Status> downloadedVotes; + private SortedMap<String, RelayNetworkStatusConsensus> + downloadedConsensuses = new TreeMap<String, + RelayNetworkStatusConsensus>(); + private RelayNetworkStatusConsensus downloadedConsensus; + private List<RelayNetworkStatusVote> downloadedVotes = + new ArrayList<RelayNetworkStatusVote>(); public void processDownloadedConsensuses( - SortedMap<String, Status> downloadedConsensuses) { - this.downloadedConsensuses = downloadedConsensuses; + List<DescriptorRequest> downloads) { + this.storeDownloads(downloads); this.findMostRecentConsensus(); this.checkMissingConsensuses(); this.checkAllConsensusesFresh(); @@ -51,10 +55,29 @@ public class Checker { } }

+ /* Store consensuses and votes in a way that we can process them more + * easily. */ + private void storeDownloads(List<DescriptorRequest> downloads) { + for (DescriptorRequest request : downloads) { + for (Descriptor descriptor : request.getDescriptors()) { + if (descriptor instanceof RelayNetworkStatusConsensus) { + this.downloadedConsensuses.put(request.getDirectoryNickname(), + (RelayNetworkStatusConsensus) descriptor); + } else if (descriptor instanceof RelayNetworkStatusVote) { + this.downloadedVotes.add((RelayNetworkStatusVote) descriptor); + } else { + System.err.println("Did not expect a descriptor of type " + + descriptor.getClass() + ". Ignoring."); + } + } + } + } + /* Find most recent consensus and corresponding votes. */ private void findMostRecentConsensus() { long mostRecentValidAfterMillis = -1L; - for (Status downloadedConsensus : downloadedConsensuses.values()) { + for (RelayNetworkStatusConsensus downloadedConsensus : + downloadedConsensuses.values()) { if (downloadedConsensus.getValidAfterMillis() > mostRecentValidAfterMillis) { this.downloadedConsensus = downloadedConsensus; @@ -62,9 +85,6 @@ public class Checker { downloadedConsensus.getValidAfterMillis(); } } - if (this.downloadedConsensus != null) { - this.downloadedVotes = this.downloadedConsensus.getVotes(); - } }

/* Check if any directory authority didn't tell us a consensus. */ @@ -87,9 +107,10 @@ public class Checker { private void checkAllConsensusesFresh() { long fresh = System.currentTimeMillis() - 60L * 60L * 1000L; SortedSet<String> nonFresh = new TreeSet<String>(); - for (Map.Entry<String, Status> e : downloadedConsensuses.entrySet()) { + for (Map.Entry<String, RelayNetworkStatusConsensus> e : + downloadedConsensuses.entrySet()) { String nickname = e.getKey(); - Status downloadedConsensus = e.getValue(); + RelayNetworkStatusConsensus downloadedConsensus = e.getValue(); if (downloadedConsensus.getValidAfterMillis() < fresh) { nonFresh.add(nickname); } @@ -107,14 +128,16 @@ public class Checker { /* Check if all downloaded consensuses contain the same set of votes. */ private void checkContainedVotes() { Set<String> allVotes = new HashSet<String>(); - for (Status consensus : downloadedConsensuses.values()) { - allVotes.addAll(consensus.getContainedVotes()); + for (RelayNetworkStatusConsensus consensus : + downloadedConsensuses.values()) { + allVotes.addAll(consensus.getDirSourceEntries().keySet()); } SortedSet<String> missingVotes = new TreeSet<String>(); - for (Map.Entry<String, Status> e : downloadedConsensuses.entrySet()) { + for (Map.Entry<String, RelayNetworkStatusConsensus> e : + downloadedConsensuses.entrySet()) { String nickname = e.getKey(); - Status downloadedConsensus = e.getValue(); - if (!downloadedConsensus.getContainedVotes().containsAll( + RelayNetworkStatusConsensus downloadedConsensus = e.getValue(); + if (!downloadedConsensus.getDirSourceEntries().keySet().containsAll( allVotes)) { missingVotes.add(nickname); } @@ -133,11 +156,13 @@ public class Checker { * authorities. */ private void checkConsensusSignatures() { SortedSet<String> missingSignatures = new TreeSet<String>(); - for (Map.Entry<String, Status> e : downloadedConsensuses.entrySet()) { + for (Map.Entry<String, RelayNetworkStatusConsensus> e : + downloadedConsensuses.entrySet()) { String nickname = e.getKey(); - Status downloadedConsensus = e.getValue(); - if (!downloadedConsensus.getContainedSignatures().containsAll( - downloadedConsensus.getContainedVotes())) { + RelayNetworkStatusConsensus downloadedConsensus = e.getValue(); + if (!downloadedConsensus.getDirectorySignatures().keySet(). + containsAll(downloadedConsensus.getDirSourceEntries(). + keySet())) { missingSignatures.add(nickname); } } @@ -152,7 +177,8 @@ public class Checker { }

/* Check if the most recent consensus is older than 1 hour. */ - private boolean isConsensusFresh(Status consensus) { + private boolean isConsensusFresh( + RelayNetworkStatusConsensus consensus) { return (consensus.getValidAfterMillis() >= System.currentTimeMillis() - 60L * 60L * 1000L); } @@ -160,9 +186,9 @@ public class Checker { /* Check supported consensus methods of all votes. */ private void checkConsensusMethods() { SortedSet<String> dirs = new TreeSet<String>(); - for (Status vote : this.downloadedVotes) { + for (RelayNetworkStatusVote vote : this.downloadedVotes) { if (!vote.getConsensusMethods().contains( - this.downloadedConsensus.getConsensusMethods().last())) { + this.downloadedConsensus.getConsensusMethod())) { dirs.add(vote.getNickname()); } } @@ -181,7 +207,7 @@ public class Checker { private void checkRecommendedVersions() { SortedSet<String> unrecommendedClientVersions = new TreeSet<String>(), unrecommendedServerVersions = new TreeSet<String>(); - for (Status vote : this.downloadedVotes) { + for (RelayNetworkStatusVote vote : this.downloadedVotes) { if (vote.getRecommendedClientVersions() != null && !downloadedConsensus.getRecommendedClientVersions().equals( vote.getRecommendedClientVersions())) { @@ -231,7 +257,7 @@ public class Checker { + "cbtmintimeout,cbtinitialtimeout,perconnbwburst,perconnbwrate"). split(","))); SortedSet<String> conflicts = new TreeSet<String>(); - for (Status vote : this.downloadedVotes) { + for (RelayNetworkStatusVote vote : this.downloadedVotes) { Map<String, String> voteConsensusParams = vote.getConsensusParams(); boolean conflictOrInvalid = false; @@ -272,7 +298,7 @@ public class Checker { SortedMap<String, String> expiringCertificates = new TreeMap<String, String>(); long now = System.currentTimeMillis(); - for (Status vote : this.downloadedVotes) { + for (RelayNetworkStatusVote vote : this.downloadedVotes) { long voteDirKeyExpiresMillis = vote.getDirKeyExpiresMillis(); if (voteDirKeyExpiresMillis - 14L * 24L * 60L * 60L * 1000L < now) { expiringCertificates.put(vote.getNickname(), @@ -299,7 +325,7 @@ public class Checker { + "tor26,urras").split(","))); SortedSet<String> missingVotes = new TreeSet<String>(knownAuthorities); - for (Status vote : this.downloadedVotes) { + for (RelayNetworkStatusVote vote : this.downloadedVotes) { missingVotes.remove(vote.getNickname()); } if (!missingVotes.isEmpty()) { @@ -316,8 +342,15 @@ public class Checker { private void checkBandwidthScanners() { SortedSet<String> missingBandwidthScanners = new TreeSet<String>( Arrays.asList("ides,urras,moria1,gabelmoo,maatuska".split(","))); - for (Status vote : this.downloadedVotes) { - if (vote.getBandwidthWeights() > 0) { + for (RelayNetworkStatusVote vote : this.downloadedVotes) { + boolean containsMeasuredBandwidths = false; + for (NetworkStatusEntry entry : vote.getStatusEntries().values()) { + if (entry.getBandwidth().contains("Measured=")) { + containsMeasuredBandwidths = true; + break; + } + } + if (containsMeasuredBandwidths) { missingBandwidthScanners.remove(vote.getNickname()); } } diff --git a/src/org/torproject/doctor/Download.java b/src/org/torproject/doctor/Download.java deleted file mode 100644 index 5c84277..0000000 --- a/src/org/torproject/doctor/Download.java +++ /dev/null @@ -1,46 +0,0 @@ -/* Copyright 2011 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.doctor; - -public class Download { - - /* Nickname of the authority from which this download was made. */ - private String authority; - public String getAuthority() { - return this.authority; - } - - /* Request URL. */ - private String url; - public String getUrl() { - return this.url; - } - - /* Unparsed response string. */ - private String responseString; - public String getResponseString() { - return this.responseString; - } - - /* Request start timestamp. */ - private long requestStartMillis; - public long getRequestStartMillis() { - return this.requestStartMillis; - } - - /* Fetch time in millis. */ - private long fetchTime; - public long getFetchTime() { - return this.fetchTime; - } - - public Download(String authority, String url, String responseString, - long requestStartMillis, long fetchTime) { - this.authority = authority; - this.url = url; - this.responseString = responseString; - this.requestStartMillis = requestStartMillis; - this.fetchTime = fetchTime; - } -} - diff --git a/src/org/torproject/doctor/DownloadStatistics.java b/src/org/torproject/doctor/DownloadStatistics.java index 9eeffde..9df0333 100644 --- a/src/org/torproject/doctor/DownloadStatistics.java +++ b/src/org/torproject/doctor/DownloadStatistics.java @@ -4,21 +4,26 @@ package org.torproject.doctor;

import java.io.*; import java.util.*; +import org.torproject.descriptor.*;

public class DownloadStatistics { - public void memorizeFetchTimes(List<Download> downloadedConsensuses) { + public void memorizeFetchTimes(List<DescriptorRequest> downloads) { try { BufferedWriter bw = new BufferedWriter(new FileWriter( this.statisticsFile, true)); - for (Download downloadedConsensus : downloadedConsensuses) { - String authority = downloadedConsensus.getAuthority(); - long requestStartMillis = - downloadedConsensus.getRequestStartMillis(); - long fetchTimeMillis = downloadedConsensus.getFetchTime(); - String line = authority + "," - + String.valueOf(requestStartMillis) + "," - + String.valueOf(fetchTimeMillis); - bw.write(line + "\n"); + for (DescriptorRequest request : downloads) { + for (Descriptor descriptor : request.getDescriptors()) { + if (descriptor instanceof RelayNetworkStatusConsensus) { + String authority = request.getDirectoryNickname(); + long requestStartMillis = request.getRequestStart(); + long fetchTimeMillis = request.getRequestEnd() + - request.getRequestStart(); + String line = authority + "," + + String.valueOf(requestStartMillis) + "," + + String.valueOf(fetchTimeMillis); + bw.write(line + "\n"); + } + } } bw.close(); } catch (IOException e) { diff --git a/src/org/torproject/doctor/Downloader.java b/src/org/torproject/doctor/Downloader.java index a82ac8d..317b3b6 100755 --- a/src/org/torproject/doctor/Downloader.java +++ b/src/org/torproject/doctor/Downloader.java @@ -13,8 +13,8 @@ import org.torproject.descriptor.*; * votes. */ public class Downloader {

- /* Download a new consensus and corresponding votes. */ - public void downloadFromAuthorities() { + /* Download the current consensus and corresponding votes. */ + public List<DescriptorRequest> downloadFromAuthorities() {

RelayDescriptorDownloader downloader = DescriptorSourceFactory.createRelayDescriptorDownloader(); @@ -33,58 +33,24 @@ public class Downloader {

downloader.setRequestTimeout(60L * 1000L);

+ List<DescriptorRequest> allRequests = + new ArrayList<DescriptorRequest>(); Iterator<DescriptorRequest> descriptorRequests = downloader.downloadDescriptors(); while (descriptorRequests.hasNext()) { - DescriptorRequest request = descriptorRequests.next(); - String authority = request.getDirectoryNickname(); - String requestUrl = request.getRequestUrl(); - long requestStart = request.getRequestStart(); - long fetchTime = request.getRequestEnd() - - request.getRequestStart(); - if (request.globalTimeoutHasExpired()) { - System.err.println("Global timeout has expired. Exiting."); - System.exit(1); - } else if (!request.requestTimeoutHasExpired()) { - if (request.getDescriptors().isEmpty()) { - /* No response. We'll realize later on if we're missing a - * consensus or vote. */ - continue; - } else if (request.getDescriptors().size() > 1) { - System.out.println("Response contains more than 1 " - + "descriptor. Considering only the first."); - } - Descriptor downloadedDescriptor = request.getDescriptors().get(0); - String response = new String(request.getDescriptors().get(0). - getRawDescriptorBytes()); - Download download = new Download(authority, requestUrl, response, - requestStart, fetchTime); - if (downloadedDescriptor instanceof - RelayNetworkStatusConsensus) { - this.downloadedConsensuses.add(download); - } else if (downloadedDescriptor instanceof - RelayNetworkStatusVote) { - this.downloadedVotes.add(download); - } else { - System.err.println("Did not expect a descriptor of type " - + downloadedDescriptor.getClass() + ". Ignoring."); - } + try { + allRequests.add(descriptorRequests.next()); + } catch (NoSuchElementException e) { + /* TODO In theory, this exception shouldn't be thrown. */ + System.err.println("Internal error: next() doesn't provide an " + + "element even though hasNext() returned true. Got " + + allRequests.size() + " elements so far. Stopping to " + + "request further elements."); + break; } } - } - - /* Return the previously downloaded (unparsed) consensus string by - * authority nickname. */ - private List<Download> downloadedConsensuses = - new ArrayList<Download>(); - public List<Download> getConsensuses() { - return this.downloadedConsensuses; - }

- /* Return the previously downloaded (unparsed) vote strings. */ - private List<Download> downloadedVotes = new ArrayList<Download>(); - public List<Download> getVotes() { - return this.downloadedVotes; + return allRequests; } }

diff --git a/src/org/torproject/doctor/Main.java b/src/org/torproject/doctor/Main.java index 0de6298..40973b4 100755 --- a/src/org/torproject/doctor/Main.java +++ b/src/org/torproject/doctor/Main.java @@ -3,53 +3,37 @@ package org.torproject.doctor;

import java.util.*; +import org.torproject.descriptor.*;

/* Coordinate the process of downloading consensus and votes to check * Tor's consensus health. */ public class Main { public static void main(String[] args) {

- /* Initialize reports. */ - List<Report> reports = new ArrayList<Report>(); - reports.add(new MetricsWebsiteReport( - "website/consensus-health.html")); - reports.add(new StatusFileReport()); - /* Download consensus and corresponding votes from the directory * authorities. */ Downloader downloader = new Downloader(); - downloader.downloadFromAuthorities(); - List<Download> downloadedConsensuses = downloader.getConsensuses(); - List<Download> downloadedVotes = downloader.getVotes(); - - /* Write fetch times for requesting consensuses to disk and prepare - * statistics about fetch times in the last 7 days. */ - DownloadStatistics fetchStatistics = new DownloadStatistics(); - fetchStatistics.memorizeFetchTimes(downloadedConsensuses); - fetchStatistics.prepareStatistics(); + List<DescriptorRequest> downloads = + downloader.downloadFromAuthorities();

- /* Parse consensus and votes. */ - Parser parser = new Parser(); - SortedMap<String, Status> parsedDownloadedConsensuses = parser.parse( - downloadedConsensuses, downloadedVotes); - - /* Check consensus and votes for possible problems. */ + /* Check consensus and votes for possible problems and write warnings + * to status files. */ + StatusFileReport statusFile = new StatusFileReport(); Checker checker = new Checker(); - checker.processDownloadedConsensuses(parsedDownloadedConsensuses); + checker.processDownloadedConsensuses(downloads); SortedMap<Warning, String> warnings = checker.getWarnings(); + statusFile.processWarnings(warnings); + statusFile.writeReport();

- /* Pass warnings, consensuses, and votes to the reports, and finish - * writing them. */ - for (Report report : reports) { - report.processWarnings(warnings); - report.processDownloadedConsensuses(parsedDownloadedConsensuses); - report.includeFetchStatistics(fetchStatistics); - report.writeReport(); - } - - /* Terminate the program including any download threads that may still - * be running. */ - System.exit(0); + /* Write a complete consensus-health report to an HTML file. */ + MetricsWebsiteReport website = + new MetricsWebsiteReport("website/consensus-health.html"); + website.processDownloadedConsensuses(downloads); + DownloadStatistics fetchStatistics = new DownloadStatistics(); + fetchStatistics.memorizeFetchTimes(downloads); + fetchStatistics.prepareStatistics(); + website.includeFetchStatistics(fetchStatistics); + website.writeReport(); } }

diff --git a/src/org/torproject/doctor/MetricsWebsiteReport.java b/src/org/torproject/doctor/MetricsWebsiteReport.java index 9ae897a..1529461 100755 --- a/src/org/torproject/doctor/MetricsWebsiteReport.java +++ b/src/org/torproject/doctor/MetricsWebsiteReport.java @@ -5,10 +5,11 @@ package org.torproject.doctor; import java.io.*; import java.text.*; import java.util.*; +import org.torproject.descriptor.*;

/* Transform the most recent consensus and corresponding votes into an * HTML page showing possible irregularities. */ -public class MetricsWebsiteReport implements Report { +public class MetricsWebsiteReport {

/* Date-time format to format timestamps. */ private static SimpleDateFormat dateTimeFormat; @@ -25,30 +26,35 @@ public class MetricsWebsiteReport implements Report { this.htmlOutputFile = new File(htmlOutputFilename); }

- /* Process warnings. */ - public void processWarnings(SortedMap<Warning, String> warnings) { - /* We could use these warnings instead of running all checks - * ourselves. But we're not doing that yet. */ - } - /* Store the downloaded consensus and corresponding votes for later * processing. */ - private Status downloadedConsensus; - private SortedSet<Status> downloadedVotes; + private RelayNetworkStatusConsensus downloadedConsensus; + private SortedMap<String, RelayNetworkStatusVote> downloadedVotes = + new TreeMap<String, RelayNetworkStatusVote>(); public void processDownloadedConsensuses( - SortedMap<String, Status> downloadedConsensuses) { + List<DescriptorRequest> downloads) { long mostRecentValidAfterMillis = -1L; - for (Status downloadedConsensus : downloadedConsensuses.values()) { - if (downloadedConsensus.getValidAfterMillis() > - mostRecentValidAfterMillis) { - this.downloadedConsensus = downloadedConsensus; - mostRecentValidAfterMillis = - downloadedConsensus.getValidAfterMillis(); + for (DescriptorRequest request : downloads) { + for (Descriptor descriptor : request.getDescriptors()) { + if (descriptor instanceof RelayNetworkStatusConsensus) { + RelayNetworkStatusConsensus downloadedConsensus = + (RelayNetworkStatusConsensus) descriptor; + if (downloadedConsensus.getValidAfterMillis() > + mostRecentValidAfterMillis) { + this.downloadedConsensus = downloadedConsensus; + mostRecentValidAfterMillis = + downloadedConsensus.getValidAfterMillis(); + } + } else if (descriptor instanceof RelayNetworkStatusVote) { + RelayNetworkStatusVote vote = + (RelayNetworkStatusVote) descriptor; + this.downloadedVotes.put(vote.getNickname(), vote); + } else { + System.err.println("Did not expect a descriptor of type " + + descriptor.getClass() + ". Ignoring."); + } } } - if (this.downloadedConsensus != null) { - this.downloadedVotes = this.downloadedConsensus.getVotes(); - } }

/* Store the DownloadStatistics reference to request download statistics @@ -177,7 +183,7 @@ public class MetricsWebsiteReport implements Report { if (this.downloadedVotes.size() < 1) { this.bw.write(" <tr><td>(No votes.)</td><td></td></tr>\n"); } else { - for (Status vote : this.downloadedVotes) { + for (RelayNetworkStatusVote vote : this.downloadedVotes.values()) { this.bw.write(" <tr>\n" + " <td>" + vote.getNickname() + "</td>\n" + " <td>known-flags"); @@ -218,24 +224,36 @@ public class MetricsWebsiteReport implements Report { this.bw.write(" <tr><td>(No votes.)</td><td></td><td></td>" + "</tr>\n"); } else { - for (Status vote : this.downloadedVotes) { + for (RelayNetworkStatusVote vote : this.downloadedVotes.values()) { + int runningRelays = 0; + for (NetworkStatusEntry entry : + vote.getStatusEntries().values()) { + if (entry.getFlags().contains("Running")) { + runningRelays++; + } + } this.bw.write(" <tr>\n" + " <td>" + vote.getNickname() + "</td>\n" + " <td>" + vote.getStatusEntries().size() + " total</td>\n" - + " <td>" + vote.getRunningRelays() - + " Running</td>\n" + + " <td>" + runningRelays + " Running</td>\n" + " </tr>\n"); } } + int runningRelays = 0; + for (NetworkStatusEntry entry : + this.downloadedConsensus.getStatusEntries().values()) { + if (entry.getFlags().contains("Running")) { + runningRelays++; + } + } this.bw.write(" <tr>\n" + " <td><font color="blue">consensus</font>" + "</td>\n" + " <td><font color="blue">" + this.downloadedConsensus.getStatusEntries().size() + " total</font></td>\n" - + " <td><font color="blue">" - + this.downloadedConsensus.getRunningRelays() + + " <td><font color="blue">" + runningRelays + " Running</font></td>\n" + " </tr>\n" + " </table>\n"); @@ -258,11 +276,10 @@ public class MetricsWebsiteReport implements Report { if (this.downloadedVotes.size() < 1) { this.bw.write(" <tr><td>(No votes.)</td><td></td></tr>\n"); } else { - for (Status vote : this.downloadedVotes) { - SortedSet<Integer> consensusMethods = - vote.getConsensusMethods(); + for (RelayNetworkStatusVote vote : this.downloadedVotes.values()) { + List<Integer> consensusMethods = vote.getConsensusMethods(); if (consensusMethods.contains( - this.downloadedConsensus.getConsensusMethods().last())) { + this.downloadedConsensus.getConsensusMethod())) { this.bw.write(" <tr>\n" + " <td>" + vote.getNickname() + "</td>\n" + " <td>consensus-methods"); @@ -289,7 +306,7 @@ public class MetricsWebsiteReport implements Report { + " <td><font color="blue">consensus</font>" + "</td>\n" + " <td><font color="blue">consensus-method " - + this.downloadedConsensus.getConsensusMethods().last() + + this.downloadedConsensus.getConsensusMethod() + "</font></td>\n" + " </tr>\n" + " </table>\n"); @@ -311,10 +328,10 @@ public class MetricsWebsiteReport implements Report { if (this.downloadedVotes.size() < 1) { this.bw.write(" <tr><td>(No votes.)</td><td></td></tr>\n"); } else { - for (Status vote : this.downloadedVotes) { + for (RelayNetworkStatusVote vote : this.downloadedVotes.values()) { SortedSet<String> voteRecommendedClientVersions = vote.getRecommendedClientVersions(); - if (voteRecommendedClientVersions != null) { + if (!voteRecommendedClientVersions.isEmpty()) { if (downloadedConsensus.getRecommendedClientVersions().equals( voteRecommendedClientVersions)) { this.bw.write(" <tr>\n" @@ -342,7 +359,7 @@ public class MetricsWebsiteReport implements Report { } SortedSet<String> voteRecommendedServerVersions = vote.getRecommendedServerVersions(); - if (voteRecommendedServerVersions != null) { + if (!voteRecommendedServerVersions.isEmpty()) { if (downloadedConsensus.getRecommendedServerVersions().equals( voteRecommendedServerVersions)) { this.bw.write(" <tr>\n" @@ -415,7 +432,7 @@ public class MetricsWebsiteReport implements Report { + "cbtmintimeout,cbtinitialtimeout").split(","))); Map<String, String> consensusConsensusParams = downloadedConsensus.getConsensusParams(); - for (Status vote : this.downloadedVotes) { + for (RelayNetworkStatusVote vote : this.downloadedVotes.values()) { Map<String, String> voteConsensusParams = vote.getConsensusParams(); boolean conflictOrInvalid = false; @@ -485,7 +502,7 @@ public class MetricsWebsiteReport implements Report { if (this.downloadedVotes.size() < 1) { this.bw.write(" <tr><td>(No votes.)</td><td></td></tr>\n"); } else { - for (Status vote : this.downloadedVotes) { + for (RelayNetworkStatusVote vote : this.downloadedVotes.values()) { long voteDirKeyExpiresMillis = vote.getDirKeyExpiresMillis(); if (voteDirKeyExpiresMillis - 14L * 24L * 60L * 60L * 1000L < System.currentTimeMillis()) { @@ -530,11 +547,17 @@ public class MetricsWebsiteReport implements Report { if (this.downloadedVotes.size() < 1) { this.bw.write(" <tr><td>(No votes.)</td><td></td></tr>\n"); } else { - for (Status vote : this.downloadedVotes) { - if (vote.getBandwidthWeights() > 0) { + for (RelayNetworkStatusVote vote : this.downloadedVotes.values()) { + int bandwidthWeights = 0; + for (NetworkStatusEntry entry : vote.getStatusEntries().values()) { + if (entry.getBandwidth().contains("Measured=")) { + bandwidthWeights++; + } + } + if (bandwidthWeights > 0) { this.bw.write(" <tr>\n" + " <td>" + vote.getNickname() + "</td>\n" - + " <td>" + vote.getBandwidthWeights() + + " <td>" + bandwidthWeights + " Measured values in w lines</td>\n" + " </tr>\n"); } @@ -550,8 +573,14 @@ public class MetricsWebsiteReport implements Report { + " <h3><a href="#authorityversions" class="anchor">" + "Authority versions</a></h3>\n" + " <br>\n"); - Map<String, String> authorityVersions = - this.downloadedConsensus.getAuthorityVersions(); + SortedMap<String, String> authorityVersions = + new TreeMap<String, String>(); + for (NetworkStatusEntry entry : + this.downloadedConsensus.getStatusEntries().values()) { + if (entry.getFlags().contains("Authority")) { + authorityVersions.put(entry.getNickname(), entry.getVersion()); + } + } if (authorityVersions.size() < 1) { this.bw.write(" <p>(No relays with Authority flag found.)" + "</p>\n"); @@ -671,13 +700,14 @@ public class MetricsWebsiteReport implements Report { } this.bw.write(" </colgroup>\n"); SortedMap<String, String> allRelays = new TreeMap<String, String>(); - for (Status vote : this.downloadedVotes) { - for (StatusEntry statusEntry : vote.getStatusEntries().values()) { + for (RelayNetworkStatusVote vote : this.downloadedVotes.values()) { + for (NetworkStatusEntry statusEntry : + vote.getStatusEntries().values()) { allRelays.put(statusEntry.getFingerprint(), statusEntry.getNickname()); } } - for (StatusEntry statusEntry : + for (NetworkStatusEntry statusEntry : this.downloadedConsensus.getStatusEntries().values()) { allRelays.put(statusEntry.getFingerprint(), statusEntry.getNickname()); @@ -699,7 +729,7 @@ public class MetricsWebsiteReport implements Report { private void writeRelayFlagsTableHeader() throws IOException { this.bw.write(" <tr><td><br><b>Fingerprint</b></td>" + "<td><br><b>Nickname</b></td>\n"); - for (Status vote : this.downloadedVotes) { + for (RelayNetworkStatusVote vote : this.downloadedVotes.values()) { String shortDirName = vote.getNickname().length() > 6 ? vote.getNickname().substring(0, 5) + "." : vote.getNickname(); @@ -714,8 +744,7 @@ public class MetricsWebsiteReport implements Report { this.bw.write(" <tr>\n"); if (this.downloadedConsensus.containsStatusEntry(fingerprint) && this.downloadedConsensus.getStatusEntry(fingerprint).getFlags(). - contains("Named") && - !Character.isDigit(nickname.charAt(0))) { + contains("Named") && !Character.isDigit(nickname.charAt(0))) { this.bw.write(" <td id="" + nickname + ""><a href="relay.html?fingerprint=" + fingerprint + "" target="_blank">" @@ -727,18 +756,18 @@ public class MetricsWebsiteReport implements Report { } this.bw.write(" <td>" + nickname + "</td>\n"); SortedSet<String> relevantFlags = new TreeSet<String>(); - for (Status vote : this.downloadedVotes) { + for (RelayNetworkStatusVote vote : this.downloadedVotes.values()) { if (vote.containsStatusEntry(fingerprint)) { relevantFlags.addAll(vote.getStatusEntry(fingerprint).getFlags()); } } SortedSet<String> consensusFlags = null; if (this.downloadedConsensus.containsStatusEntry(fingerprint)) { - consensusFlags = this.downloadedConsensus. - getStatusEntry(fingerprint).getFlags(); + consensusFlags = this.downloadedConsensus.getStatusEntries().get( + fingerprint).getFlags(); relevantFlags.addAll(consensusFlags); } - for (Status vote : this.downloadedVotes) { + for (RelayNetworkStatusVote vote : this.downloadedVotes.values()) { if (vote.containsStatusEntry(fingerprint)) { SortedSet<String> flags = vote.getStatusEntry(fingerprint). getFlags(); @@ -818,7 +847,7 @@ public class MetricsWebsiteReport implements Report { + "<td><b>In vote and consensus</b></td>" + "<td><b>Only in consensus</b></td>\n"); Set<String> allFingerprints = new HashSet<String>(); - for (Status vote : this.downloadedVotes) { + for (RelayNetworkStatusVote vote : this.downloadedVotes.values()) { allFingerprints.addAll(vote.getStatusEntries().keySet()); } allFingerprints.addAll(this.downloadedConsensus.getStatusEntries(). @@ -832,9 +861,9 @@ public class MetricsWebsiteReport implements Report { for (String fingerprint : allFingerprints) { SortedSet<String> consensusFlags = this.downloadedConsensus.containsStatusEntry(fingerprint) ? - this.downloadedConsensus.getStatusEntry(fingerprint).getFlags() : - null; - for (Status vote : this.downloadedVotes) { + this.downloadedConsensus.getStatusEntry(fingerprint).getFlags() + : null; + for (RelayNetworkStatusVote vote : this.downloadedVotes.values()) { String dir = vote.getNickname(); if (vote.containsStatusEntry(fingerprint)) { SortedSet<String> flags = vote.getStatusEntry(fingerprint). @@ -868,7 +897,7 @@ public class MetricsWebsiteReport implements Report { } } } - for (Status vote : this.downloadedVotes) { + for (RelayNetworkStatusVote vote : this.downloadedVotes.values()) { String dir = vote.getNickname(); int i = 0; for (String flag : vote.getKnownFlags()) { diff --git a/src/org/torproject/doctor/Parser.java b/src/org/torproject/doctor/Parser.java deleted file mode 100755 index d318f29..0000000 --- a/src/org/torproject/doctor/Parser.java +++ /dev/null @@ -1,180 +0,0 @@ -/* Copyright 2011 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.doctor; - -import java.io.*; -import java.text.*; -import java.util.*; -import org.apache.commons.codec.binary.*; - -/* Parse a network status consensus or vote. */ -public class Parser { - - /* Parse and return a consensus and corresponding votes, or null if - * something goes wrong. */ - public SortedMap<String, Status> parse( - List<Download> downloadedConsensuses, - List<Download> downloadedVotes) { - SortedSet<Status> parsedVotes = new TreeSet<Status>(); - for (Download downloadedVote : downloadedVotes) { - String voteString = downloadedVote.getResponseString(); - long fetchTime = downloadedVote.getFetchTime(); - Status parsedVote = this.parseConsensusOrVote(voteString, fetchTime, - false); - if (parsedVote != null) { - parsedVotes.add(parsedVote); - } - } - SortedMap<String, Status> parsedConsensuses = - new TreeMap<String, Status>(); - for (Download downloadedConsensus : downloadedConsensuses) { - String nickname = downloadedConsensus.getAuthority(); - String consensusString = downloadedConsensus.getResponseString(); - long fetchTime = downloadedConsensus.getFetchTime(); - Status parsedConsensus = this.parseConsensusOrVote(consensusString, - fetchTime, true); - if (parsedConsensus != null) { - for (Status parsedVote : parsedVotes) { - if (parsedConsensus.getValidAfterMillis() == - parsedVote.getValidAfterMillis()) { - parsedConsensus.addVote(parsedVote); - } - } - parsedConsensuses.put(nickname, parsedConsensus); - } - } - return parsedConsensuses; - } - - /* Date-time formats to parse and format timestamps. */ - private static SimpleDateFormat dateTimeFormat; - static { - dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - } - - /* Parse a consensus or vote string into a Status instance. */ - private Status parseConsensusOrVote(String statusString, long fetchTime, - boolean isConsensus) { - if (statusString == null) { - return null; - } - Status status = new Status(); - status.setUnparsedString(statusString); - status.setFetchTime(fetchTime); - try { - BufferedReader br = new BufferedReader(new StringReader( - statusString)); - String line, rLine = null, sLine = null; - int totalRelays = 0, runningRelays = 0, bandwidthWeights = 0; - while ((line = br.readLine()) != null) { - if (line.startsWith("consensus-method ") || - line.startsWith("consensus-methods ")) { - SortedSet<Integer> consensusMethods = new TreeSet<Integer>(); - String[] parts = line.split(" "); - for (int i = 1; i < parts.length; i++) { - consensusMethods.add(Integer.parseInt(parts[i])); - } - status.setConsensusMethods(consensusMethods); - } else if (line.startsWith("valid-after ")) { - try { - status.setValidAfterMillis(dateTimeFormat.parse( - line.substring("valid-after ".length())).getTime()); - } catch (ParseException e) { - System.err.println("Could not parse valid-after timestamp in " - + "line '" + line + "' of a " - + (isConsensus ? "consensus" : "vote") + ". Skipping."); - return null; - } - } else if (line.startsWith("client-versions ")) { - status.setRecommendedClientVersions( - new TreeSet<String>(Arrays.asList( - line.split(" ")[1].split(",")))); - } else if (line.startsWith("server-versions ")) { - status.setRecommendedServerVersions( - new TreeSet<String>(Arrays.asList( - line.split(" ")[1].split(",")))); - } else if (line.startsWith("known-flags ")) { - for (String flag : line.substring("known-flags ".length()). - split(" ")) { - status.addKnownFlag(flag); - } - } else if (line.startsWith("params ")) { - if (line.length() > "params ".length()) { - for (String param : - line.substring("params ".length()).split(" ")) { - String paramName = param.split("=")[0]; - String paramValue = param.split("=")[1]; - status.addConsensusParam(paramName, paramValue); - } - } - } else if (line.startsWith("dir-source ")) { - String nickname = line.split(" ")[1]; - String fingerprint = line.split(" ")[2]; - if (isConsensus) { - status.addContainedVote(fingerprint); - } else { - status.setNickname(line.split(" ")[1]); - status.setFingerprint(line.split(" ")[2]); - } - } else if (line.startsWith("dir-key-expires ")) { - try { - status.setDirKeyExpiresMillis(dateTimeFormat.parse( - line.substring("dir-key-expires ".length())).getTime()); - } catch (ParseException e) { - System.err.println("Could not parse dir-key-expires " - + "timestamp in line '" + line + "' of a " - + (isConsensus ? "consensus" : "vote") + ". Skipping."); - return null; - } - } else if (line.startsWith("r ") || - line.equals("directory-footer")) { - if (rLine != null) { - StatusEntry statusEntry = new StatusEntry(); - statusEntry.setNickname(rLine.split(" ")[1]); - statusEntry.setFingerprint(Hex.encodeHexString( - Base64.decodeBase64(rLine.split(" ")[2] + "=")). - toUpperCase()); - SortedSet<String> flags = new TreeSet<String>(); - if (sLine.length() > 2) { - for (String flag : sLine.substring(2).split(" ")) { - flags.add(flag); - } - } - statusEntry.setFlags(flags); - status.addStatusEntry(statusEntry); - } - if (line.startsWith("r ")) { - rLine = line; - } - } else if (line.startsWith("s ") || line.equals("s")) { - sLine = line; - if (line.contains(" Running")) { - runningRelays++; - } - } else if (line.startsWith("v ") && - sLine.contains(" Authority")) { - String nickname = rLine.split(" ")[1]; - String versionString = line.substring(2); - status.addAuthorityVersion(nickname, versionString); - } else if (line.startsWith("w ") && !isConsensus && - line.contains(" Measured")) { - bandwidthWeights++; - } else if (line.startsWith("directory-signature ") && - isConsensus) { - String fingerprint = line.split(" ")[1]; - status.addContainedSignature(fingerprint); - } - } - br.close(); - status.setRunningRelays(runningRelays); - status.setBandwidthWeights(bandwidthWeights); - } catch (IOException e) { - System.err.println("Caught an IOException while parsing a " - + (isConsensus ? "consensus" : "vote") + " string. Skipping."); - return null; - } - return status; - } -} - diff --git a/src/org/torproject/doctor/Report.java b/src/org/torproject/doctor/Report.java deleted file mode 100755 index 2e508bc..0000000 --- a/src/org/torproject/doctor/Report.java +++ /dev/null @@ -1,27 +0,0 @@ -/* Copyright 2011 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.doctor; - -import java.util.*; - -/* Transform findings from parsing consensuses and votes into a report of - * some form. */ -public interface Report { - - /* Process the downloaded current consensus and corresponding votes to - * find irregularities between them. */ - public abstract void processDownloadedConsensuses( - SortedMap<String, Status> downloadedConsensuses); - - /* Process warnings consisting of warning type and details. */ - public abstract void processWarnings( - SortedMap<Warning, String> warnings); - - /* Include download statistics. */ - public abstract void includeFetchStatistics( - DownloadStatistics statistics); - - /* Finish writing report. */ - public abstract void writeReport(); -} - diff --git a/src/org/torproject/doctor/Status.java b/src/org/torproject/doctor/Status.java deleted file mode 100755 index df41e47..0000000 --- a/src/org/torproject/doctor/Status.java +++ /dev/null @@ -1,205 +0,0 @@ -/* Copyright 2011 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.doctor; - -import java.util.*; - -/* Contains the unparsed string and parsed fields from a network status - * consensus or vote. */ -public class Status implements Comparable<Status> { - - /* Helper methods to implement the Comparable interface; Status - * instances are compared by nickname of the publishing directory - * authorities. */ - public int compareTo(Status o) { - return this.nickname.compareTo(o.nickname); - } - public boolean equals(Object o) { - return (o instanceof Status && - this.nickname.equals(((Status) o).nickname)); - } - - /* Unparsed string that was downloaded or read from disk and that can - * be written to disk. */ - private String unparsedString; - public void setUnparsedString(String unparsedString) { - this.unparsedString = unparsedString; - } - public String getUnparsedString() { - return this.unparsedString; - } - - /* Fetch time in millis. */ - private long fetchTime; - public void setFetchTime(long fetchTime) { - this.fetchTime = fetchTime; - } - public long getFetchTime() { - return this.fetchTime; - } - - /* Votes published at the same time as this consensus; votes don't - * reference any statuses. */ - private SortedSet<Status> votes = new TreeSet<Status>(); - public void addVote(Status vote) { - this.votes.add(vote); - } - public SortedSet<Status> getVotes() { - return this.votes; - } - - /* Fingerprint of the directory authority publishing this vote; left - * empty for consensuses. */ - private String fingerprint; - public void setFingerprint(String fingerprint) { - this.fingerprint = fingerprint; - } - public String getFingerprint() { - return this.fingerprint; - } - - /* Nickname of the directory authority publishing this vote; left empty - * for consensuses. */ - private String nickname; - public void setNickname(String nickname) { - this.nickname= nickname; - } - public String getNickname() { - return this.nickname; - } - - /* Valid-after time in milliseconds. */ - private long validAfterMillis; - public void setValidAfterMillis(long validAfterMillis) { - this.validAfterMillis = validAfterMillis; - } - public long getValidAfterMillis() { - return this.validAfterMillis; - } - - /* Consensus parameters. */ - private SortedMap<String, String> consensusParams = - new TreeMap<String, String>(); - public void addConsensusParam(String paramName, String paramValue) { - this.consensusParams.put(paramName, paramValue); - } - public SortedMap<String, String> getConsensusParams() { - return this.consensusParams; - } - - /* Consensus methods supported by the directory authority sending a vote - * or of the produced consensus. */ - private SortedSet<Integer> consensusMethods; - public void setConsensusMethods(SortedSet<Integer> consensusMethods) { - this.consensusMethods = consensusMethods; - } - public SortedSet<Integer> getConsensusMethods() { - return this.consensusMethods; - } - - /* Recommended server versions. */ - private SortedSet<String> recommendedServerVersions; - public void setRecommendedServerVersions( - SortedSet<String> recommendedServerVersions) { - this.recommendedServerVersions = recommendedServerVersions; - } - public SortedSet<String> getRecommendedServerVersions() { - return this.recommendedServerVersions; - } - - /* Recommended client versions. */ - private SortedSet<String> recommendedClientVersions; - public void setRecommendedClientVersions( - SortedSet<String> recommendedClientVersions) { - this.recommendedClientVersions = recommendedClientVersions; - } - public SortedSet<String> getRecommendedClientVersions() { - return this.recommendedClientVersions; - } - - /* Expiration times of directory signing keys. */ - private long dirKeyExpiresMillis; - public void setDirKeyExpiresMillis(long dirKeyExpiresMillis) { - this.dirKeyExpiresMillis = dirKeyExpiresMillis; - } - public long getDirKeyExpiresMillis() { - return this.dirKeyExpiresMillis; - } - - /* Known flags by the directory authority sending a vote or of the - * produced consensus. */ - private SortedSet<String> knownFlags = new TreeSet<String>(); - public void addKnownFlag(String knownFlag) { - this.knownFlags.add(knownFlag); - } - public SortedSet<String> getKnownFlags() { - return this.knownFlags; - } - - /* Fingerprints of directory authorities of contained votes (only - * relevant for consensuses). */ - private SortedSet<String> containedVotes = new TreeSet<String>(); - public void addContainedVote(String fingerprint) { - this.containedVotes.add(fingerprint); - } - public SortedSet<String> getContainedVotes() { - return this.containedVotes; - } - - /* Fingerprints of directory authorities of contained signatures (only - * relevant for consensuses). */ - private SortedSet<String> containedSignatures = new TreeSet<String>(); - public void addContainedSignature(String fingerprint) { - this.containedSignatures.add(fingerprint); - } - public SortedSet<String> getContainedSignatures() { - return this.containedSignatures; - } - - /* Number of status entries with the Running flag. */ - private int runningRelays; - public void setRunningRelays(int runningRelays) { - this.runningRelays = runningRelays; - } - public int getRunningRelays() { - return this.runningRelays; - } - - /* Number of status entries containing bandwidth weights (only relevant - * in votes). */ - private int bandwidthWeights; - public void setBandwidthWeights(int bandwidthWeights) { - this.bandwidthWeights = bandwidthWeights; - } - public int getBandwidthWeights() { - return this.bandwidthWeights; - } - - /* Status entries contained in this status. */ - private SortedMap<String, StatusEntry> statusEntries = - new TreeMap<String, StatusEntry>(); - public void addStatusEntry(StatusEntry statusEntry) { - this.statusEntries.put(statusEntry.getFingerprint(), statusEntry); - } - public SortedMap<String, StatusEntry> getStatusEntries() { - return this.statusEntries; - } - public boolean containsStatusEntry(String fingerprint) { - return this.statusEntries.containsKey(fingerprint); - } - public StatusEntry getStatusEntry(String fingerprint) { - return this.statusEntries.get(fingerprint); - } - - /* Versions of directory authorities (only set in a consensus). */ - private SortedMap<String, String> authorityVersions = - new TreeMap<String, String>(); - public void addAuthorityVersion(String fingerprint, - String versionString) { - this.authorityVersions.put(fingerprint, versionString); - } - public SortedMap<String, String> getAuthorityVersions() { - return this.authorityVersions; - } -} - diff --git a/src/org/torproject/doctor/StatusEntry.java b/src/org/torproject/doctor/StatusEntry.java deleted file mode 100755 index 18f1149..0000000 --- a/src/org/torproject/doctor/StatusEntry.java +++ /dev/null @@ -1,48 +0,0 @@ -/* Copyright 2011 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.doctor; - -import java.util.*; - -/* Contains the parsed data from a network status entry contained in a - * network status consensus or vote. */ -public class StatusEntry implements Comparable<StatusEntry> { - - /* Helper methods to implement the Comparable interface; StatusEntry - * instances are compared by fingerprint. */ - public int compareTo(StatusEntry o) { - return this.fingerprint.compareTo(o.fingerprint); - } - public boolean equals(Object o) { - return (o instanceof StatusEntry && - this.fingerprint.equals(((StatusEntry) o).fingerprint)); - } - - /* Relay fingerprint. */ - private String fingerprint; - public void setFingerprint(String fingerprint) { - this.fingerprint = fingerprint; - } - public String getFingerprint() { - return this.fingerprint; - } - - /* Relay nickname. */ - private String nickname; - public void setNickname(String nickname) { - this.nickname = nickname; - } - public String getNickname() { - return this.nickname; - } - - /* Relay flags. */ - private SortedSet<String> flags; - public void setFlags(SortedSet<String> flags) { - this.flags = flags; - } - public SortedSet<String> getFlags() { - return this.flags; - } -} - diff --git a/src/org/torproject/doctor/StatusFileReport.java b/src/org/torproject/doctor/StatusFileReport.java index 884dc18..7890005 100755 --- a/src/org/torproject/doctor/StatusFileReport.java +++ b/src/org/torproject/doctor/StatusFileReport.java @@ -8,7 +8,7 @@ import java.util.*;

/* Check a given consensus and votes for irregularities and write results * to stdout while rate-limiting warnings based on severity. */ -public class StatusFileReport implements Report { +public class StatusFileReport {

/* Date-time format to format timestamps. */ private static SimpleDateFormat dateTimeFormat; @@ -17,27 +17,12 @@ public class StatusFileReport implements Report { dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); }

- /* Downloaded consensus and corresponding votes for later - * processing. */ - private SortedMap<String, Status> downloadedConsensuses; - private Status downloadedConsensus; - private SortedSet<Status> downloadedVotes; - public void processDownloadedConsensuses( - SortedMap<String, Status> downloadedConsensuses) { - this.downloadedConsensuses = downloadedConsensuses; - } - /* Warnings obtained from checking the current consensus and votes. */ private SortedMap<Warning, String> warnings; public void processWarnings(SortedMap<Warning, String> warnings) { this.warnings = warnings; }

- /* Ignore download statistics for this report. */ - public void includeFetchStatistics(DownloadStatistics statistics) { - /* Do nothing. */ - } - /* Check consensuses and votes for irregularities and write output to * stdout. */ public void writeReport() {