- tor-commits - lists.torproject.org

[metrics-db/master] Minor fix that only affects the logs.
by karsten＠torproject.org 03 Apr '11

03 Apr '11

commit 215aac6d30a7c62ef6d1ec1c6c3ef655865db784 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sun Apr 3 13:23:21 2011 +0200 Minor fix that only affects the logs. Wait with putting the current consensus on the missing list until we start downloading descriptors. We might have learned it before when parsing a local descriptor directory. --- .../ernie/db/RelayDescriptorDownloader.java | 17 ++++++++--------- 1 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/org/torproject/ernie/db/RelayDescriptorDownloader.java b/src/org/torproject/ernie/db/RelayDescriptorDownloader.java index 190d1ae..835d821 100644 --- a/src/org/torproject/ernie/db/RelayDescriptorDownloader.java +++ b/src/org/torproject/ernie/db/RelayDescriptorDownloader.java @@ -275,14 +275,6 @@ public class RelayDescriptorDownloader { } } - /* Put the current consensus on the missing list, unless we already - * have it. */ - String consensusKey = "consensus," + this.currentValidAfter; - if (!this.missingDescriptors.containsKey(consensusKey)) { - this.missingDescriptors.put(consensusKey, "NA"); - this.newMissingConsensuses++; - } - /* Read list of directory authorities and when we last downloaded all * server and extra-info descriptors from them. */ this.lastDownloadedAllDescriptors = new HashMap<String, String>(); @@ -453,6 +445,14 @@ public class RelayDescriptorDownloader { */ public void downloadDescriptors() { + /* Put the current consensus on the missing list, unless we already + * have it. */ + String consensusKey = "consensus," + this.currentValidAfter; + if (!this.missingDescriptors.containsKey(consensusKey)) { + this.missingDescriptors.put(consensusKey, "NA"); + this.newMissingConsensuses++; + } + /* Download descriptors from authorities which are in random order, so * that we distribute the load somewhat fairly over time. */ for (String authority : authorities) { @@ -469,7 +469,6 @@ public class RelayDescriptorDownloader { /* Start with downloading the current consensus, unless we already * have it. */ if (downloadCurrentConsensus) { - String consensusKey = "consensus," + this.currentValidAfter; if (this.missingDescriptors.containsKey(consensusKey) && this.missingDescriptors.get(consensusKey).equals("NA")) { this.requestedConsensuses++;

1 0

[metrics-db/master] Limit download of all descriptors to 2 authorities.
by karsten＠torproject.org 03 Apr '11

03 Apr '11

commit 1ff4ce519f26fbdfd5cb3428ad7251e580764666 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sun Apr 3 14:26:08 2011 +0200 Limit download of all descriptors to 2 authorities. --- .../ernie/db/RelayDescriptorDownloader.java | 10 +++++----- 1 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/org/torproject/ernie/db/RelayDescriptorDownloader.java b/src/org/torproject/ernie/db/RelayDescriptorDownloader.java index 835d821..4b5d729 100644 --- a/src/org/torproject/ernie/db/RelayDescriptorDownloader.java +++ b/src/org/torproject/ernie/db/RelayDescriptorDownloader.java @@ -313,8 +313,8 @@ public class RelayDescriptorDownloader { } } - /* Make a list of directory authorities that we want to download all - * server and extra-info descriptors from. */ + /* Make a list of at most two directory authorities that we want to + * download all server and extra-info descriptors from. */ this.downloadAllDescriptorsFromAuthorities = new HashSet<String>(); for (String authority : this.authorities) { if (!this.lastDownloadedAllDescriptors.containsKey(authority) || @@ -322,6 +322,9 @@ public class RelayDescriptorDownloader { this.downloadAllDescriptorsCutOff) < 0) { this.downloadAllDescriptorsFromAuthorities.add(authority); } + if (this.downloadAllDescriptorsFromAuthorities.size() >= 2) { + break; + } } /* Prepare statistics on this execution. */ @@ -508,9 +511,6 @@ public class RelayDescriptorDownloader { /* Download all server or extra-info descriptors from this * authority if we haven't done so for 24 hours and if we're * configured to do so. */ - /* TODO Distribute downloads of all descriptors over the day for - * different authorities. Maybe limit the number of these - * downloads to 1 or 2 per execution. */ if (this.downloadAllDescriptorsFromAuthorities.contains( authority) && ((type.equals("server") && this.downloadAllServerDescriptors) ||

1 0

[metrics-db/master] Disable compression when downloading extra-info descriptors.
by karsten＠torproject.org 03 Apr '11

03 Apr '11

commit 7fe22d2bbbba84a5f59bbf496858832b2fe761b1 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sun Apr 3 13:05:54 2011 +0200 Disable compression when downloading extra-info descriptors. --- .../ernie/db/RelayDescriptorDownloader.java | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diff --git a/src/org/torproject/ernie/db/RelayDescriptorDownloader.java b/src/org/torproject/ernie/db/RelayDescriptorDownloader.java index ebbf761..190d1ae 100644 --- a/src/org/torproject/ernie/db/RelayDescriptorDownloader.java +++ b/src/org/torproject/ernie/db/RelayDescriptorDownloader.java @@ -623,7 +623,8 @@ public class RelayDescriptorDownloader { huc.connect(); int response = huc.getResponseCode(); if (response == 200) { - BufferedInputStream in = downloadCompressed + BufferedInputStream in = this.downloadCompressed && + !resource.startsWith("/tor/extra/") ? new BufferedInputStream(new InflaterInputStream( huc.getInputStream())) : new BufferedInputStream(huc.getInputStream());

1 0

[metrics-db/master] Download all server/extra-info descriptors once per day.
by karsten＠torproject.org 03 Apr '11

03 Apr '11

commit be6e7c3d4f9f9acdf5e2f459453c23a230c129d0 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Apr 1 15:43:38 2011 +0200 Download all server/extra-info descriptors once per day. The relay descriptor downloader so far only downloads server and extra-info descriptors that are referenced in other directory documents and that we are missing. With this patch we also download /tor/server/all and /tor/extra/all from the directories once per day to learn about server and extra-info descriptors that are not referenced anywhere. This patch further requests the compressed versions of descriptors by adding ".z" to the URLs in order to save bandwidth. This patch is a 50% rewrite of the relay descriptor downloader which became too complex over time. --- config.template | 27 +- src/org/torproject/ernie/db/Configuration.java | 56 ++- src/org/torproject/ernie/db/Main.java | 15 +- .../ernie/db/RelayDescriptorDownloader.java | 857 ++++++++++++-------- 4 files changed, 600 insertions(+), 355 deletions(-) diff --git a/config.template b/config.template index 47a8c8d..d744f25 100644 --- a/config.template +++ b/config.template @@ -24,7 +24,32 @@ # ## Comma separated list of directory authority addresses (IP[:port]) to ## download missing relay descriptors from -#DownloadFromDirectoryAuthorities 86.59.21.38,194.109.206.212,212.112.245.170 +#DownloadFromDirectoryAuthorities 86.59.21.38,216.224.124.114:9030,213.115.239.118:443,193.23.244.244,208.83.223.34:443,128.31.0.34:9131,194.109.206.212,212.112.245.170 +# +## Download the current consensus (only if DownloadRelayDescriptors is 1) +#DownloadCurrentConsensus 1 +# +## Download current votes (only if DownloadRelayDescriptors is 1) +#DownloadCurrentVotes 1 +# +## Download missing server descriptors (only if DownloadRelayDescriptors +## is 1) +#DownloadMissingServerDescriptors 1 +# +## Download missing extra-info descriptors (only if +## DownloadRelayDescriptors is 1) +#DownloadMissingExtraInfoDescriptors 1 +# +## Download all server descriptors from the directory authorities at most +## once a day (only if DownloadRelayDescriptors is 1) +#DownloadAllServerDescriptors 0 +# +## Download all extra-info descriptors from the directory authorities at +## most once a day (only if DownloadRelayDescriptors is 1) +#DownloadAllExtraInfoDescriptors 0 +# +## Compress relay descriptors downloads by adding .z to the URLs +#CompressRelayDescriptorDownloads 0 # ## Import bridge snapshots from disk, if available #ImportBridgeSnapshots 0 diff --git a/src/org/torproject/ernie/db/Configuration.java b/src/org/torproject/ernie/db/Configuration.java index ae24175..b4e6f8b 100644 --- a/src/org/torproject/ernie/db/Configuration.java +++ b/src/org/torproject/ernie/db/Configuration.java @@ -30,8 +30,17 @@ public class Configuration { private boolean importBridgeSnapshots = false; private String bridgeSnapshotsDirectory = "bridge-directories/"; private boolean downloadRelayDescriptors = false; - private List<String> downloadFromDirectoryAuthorities = Arrays.asList( - "86.59.21.38,194.109.206.212,80.190.246.100:8180".split(",")); + private List<String> downloadFromDirectoryAuthorities = Arrays.asList(( + "86.59.21.38,216.224.124.114:9030,213.115.239.118:443," + + "193.23.244.244,208.83.223.34:443,128.31.0.34:9131," + + "194.109.206.212,212.112.245.170").split(",")); + private boolean downloadCurrentConsensus = true; + private boolean downloadCurrentVotes = true; + private boolean downloadMissingServerDescriptors = true; + private boolean downloadMissingExtraInfoDescriptors = true; + private boolean downloadAllServerDescriptors = false; + private boolean downloadAllExtraInfoDescriptors = false; + private boolean compressRelayDescriptorDownloads; private boolean downloadGetTorStats = false; private String getTorStatsUrl = "http://gettor.torproject.org:8080/" + "~gettor/gettor_stats.txt"; @@ -117,6 +126,28 @@ public class Configuration { new URL("http://" + dir + "/"); this.downloadFromDirectoryAuthorities.add(dir); } + } else if (line.startsWith("DownloadCurrentConsensus")) { + this.downloadCurrentConsensus = Integer.parseInt( + line.split(" ")[1]) != 0; + } else if (line.startsWith("DownloadCurrentVotes")) { + this.downloadCurrentVotes = Integer.parseInt( + line.split(" ")[1]) != 0; + } else if (line.startsWith("DownloadMissingServerDescriptors")) { + this.downloadMissingServerDescriptors = Integer.parseInt( + line.split(" ")[1]) != 0; + } else if (line.startsWith( + "DownloadMissingExtraInfoDescriptors")) { + this.downloadMissingExtraInfoDescriptors = Integer.parseInt( + line.split(" ")[1]) != 0; + } else if (line.startsWith("DownloadAllServerDescriptors")) { + this.downloadAllServerDescriptors = Integer.parseInt( + line.split(" ")[1]) != 0; + } else if (line.startsWith("DownloadAllExtraInfoDescriptors")) { + this.downloadAllExtraInfoDescriptors = Integer.parseInt( + line.split(" ")[1]) != 0; + } else if (line.startsWith("CompressRelayDescriptorDownloads")) { + this.compressRelayDescriptorDownloads = Integer.parseInt( + line.split(" ")[1]) != 0; } else if (line.startsWith("DownloadGetTorStats")) { this.downloadGetTorStats = Integer.parseInt( line.split(" ")[1]) != 0; @@ -244,6 +275,27 @@ public class Configuration { public List<String> getDownloadFromDirectoryAuthorities() { return this.downloadFromDirectoryAuthorities; } + public boolean getDownloadCurrentConsensus() { + return this.downloadCurrentConsensus; + } + public boolean getDownloadCurrentVotes() { + return this.downloadCurrentVotes; + } + public boolean getDownloadMissingServerDescriptors() { + return this.downloadMissingServerDescriptors; + } + public boolean getDownloadMissingExtraInfoDescriptors() { + return this.downloadMissingExtraInfoDescriptors; + } + public boolean getDownloadAllServerDescriptors() { + return this.downloadAllServerDescriptors; + } + public boolean getDownloadAllExtraInfoDescriptors() { + return this.downloadAllExtraInfoDescriptors; + } + public boolean getCompressRelayDescriptorDownloads() { + return this.compressRelayDescriptorDownloads; + } public boolean getDownloadGetTorStats() { return this.downloadGetTorStats; } diff --git a/src/org/torproject/ernie/db/Main.java b/src/org/torproject/ernie/db/Main.java index 50c06bb..5a06900 100644 --- a/src/org/torproject/ernie/db/Main.java +++ b/src/org/torproject/ernie/db/Main.java @@ -49,13 +49,14 @@ public class Main { if (config.getDownloadRelayDescriptors()) { List<String> dirSources = config.getDownloadFromDirectoryAuthorities(); - boolean downloadCurrentConsensus = aw != null; - boolean downloadCurrentVotes = aw != null; - boolean downloadAllServerDescriptors = aw != null; - boolean downloadAllExtraInfos = aw != null; rdd = new RelayDescriptorDownloader(rdp, dirSources, - downloadCurrentConsensus, downloadCurrentVotes, - downloadAllServerDescriptors, downloadAllExtraInfos); + config.getDownloadCurrentConsensus(), + config.getDownloadCurrentVotes(), + config.getDownloadMissingServerDescriptors(), + config.getDownloadMissingExtraInfoDescriptors(), + config.getDownloadAllServerDescriptors(), + config.getDownloadAllExtraInfoDescriptors(), + config.getCompressRelayDescriptorDownloads()); rdp.setRelayDescriptorDownloader(rdd); } if (config.getImportCachedRelayDescriptors()) { @@ -77,7 +78,7 @@ public class Main { } } if (rdd != null) { - rdd.downloadMissingDescriptors(); + rdd.downloadDescriptors(); rdd.writeFile(); rdd = null; if (aw != null) { diff --git a/src/org/torproject/ernie/db/RelayDescriptorDownloader.java b/src/org/torproject/ernie/db/RelayDescriptorDownloader.java index 56d7e0d..ebbf761 100644 --- a/src/org/torproject/ernie/db/RelayDescriptorDownloader.java +++ b/src/org/torproject/ernie/db/RelayDescriptorDownloader.java @@ -7,27 +7,29 @@ import java.net.*; import java.text.*; import java.util.*; import java.util.logging.*; +import java.util.zip.*; import org.apache.commons.codec.digest.*; import org.apache.commons.codec.binary.*; /** - * Downloads missing relay descriptors from the directories via HTTP. + * Downloads relay descriptors from the directory authorities via HTTP. * Keeps a list of missing descriptors that gets updated by parse results - * from <code>RelayDescriptorParser</code>. Only descriptors on that - * missing list that we think might be available on the directories are - * downloaded. + * from <code>RelayDescriptorParser</code> and downloads all missing + * descriptors that have been published in the last 24 hours. Also + * downloads all server and extra-info descriptors known to a directory + * authority at most once a day. */ public class RelayDescriptorDownloader { /** * Text file containing the descriptors that we are missing and that we - * want to download in <code>downloadMissingDescriptors</code>. - * Lines are formatted as: + * want to download. Lines are formatted as: + * * - "consensus,<validafter>,<parsed>", * - "vote,<validafter>,<fingerprint>,<parsed>", * - "server,<published>,<relayid>,<descid>,<parsed>", or - * - "extra,<published>,<relayid>,<descid><parsed>". + * - "extra,<published>,<relayid>,<descid>,<parsed>". */ private File missingDescriptorsFile; @@ -35,25 +37,36 @@ public class RelayDescriptorDownloader { * Relay descriptors that we are missing and that we want to download * either in this execution or write to disk and try next time. Map keys * contain comma-separated values as in the missing descriptors files - * without the parsed column. Map values contain the parsed column. + * without the "parsed" column. Map values contain the "parsed" column. */ private SortedMap<String, String> missingDescriptors; /** - * <code>RelayDescriptorParser</code> that we will hand over the - * downloaded descriptors for parsing. + * Text file containing the IP addresses (and Dir ports if not 80) of + * directory authorities and when we last downloaded all server and + * extra-info descriptors from them, so that we can avoid downloading + * them too often. */ - private RelayDescriptorParser rdp; + private File lastDownloadedAllDescriptorsFile; + + /** + * Map of directory authorities and when we last downloaded all server + * and extra-info descriptors from them. Map keys are IP addresses (and + * Dir ports if not 80), map values are timestamps. + */ + private Map<String, String> lastDownloadedAllDescriptors; /** - * Directories that we will try to download missing descriptors from. + * <code>RelayDescriptorParser</code> that we will hand over the + * downloaded descriptors for parsing. */ - private List<String> dirSources; + private RelayDescriptorParser rdp; /** - * Number of descriptors requested by directory to be included in logs. + * Directory authorities that we will try to download missing + * descriptors from. */ - private Map<String, Integer> dirRequests; + private List<String> authorities; /** * Should we try to download the current consensus if we don't have it? @@ -66,22 +79,41 @@ public class RelayDescriptorDownloader { private boolean downloadCurrentVotes; /** - * Should we try to download all missing server descriptors that have + * Should we try to download missing server descriptors that have been + * published within the past 24 hours? + */ + private boolean downloadMissingServerDescriptors; + + /** + * Should we try to download missing extra-info descriptors that have * been published within the past 24 hours? */ + private boolean downloadMissingExtraInfos; + + /** + * Should we try to download all server descriptors from the authorities + * once every 24 hours? + */ private boolean downloadAllServerDescriptors; /** - * Should we try to download all missing extra-info descriptors that - * have been published within the past 24 hours? + * Should we try to download all extra-info descriptors from the + * authorities once every 24 hours? */ private boolean downloadAllExtraInfos; /** + * Should we download zlib-compressed versions of descriptors by adding + * ".z" to URLs? + */ + private boolean downloadCompressed; + + /** * valid-after time that we expect the current consensus and votes to * have, formatted "yyyy-MM-dd HH:mm:ss". We only expect to find - * consensuses and votes with this valid-after time on the directories. - * This time is initialized as the beginning of the current hour. + * consensuses and votes with this valid-after time on the directory + * authorities. This time is initialized as the beginning of the current + * hour. */ private String currentValidAfter; @@ -93,77 +125,115 @@ public class RelayDescriptorDownloader { private String descriptorCutOff; /** + * Cut-off time for downloading all server and extra-info descriptors + * from the directory authorities, formatted "yyyy-MM-dd HH:mm:ss". This + * time is initialized as the current system time minus 23:30 hours. + */ + private String downloadAllDescriptorsCutOff; + + /** + * Directory authorities that we plan to download all server and + * extra-info descriptors from in this execution. + */ + private Set<String> downloadAllDescriptorsFromAuthorities; + + /** * Current timestamp that is written to the missing list for descriptors - * that we parsed in this execution. This timestamp is most useful for - * debugging purposes when looking at the missing list. For execution it - * only matters whether the parsed time is "NA" or has some other value. + * that we parsed in this execution and for authorities that we + * downloaded all server and extra-info descriptors from. */ - private String parsedTimestampString; + private String currentTimestamp; /** * Logger for this class. */ private Logger logger; - private StringBuilder dumpStats; - private int newMissingConsensuses = 0, newMissingVotes = 0, + /** + * Number of descriptors requested by directory authority to be included + * in logs. + */ + private Map<String, Integer> requestsByAuthority; + + /** + * Counters for descriptors that we had on the missing list at the + * beginning of the execution, that we added to the missing list, + * that we requested, and that we successfully downloaded in this + * execution. + */ + private int oldMissingConsensuses = 0, oldMissingVotes = 0, + oldMissingServerDescriptors = 0, oldMissingExtraInfoDescriptors = 0, + newMissingConsensuses = 0, newMissingVotes = 0, newMissingServerDescriptors = 0, newMissingExtraInfoDescriptors = 0, - triedConsensuses = 0, triedVotes = 0, triedServerDescriptors = 0, - triedExtraInfoDescriptors = 0, downloadedConsensuses = 0, - downloadedVotes = 0, downloadedServerDescriptors = 0, - downloadedExtraInfoDescriptors = 0; + requestedConsensuses = 0, requestedVotes = 0, + requestedMissingServerDescriptors = 0, + requestedAllServerDescriptors = 0, + requestedMissingExtraInfoDescriptors = 0, + requestedAllExtraInfoDescriptors = 0, downloadedConsensuses = 0, + downloadedVotes = 0, downloadedMissingServerDescriptors = 0, + downloadedAllServerDescriptors = 0, + downloadedMissingExtraInfoDescriptors = 0, + downloadedAllExtraInfoDescriptors = 0; + /** * Initializes this class, including reading in missing descriptors from - * <code>stats/missing-relay-descriptors</code>. + * <code>stats/missing-relay-descriptors</code> and the times when we + * last downloaded all server and extra-info descriptors from + * <code>stats/last-downloaded-all-descriptors</code>. */ public RelayDescriptorDownloader(RelayDescriptorParser rdp, - List<String> dirSources, boolean downloadCurrentConsensus, - boolean downloadCurrentVotes, boolean downloadAllServerDescriptors, - boolean downloadAllExtraInfos) { + List<String> authorities, boolean downloadCurrentConsensus, + boolean downloadCurrentVotes, + boolean downloadMissingServerDescriptors, + boolean downloadMissingExtraInfos, + boolean downloadAllServerDescriptors, boolean downloadAllExtraInfos, + boolean downloadCompressed) { /* Memorize argument values. */ this.rdp = rdp; - this.dirSources = dirSources; + this.authorities = new ArrayList<String>(authorities); this.downloadCurrentConsensus = downloadCurrentConsensus; this.downloadCurrentVotes = downloadCurrentVotes; + this.downloadMissingServerDescriptors = + downloadMissingServerDescriptors; + this.downloadMissingExtraInfos = downloadMissingExtraInfos; this.downloadAllServerDescriptors = downloadAllServerDescriptors; this.downloadAllExtraInfos = downloadAllExtraInfos; + this.downloadCompressed = downloadCompressed; + + /* Shuffle list of authorities for better load balancing over time. */ + Collections.shuffle(this.authorities); /* Initialize logger. */ - this.logger = Logger.getLogger(RelayDescriptorParser.class.getName()); + this.logger = Logger.getLogger( + RelayDescriptorDownloader.class.getName()); - /* Prepare cut-off times and timestamp for missing descriptors - * list. */ + /* Prepare cut-off times and timestamp for the missing descriptors + * list and the list of authorities to download all server and + * extra-info descriptors from. */ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); format.setTimeZone(TimeZone.getTimeZone("UTC")); long now = System.currentTimeMillis(); this.currentValidAfter = format.format((now / (60L * 60L * 1000L)) * (60L * 60L * 1000L)); this.descriptorCutOff = format.format(now - 24L * 60L * 60L * 1000L); - this.parsedTimestampString = format.format(now); - - /* Initialize missing list and put current consensus on it if we want - * it. */ - this.missingDescriptors = new TreeMap<String, String>(); - if (this.downloadCurrentConsensus) { - this.missingDescriptors.put("consensus," + this.currentValidAfter, - "NA"); - } + this.currentTimestamp = format.format(now); + this.downloadAllDescriptorsCutOff = format.format(now + - 23L * 60L * 60L * 1000L - 30L * 60L * 1000L); /* Read list of missing descriptors from disk and memorize those that * we are interested in and that are likely to be found on the - * directory servers. */ + * directory authorities. */ + this.missingDescriptors = new TreeMap<String, String>(); this.missingDescriptorsFile = new File( "stats/missing-relay-descriptors"); - int missingConsensuses = 0, missingVotes = 0, - missingServerDescriptors = 0, missingExtraInfoDescriptors = 0; if (this.missingDescriptorsFile.exists()) { try { this.logger.fine("Reading file " + this.missingDescriptorsFile.getAbsolutePath() + "..."); BufferedReader br = new BufferedReader(new FileReader( this.missingDescriptorsFile)); - String line = null; + String line; while ((line = br.readLine()) != null) { if (line.split(",").length > 2) { String published = line.split(",")[1]; @@ -172,17 +242,17 @@ public class RelayDescriptorDownloader { this.currentValidAfter.equals(published)) || ((line.startsWith("server,") || line.startsWith("extra,")) && - this.descriptorCutOff.compareTo(published) <= 0)) { + this.descriptorCutOff.compareTo(published) < 0)) { if (!line.endsWith("NA")) { /* Not missing. */ } else if (line.startsWith("consensus,")) { - missingConsensuses++; + oldMissingConsensuses++; } else if (line.startsWith("vote,")) { - missingVotes++; + oldMissingVotes++; } else if (line.startsWith("server,")) { - missingServerDescriptors++; + oldMissingServerDescriptors++; } else if (line.startsWith("extra,")) { - missingExtraInfoDescriptors++; + oldMissingExtraInfoDescriptors++; } int separateAt = line.lastIndexOf(","); this.missingDescriptors.put(line.substring(0, @@ -205,86 +275,131 @@ public class RelayDescriptorDownloader { } } - dumpStats = new StringBuilder(); - dumpStats.append("Finished downloading relay descriptors from the " - + "directory authorities:\nAt the beginning of this execution, " - + "we were missing " + missingConsensuses + " consensus(es), " - + missingVotes + " vote(s), " + missingServerDescriptors - + " server descriptor(s), and " + missingExtraInfoDescriptors - + " extra-info descriptor(s).\n"); - - dirRequests = new HashMap<String, Integer>(); - for (String dirSource : dirSources) { - dirRequests.put(dirSource, 0); + /* Put the current consensus on the missing list, unless we already + * have it. */ + String consensusKey = "consensus," + this.currentValidAfter; + if (!this.missingDescriptors.containsKey(consensusKey)) { + this.missingDescriptors.put(consensusKey, "NA"); + this.newMissingConsensuses++; + } + + /* Read list of directory authorities and when we last downloaded all + * server and extra-info descriptors from them. */ + this.lastDownloadedAllDescriptors = new HashMap<String, String>(); + this.lastDownloadedAllDescriptorsFile = new File( + "stats/last-downloaded-all-descriptors"); + if (this.lastDownloadedAllDescriptorsFile.exists()) { + try { + this.logger.fine("Reading file " + + this.lastDownloadedAllDescriptorsFile.getAbsolutePath() + + "..."); + BufferedReader br = new BufferedReader(new FileReader( + this.lastDownloadedAllDescriptorsFile)); + String line; + while ((line = br.readLine()) != null) { + if (line.split(",").length != 2) { + this.logger.fine("Invalid line '" + line + "' in " + + this.lastDownloadedAllDescriptorsFile.getAbsolutePath() + + ". Ignoring."); + } else { + String[] parts = line.split(","); + String authority = parts[0]; + String lastDownloaded = parts[1]; + this.lastDownloadedAllDescriptors.put(authority, + lastDownloaded); + } + } + br.close(); + this.logger.fine("Finished reading file " + + this.lastDownloadedAllDescriptorsFile.getAbsolutePath() + + "."); + } catch (IOException e) { + this.logger.log(Level.WARNING, "Failed to read file " + + this.lastDownloadedAllDescriptorsFile.getAbsolutePath() + + "! This means that we might download all server and " + + "extra-info descriptors more often than we should.", e); + } + } + + /* Make a list of directory authorities that we want to download all + * server and extra-info descriptors from. */ + this.downloadAllDescriptorsFromAuthorities = new HashSet<String>(); + for (String authority : this.authorities) { + if (!this.lastDownloadedAllDescriptors.containsKey(authority) || + this.lastDownloadedAllDescriptors.get(authority).compareTo( + this.downloadAllDescriptorsCutOff) < 0) { + this.downloadAllDescriptorsFromAuthorities.add(authority); + } + } + + /* Prepare statistics on this execution. */ + this.requestsByAuthority = new HashMap<String, Integer>(); + for (String authority : this.authorities) { + this.requestsByAuthority.put(authority, 0); } } /** * We have parsed a consensus. Take this consensus off the missing list - * and add the votes created by the given <code>dirSources</code> and - * the <code>serverDescriptors</code> in the format + * and add the votes created by the given <code>authorities</code> and + * the <code>serverDescriptors</code> which are in the format * "<published>,<relayid>,<descid>" to that list. */ public void haveParsedConsensus(String validAfter, - Set<String> dirSources, Set<String> serverDescriptors) { + Set<String> authorities, Set<String> serverDescriptors) { /* Mark consensus as parsed. */ if (this.currentValidAfter.equals(validAfter)) { String consensusKey = "consensus," + validAfter; - this.missingDescriptors.put(consensusKey, - this.parsedTimestampString); + this.missingDescriptors.put(consensusKey, this.currentTimestamp); /* Add votes to missing list. */ - if (this.downloadCurrentVotes) { - for (String dirSource : dirSources) { - String voteKey = "vote," + validAfter + "," + dirSource; - if (!this.missingDescriptors.containsKey(voteKey)) { - this.missingDescriptors.put(voteKey, "NA"); - this.newMissingVotes++; - } + for (String authority : authorities) { + String voteKey = "vote," + validAfter + "," + authority; + if (!this.missingDescriptors.containsKey(voteKey)) { + this.missingDescriptors.put(voteKey, "NA"); + this.newMissingVotes++; } } } /* Add server descriptors to missing list. */ - if (this.downloadAllServerDescriptors) { - for (String serverDescriptor : serverDescriptors) { - String published = serverDescriptor.split(",")[0]; - if (this.descriptorCutOff.compareTo(published) <= 0) { - String serverDescriptorKey = "server," + serverDescriptor; - if (!this.missingDescriptors.containsKey( - serverDescriptorKey)) { - this.missingDescriptors.put(serverDescriptorKey, "NA"); - this.newMissingServerDescriptors++; - } + for (String serverDescriptor : serverDescriptors) { + String published = serverDescriptor.split(",")[0]; + if (this.descriptorCutOff.compareTo(published) < 0) { + String serverDescriptorKey = "server," + serverDescriptor; + if (!this.missingDescriptors.containsKey( + serverDescriptorKey)) { + this.missingDescriptors.put(serverDescriptorKey, "NA"); + this.newMissingServerDescriptors++; } } } } /** - * We have parsed a vote. Take this vote off the missing list. + * We have parsed a vote. Take this vote off the missing list and add + * the <code>serverDescriptors</code> which are in the format + * "<published>,<relayid>,<descid>" to that list. */ public void haveParsedVote(String validAfter, String fingerprint, Set<String> serverDescriptors) { - /* Mark consensus as parsed. */ + /* Mark vote as parsed. */ if (this.currentValidAfter.equals(validAfter)) { String voteKey = "vote," + validAfter + "," + fingerprint; - this.missingDescriptors.put(voteKey, this.parsedTimestampString); + this.missingDescriptors.put(voteKey, this.currentTimestamp); } /* Add server descriptors to missing list. */ - if (this.downloadAllServerDescriptors) { - for (String serverDescriptor : serverDescriptors) { - String published = serverDescriptor.split(",")[0]; - if (this.descriptorCutOff.compareTo(published) < 0) { - String serverDescriptorKey = "server," + serverDescriptor; - if (!this.missingDescriptors.containsKey( - serverDescriptorKey)) { - this.missingDescriptors.put(serverDescriptorKey, "NA"); - this.newMissingServerDescriptors++; - } + for (String serverDescriptor : serverDescriptors) { + String published = serverDescriptor.split(",")[0]; + if (this.descriptorCutOff.compareTo(published) < 0) { + String serverDescriptorKey = "server," + serverDescriptor; + if (!this.missingDescriptors.containsKey( + serverDescriptorKey)) { + this.missingDescriptors.put(serverDescriptorKey, "NA"); + this.newMissingServerDescriptors++; } } } @@ -300,14 +415,14 @@ public class RelayDescriptorDownloader { String extraInfoDigest) { /* Mark server descriptor as parsed. */ - if (this.descriptorCutOff.compareTo(published) <= 0) { + if (this.descriptorCutOff.compareTo(published) < 0) { String serverDescriptorKey = "server," + published + "," + relayIdentity + "," + serverDescriptorDigest; this.missingDescriptors.put(serverDescriptorKey, - this.parsedTimestampString); + this.currentTimestamp); /* Add extra-info descriptor to missing list. */ - if (extraInfoDigest != null && this.downloadAllExtraInfos) { + if (extraInfoDigest != null) { String extraInfoKey = "extra," + published + "," + relayIdentity + "," + extraInfoDigest; if (!this.missingDescriptors.containsKey(extraInfoKey)) { @@ -324,253 +439,261 @@ public class RelayDescriptorDownloader { */ public void haveParsedExtraInfoDescriptor(String published, String relayIdentity, String extraInfoDigest) { - if (this.descriptorCutOff.compareTo(published) <= 0) { + if (this.descriptorCutOff.compareTo(published) < 0) { String extraInfoKey = "extra," + published + "," + relayIdentity + "," + extraInfoDigest; - this.missingDescriptors.put(extraInfoKey, - this.parsedTimestampString); + this.missingDescriptors.put(extraInfoKey, this.currentTimestamp); } } /** * Downloads missing descriptors that we think might still be available - * on the directories. + * on the directory authorities as well as all server and extra-info + * descriptors once per day. */ - public void downloadMissingDescriptors() { - - /* Update cut-off times to reflect that execution so far might have - * taken a few minutes and that some descriptors aren't available on - * the directories anymore. */ - SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - format.setTimeZone(TimeZone.getTimeZone("UTC")); - long now = System.currentTimeMillis(); - this.currentValidAfter = format.format((now / (60L * 60L * 1000L)) * - (60L * 60L * 1000L)); - this.descriptorCutOff = format.format(now - 24L * 60L * 60L * 1000L); + public void downloadDescriptors() { + + /* Download descriptors from authorities which are in random order, so + * that we distribute the load somewhat fairly over time. */ + for (String authority : authorities) { + + /* Make all requests to an authority in a single try block. If + * something goes wrong with this authority, we give up on all + * downloads and continue with the next authority. */ + /* TODO Some authorities provide very little bandwidth and could + * slow down the entire download process. Ponder adding a timeout of + * 3 or 5 minutes per authority to avoid getting in the way of the + * next execution. */ + try { - /* Remember which directories remain as source for downloading - * descriptors. */ - List<String> remainingDirSources = - new ArrayList<String>(this.dirSources); - - /* URLs of descriptors we want to download. */ - SortedSet<String> urls = new TreeSet<String>(); - - /* Complete URLs of authorities and descriptors we have downloaded or - * tried to download. Ensures that we're not attempting to download - * the same thing from an authority more than once. There are edge - * cases when an authority returns a valid response to something we - * asked for, but which is not what we wanted (e.g. old consensus). */ - SortedSet<String> downloaded = new TreeSet<String>(); - - /* We might need more than one iteration for downloading descriptors, - * because we might learn about descriptors while parsing those that - * we got. In every iteration, compile a new list of URLs, remove - * those that we tried before, and download the remaining ones. Stop - * when there are no new URLs anymore. */ - while (true) { - - /* Compile list of URLs to download in this iteration. */ - urls.clear(); - for (Map.Entry<String, String> e : - this.missingDescriptors.entrySet()) { - if (e.getValue().equals("NA")) { - String[] parts = e.getKey().split(","); - if (parts[0].equals("consensus") && - this.downloadCurrentConsensus && - this.currentValidAfter.equals(parts[1])) { - urls.add("/tor/status-vote/current/consensus"); - } else if (parts[0].equals("vote") && - this.downloadCurrentVotes && - this.currentValidAfter.equals(parts[1])) { - urls.add("/tor/status-vote/current/" + parts[2]); - } else if (parts[0].equals("server") && - this.downloadAllServerDescriptors && - this.descriptorCutOff.compareTo(parts[1]) <= 0) { - urls.add("/tor/server/d/" + parts[3]); - } else if (parts[0].equals("extra") && - this.downloadAllExtraInfos && - this.descriptorCutOff.compareTo(parts[1]) <= 0) { - urls.add("/tor/extra/d/" + parts[3]); + /* Start with downloading the current consensus, unless we already + * have it. */ + if (downloadCurrentConsensus) { + String consensusKey = "consensus," + this.currentValidAfter; + if (this.missingDescriptors.containsKey(consensusKey) && + this.missingDescriptors.get(consensusKey).equals("NA")) { + this.requestedConsensuses++; + this.downloadedConsensuses += + this.downloadResourceFromAuthority(authority, + "/tor/status-vote/current/consensus"); } } - } - /* Stop if we don't have (new) URLs to download. */ - if (urls.isEmpty()) { - break; - } - - for (String url : urls) { - if (url.endsWith("consensus")) { - this.triedConsensuses++; - } else if (url.contains("status-vote")) { - this.triedVotes++; - } else if (url.contains("server")) { - this.triedServerDescriptors++; - } else if (url.contains("extra")) { - this.triedExtraInfoDescriptors++; - } - } - - /* Log what we're downloading. */ - StringBuilder sb = new StringBuilder("Downloading " + urls.size() - + " descriptors:"); - for (String url : urls) { - sb.append("\n" + url); - } - this.logger.fine(sb.toString()); - - /* We are trying to download these descriptors from each directory - * source one after the other until we got it from one. For each - * directory source we are removing the URLs from urls and putting - * those the we want to retry into retryUrls. Once we are done, we - * move the URLs back to urls and try the next directory source. */ - SortedSet<String> currentDirSources = - new TreeSet<String>(remainingDirSources); - SortedSet<String> retryUrls = new TreeSet<String>(); - int numDownloaded = 0; - while (!currentDirSources.isEmpty() && !urls.isEmpty()) { - String authority = currentDirSources.first(); - String url = urls.first(); - urls.remove(url); - Set<String> requestedUrls = new HashSet<String>(); - requestedUrls.add(url); - if (url.contains("server")) { - StringBuilder combinedUrl = new StringBuilder(url); - int descriptors = 1; - while (descriptors++ <= 96 && !urls.isEmpty() && - urls.first().contains("server")) { - url = urls.first(); - urls.remove(url); - requestedUrls.add(url); - combinedUrl.append("+" + url.substring("/tor/server/d/". - length())); + /* Next, try to download current votes that we're missing. */ + if (downloadCurrentVotes) { + String voteKeyPrefix = "vote," + this.currentValidAfter; + SortedSet<String> fingerprints = new TreeSet<String>(); + for (Map.Entry<String, String> e : + this.missingDescriptors.entrySet()) { + if (e.getValue().equals("NA") && + e.getKey().startsWith(voteKeyPrefix)) { + String fingerprint = e.getKey().split(",")[2]; + fingerprints.add(fingerprint); + } } - url = combinedUrl.toString(); - } else if (url.contains("extra")) { - StringBuilder combinedUrl = new StringBuilder(url); - int descriptors = 1; - while (descriptors++ <= 96 && !urls.isEmpty() && - urls.first().contains("extra")) { - url = urls.first(); - urls.remove(url); - requestedUrls.add(url); - combinedUrl.append("+" + url.substring("/tor/extra/d/". - length())); + for (String fingerprint : fingerprints) { + this.requestedVotes++; + this.downloadedVotes += + this.downloadResourceFromAuthority(authority, + "/tor/status-vote/current/" + fingerprint); } - url = combinedUrl.toString(); } - String fullUrl = "http://" + authority + url; - byte[] allData = null; - if (!downloaded.contains(fullUrl)) { - downloaded.add(fullUrl); - numDownloaded++; - this.dirRequests.put(authority, dirRequests.get(authority) + 1); - try { - URL u = new URL(fullUrl); - HttpURLConnection huc = - (HttpURLConnection) u.openConnection(); - huc.setRequestMethod("GET"); - huc.connect(); - int response = huc.getResponseCode(); - logger.fine("Downloading http://" + authority + url + " -> " - + response); - if (response == 200) { - BufferedInputStream in = new BufferedInputStream( - huc.getInputStream()); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - int len; - byte[] data = new byte[1024]; - while ((len = in.read(data, 0, 1024)) >= 0) { - baos.write(data, 0, len); - } - in.close(); - allData = baos.toByteArray(); - } - } catch (IOException e) { - remainingDirSources.remove(authority); - currentDirSources.remove(authority); - if (!remainingDirSources.isEmpty()) { - logger.log(Level.FINE, "Failed downloading from " - + authority + "!", e); + + /* Download either all server and extra-info descriptors or only + * those that we're missing. Start with server descriptors, then + * request extra-info descriptors. */ + List<String> types = new ArrayList<String>(Arrays.asList( + "server,extra".split(","))); + for (String type : types) { + + /* Download all server or extra-info descriptors from this + * authority if we haven't done so for 24 hours and if we're + * configured to do so. */ + /* TODO Distribute downloads of all descriptors over the day for + * different authorities. Maybe limit the number of these + * downloads to 1 or 2 per execution. */ + if (this.downloadAllDescriptorsFromAuthorities.contains( + authority) && ((type.equals("server") && + this.downloadAllServerDescriptors) || + (type.equals("extra") && this.downloadAllExtraInfos))) { + int downloadedAllDescriptors = + this.downloadResourceFromAuthority(authority, "/tor/" + + type + "/all"); + if (type.equals("server")) { + this.requestedAllServerDescriptors++; + this.downloadedAllServerDescriptors += + downloadedAllDescriptors; } else { - logger.log(Level.WARNING, "Failed downloading from " - + authority + "! We have no authorities left to download " - + "from!", e); + this.requestedAllExtraInfoDescriptors++; + this.downloadedAllExtraInfoDescriptors += + downloadedAllDescriptors; } - } - } - if (allData != null) { - if (url.endsWith("consensus")) { - this.rdp.parse(allData); - this.downloadedConsensuses++; - } else if (url.contains("status-vote")) { - this.rdp.parse(allData); - this.downloadedVotes++; - } else if (url.contains("server") || - url.contains("extra")) { - String ascii = null; - try { - ascii = new String(allData, "US-ASCII"); - } catch (UnsupportedEncodingException e) { - } - int start = -1, sig = -1, end = -1; - String startToken = url.contains("server") ? - "router " : "extra-info "; - String sigToken = "\nrouter-signature\n"; - String endToken = "\n-----END SIGNATURE-----\n"; - while (end < ascii.length()) { - start = ascii.indexOf(startToken, end); - if (start < 0) { - break; - } - sig = ascii.indexOf(sigToken, start); - if (sig < 0) { - break; + + /* Download missing server or extra-info descriptors if we're + * configured to do so. */ + } else if ((type.equals("server") && + this.downloadMissingServerDescriptors) || + (type.equals("extra") && this.downloadMissingExtraInfos)) { + + /* Go through the list of missing descriptors of this type + * and combine the descriptor identifiers to a URL of up to + * 96 descriptors that we can download at once. */ + SortedSet<String> descriptorIdentifiers = + new TreeSet<String>(); + for (Map.Entry<String, String> e : + this.missingDescriptors.entrySet()) { + if (e.getValue().equals("NA") && + e.getKey().startsWith(type + ",") && + this.descriptorCutOff.compareTo( + e.getKey().split(",")[1]) < 0) { + String descriptorIdentifier = e.getKey().split(",")[3]; + descriptorIdentifiers.add(descriptorIdentifier); } - sig += sigToken.length(); - end = ascii.indexOf(endToken, sig); - if (end < 0) { - break; + } + StringBuilder combinedResource = null; + int descriptorsInCombinedResource = 0, + requestedDescriptors = 0, downloadedDescriptors = 0; + for (String descriptorIdentifier : descriptorIdentifiers) { + if (descriptorsInCombinedResource >= 96) { + requestedDescriptors += descriptorsInCombinedResource; + downloadedDescriptors += + this.downloadResourceFromAuthority(authority, + combinedResource.toString()); + combinedResource = null; + descriptorsInCombinedResource = 0; } - end += endToken.length(); - byte[] descBytes = new byte[end - start]; - System.arraycopy(allData, start, descBytes, 0, end - start); - String digest = Hex.encodeHexString(DigestUtils.sha( - descBytes)); - this.rdp.parse(descBytes); - if (url.contains("server")) { - this.downloadedServerDescriptors++; - requestedUrls.remove("/tor/server/d/" + digest); + if (descriptorsInCombinedResource == 0) { + combinedResource = new StringBuilder("/tor/" + type + + "/d/" + descriptorIdentifier); } else { - this.downloadedExtraInfoDescriptors++; - requestedUrls.remove("/tor/extra/d/" + digest); + combinedResource.append("+" + descriptorIdentifier); } + descriptorsInCombinedResource++; + } + if (descriptorsInCombinedResource > 0) { + requestedDescriptors += descriptorsInCombinedResource; + downloadedDescriptors += + this.downloadResourceFromAuthority(authority, + combinedResource.toString()); + } + if (type.equals("server")) { + this.requestedMissingServerDescriptors += + requestedDescriptors; + this.downloadedMissingServerDescriptors += + downloadedDescriptors; + } else { + this.requestedMissingExtraInfoDescriptors += + requestedDescriptors; + this.downloadedMissingExtraInfoDescriptors += + downloadedDescriptors; } } } - if (requestedUrls.isEmpty()) { - retryUrls.addAll(requestedUrls); - logger.fine("Retrying " + requestedUrls.size() + " URLs for " - + "which we didn't receive descriptors."); - } - if (urls.isEmpty()) { - currentDirSources.remove(authority); - urls.addAll(retryUrls); - retryUrls.clear(); - } + + /* If a download failed, stop requesting descriptors from this + * authority and move on to the next. */ + } catch (IOException e) { + logger.log(Level.FINE, "Failed downloading from " + authority + + "!", e); } + } + } - /* If we haven't downloaded a single descriptor in this iteration, - * we cannot have learned something new, so we're done. */ - if (numDownloaded < 1) { - break; + /** + * Attempts to download one or more descriptors identified by a resource + * string from a directory authority and passes the returned + * descriptor(s) to the <code>RelayDescriptorParser</code> upon success. + * Returns the number of descriptors contained in the reply. Throws an + * <code>IOException</code> if something goes wrong while downloading. + */ + private int downloadResourceFromAuthority(String authority, + String resource) throws IOException { + byte[] allData = null; + this.requestsByAuthority.put(authority, + this.requestsByAuthority.get(authority) + 1); + /* TODO Disable compressed downloads for extra-info descriptors, + * because zlib decompression doesn't work correctly. Figure out why + * this is and fix it. */ + String fullUrl = "http://" + authority + resource + + (this.downloadCompressed && !resource.startsWith("/tor/extra/") + ? ".z" : ""); + URL u = new URL(fullUrl); + HttpURLConnection huc = (HttpURLConnection) u.openConnection(); + huc.setRequestMethod("GET"); + huc.connect(); + int response = huc.getResponseCode(); + if (response == 200) { + BufferedInputStream in = downloadCompressed + ? new BufferedInputStream(new InflaterInputStream( + huc.getInputStream())) + : new BufferedInputStream(huc.getInputStream()); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + int len; + byte[] data = new byte[1024]; + while ((len = in.read(data, 0, 1024)) >= 0) { + baos.write(data, 0, len); + } + in.close(); + allData = baos.toByteArray(); + } + logger.fine("Downloaded " + fullUrl + " -> " + response + " (" + + (allData == null ? 0 : allData.length) + " bytes)"); + int receivedDescriptors = 0; + if (allData != null) { + if (resource.startsWith("/tor/status-vote/current/")) { + this.rdp.parse(allData); + receivedDescriptors = 1; + } else if (resource.startsWith("/tor/server/") || + resource.startsWith("/tor/extra/")) { + if (resource.equals("/tor/server/all")) { + this.lastDownloadedAllDescriptors.put(authority, + this.currentTimestamp); + } + String ascii = null; + try { + ascii = new String(allData, "US-ASCII"); + } catch (UnsupportedEncodingException e) { + /* No way that US-ASCII is not supported. */ + } + int start = -1, sig = -1, end = -1; + String startToken = resource.startsWith("/tor/server/") ? + "router " : "extra-info "; + String sigToken = "\nrouter-signature\n"; + String endToken = "\n-----END SIGNATURE-----\n"; + while (end < ascii.length()) { + start = ascii.indexOf(startToken, end); + if (start < 0) { + break; + } + sig = ascii.indexOf(sigToken, start); + if (sig < 0) { + break; + } + sig += sigToken.length(); + end = ascii.indexOf(endToken, sig); + if (end < 0) { + break; + } + end += endToken.length(); + byte[] descBytes = new byte[end - start]; + System.arraycopy(allData, start, descBytes, 0, end - start); + String digest = Hex.encodeHexString(DigestUtils.sha( + descBytes)); + this.rdp.parse(descBytes); + receivedDescriptors++; + } } } + return receivedDescriptors; } + /** + * Writes status files to disk and logs statistics about downloading + * relay descriptors in this execution. + */ public void writeFile() { + + /* Write missing descriptors file to disk. */ int missingConsensuses = 0, missingVotes = 0, missingServerDescriptors = 0, missingExtraInfoDescriptors = 0; try { @@ -581,8 +704,8 @@ public class RelayDescriptorDownloader { this.missingDescriptorsFile)); for (Map.Entry<String, String> e : this.missingDescriptors.entrySet()) { - String key = e.getKey(); - if (!e.getValue().equals("NA")) { + String key = e.getKey(), value = e.getValue(); + if (!value.equals("NA")) { /* Not missing. */ } else if (key.startsWith("consensus,")) { missingConsensuses++; @@ -593,7 +716,7 @@ public class RelayDescriptorDownloader { } else if (key.startsWith("extra,")) { missingExtraInfoDescriptors++; } - bw.write(e.getKey() + "," + e.getValue() + "\n"); + bw.write(key + "," + value + "\n"); } bw.close(); this.logger.fine("Finished writing file " @@ -603,35 +726,79 @@ public class RelayDescriptorDownloader { + this.missingDescriptorsFile.getAbsolutePath() + "!", e); } - dumpStats.append("During this execution, we added " + /* Write text file containing the directory authorities and when we + * last downloaded all server and extra-info descriptors from them to + * disk. */ + try { + this.logger.fine("Writing file " + + this.lastDownloadedAllDescriptorsFile.getAbsolutePath() + + "..."); + this.lastDownloadedAllDescriptorsFile.getParentFile().mkdirs(); + BufferedWriter bw = new BufferedWriter(new FileWriter( + this.lastDownloadedAllDescriptorsFile)); + for (Map.Entry<String, String> e : + this.lastDownloadedAllDescriptors.entrySet()) { + String authority = e.getKey(); + String lastDownloaded = e.getValue(); + bw.write(authority + "," + lastDownloaded + "\n"); + } + bw.close(); + this.logger.fine("Finished writing file " + + this.lastDownloadedAllDescriptorsFile.getAbsolutePath() + + "."); + } catch (IOException e) { + this.logger.log(Level.WARNING, "Failed writing " + + this.lastDownloadedAllDescriptorsFile.getAbsolutePath() + "!", + e); + } + + /* Log statistics about this execution. */ + this.logger.info("Finished downloading relay descriptors from the " + + "directory authorities."); + this.logger.info("At the beginning of this execution, we were " + + "missing " + oldMissingConsensuses + " consensus(es), " + + oldMissingVotes + " vote(s), " + oldMissingServerDescriptors + + " server descriptor(s), and " + oldMissingExtraInfoDescriptors + + " extra-info descriptor(s)."); + this.logger.info("During this execution, we added " + this.newMissingConsensuses + " consensus(es), " + this.newMissingVotes + " vote(s), " + this.newMissingServerDescriptors + " server descriptor(s), and " + this.newMissingExtraInfoDescriptors + " extra-info " - + "descriptor(s) to the missing list.\n"); - dumpStats.append("We attempted to download " + this.triedConsensuses - + " consensus(es), " + this.triedVotes + " vote(s), " - + this.triedServerDescriptors + " server descriptor(s), and " - + this.triedExtraInfoDescriptors + " extra-info descriptor(s) " - + "from the directory authorities.\n"); - dumpStats.append("Requests were sent to these directory " - + "authorities:"); - for (String dirSource : this.dirSources) { - dumpStats.append(" " + dirSource + "=" - + this.dirRequests.get(dirSource)); + + "descriptor(s) to the missing list, some of which we also " + + "requested and removed from the list again."); + this.logger.info("We requested " + this.requestedConsensuses + + " consensus(es), " + this.requestedVotes + " vote(s), " + + this.requestedMissingServerDescriptors + " missing server " + + "descriptor(s), " + this.requestedAllServerDescriptors + + " times all server descriptors, " + + this.requestedMissingExtraInfoDescriptors + " missing " + + "extra-info descriptor(s), and " + + this.requestedAllExtraInfoDescriptors + " times all extra-info " + + "descriptors from the directory authorities."); + StringBuilder sb = new StringBuilder(); + for (String authority : this.authorities) { + sb.append(" " + authority + "=" + + this.requestsByAuthority.get(authority)); } - dumpStats.append("\nWe successfully downloaded " + this.logger.info("We sent these numbers of requests to the directory " + + "authorities:" + sb.toString()); + this.logger.info("We successfully downloaded " + this.downloadedConsensuses + " consensus(es), " + this.downloadedVotes + " vote(s), " - + this.downloadedServerDescriptors + " server descriptor(s), and " - + this.downloadedExtraInfoDescriptors + " extra-info " - + "descriptor(s) from the directory authorities.\n"); - dumpStats.append("At the end of this execution, " - + "we are missing " + missingConsensuses + " consensus(es), " - + missingVotes + " vote(s), " + missingServerDescriptors - + " server descriptor(s), and " + missingExtraInfoDescriptors + + this.downloadedMissingServerDescriptors + " missing server " + + "descriptor(s), " + this.downloadedAllServerDescriptors + + " server descriptor(s) when downloading all descriptors, " + + this.downloadedMissingExtraInfoDescriptors + " missing " + + "extra-info descriptor(s) and " + + this.downloadedAllExtraInfoDescriptors + " extra-info " + + "descriptor(s) when downloading all descriptors."); + this.logger.info("At the end of this execution, we are missing " + + missingConsensuses + " consensus(es), " + missingVotes + + " vote(s), " + missingServerDescriptors + " server " + + "descriptor(s), and " + missingExtraInfoDescriptors + " extra-info descriptor(s), some of which we may try in the next " + "execution."); - this.logger.info(dumpStats.toString()); } } +

1 0

r24548: {arm} Using both the IP and port as sort attibutes when sorting by (arm/trunk/src/interface/connections)
by Damian Johnson 03 Apr '11

03 Apr '11

Author: atagar Date: 2011-04-03 06:36:26 +0000 (Sun, 03 Apr 2011) New Revision: 24548 Modified: arm/trunk/src/interface/connections/entries.py Log: Using both the IP and port as sort attibutes when sorting by listed IP entries. Modified: arm/trunk/src/interface/connections/entries.py =================================================================== --- arm/trunk/src/interface/connections/entries.py 2011-04-03 06:28:09 UTC (rev 24547) +++ arm/trunk/src/interface/connections/entries.py 2011-04-03 06:36:26 UTC (rev 24548) @@ -18,6 +18,9 @@ SortAttr.FINGERPRINT: "cyan", SortAttr.NICKNAME: "cyan", SortAttr.COUNTRY: "blue"} +# maximum number of ports a system can have +PORT_COUNT = 65536 + class ConnectionPanelEntry: """ Common parent for connection panel entries. This consists of a list of lines @@ -73,7 +76,10 @@ if attr == SortAttr.LISTING: if listingType == ListingType.IP_ADDRESS: - return self.getSortValue(SortAttr.IP_ADDRESS, listingType) + # uses the IP address as the primary value, and port as secondary + sortValue = self.getSortValue(SortAttr.IP_ADDRESS, listingType) * PORT_COUNT + sortValue += self.getSortValue(SortAttr.PORT, listingType) + return sortValue elif listingType == ListingType.HOSTNAME: return self.getSortValue(SortAttr.HOSTNAME, listingType) elif listingType == ListingType.FINGERPRINT:

1 0

r24547: {arm} Swapping the external and internal src address positions for (arm/trunk/src/interface/connections)
by Damian Johnson 03 Apr '11

03 Apr '11

Author: atagar Date: 2011-04-03 06:28:09 +0000 (Sun, 03 Apr 2011) New Revision: 24547 Modified: arm/trunk/src/interface/connections/connEntry.py Log: Swapping the external and internal src address positions for expanded entries (it was "<foreign> --> <internal> --> <external>" which is, of course, wrong) Modified: arm/trunk/src/interface/connections/connEntry.py =================================================================== --- arm/trunk/src/interface/connections/connEntry.py 2011-04-03 06:15:38 UTC (rev 24546) +++ arm/trunk/src/interface/connections/connEntry.py 2011-04-03 06:28:09 UTC (rev 24547) @@ -597,7 +597,14 @@ if addrDiffer and isExpansionType and isExpandedAddrVisible and self.includeExpandedIpAddr and CONFIG["features.connection.showColumn.expandedIp"]: # include the internal address in the src (extra 28 characters) internalAddress = self.local.getIpAddr() + localPort - src = "%-21s --> %s" % (internalAddress, src) + + # If this is an inbound connection then reverse ordering so it's: + # <foreign> --> <external> --> <internal> + # when the src and dst are swapped later + + if myType == Category.INBOUND: src = "%-21s --> %s" % (src, internalAddress) + else: src = "%-21s --> %s" % (internalAddress, src) + usedSpace += 28 etc = self.getEtcContent(width - usedSpace, listingType)

1 0

r24546: {arm} Rearanging config options to fill temporary gap in the help (arm/trunk/src/interface)
by Damian Johnson 03 Apr '11

03 Apr '11

Author: atagar Date: 2011-04-03 06:15:38 +0000 (Sun, 03 Apr 2011) New Revision: 24546 Modified: arm/trunk/src/interface/controller.py Log: Rearanging config options to fill temporary gap in the help dialog (the descriptor popup won't be available until 1.4.3). Modified: arm/trunk/src/interface/controller.py =================================================================== --- arm/trunk/src/interface/controller.py 2011-04-03 06:12:59 UTC (rev 24545) +++ arm/trunk/src/interface/controller.py 2011-04-03 06:15:38 UTC (rev 24546) @@ -1016,7 +1016,7 @@ resolverUtil = connections.getResolver("tor").overwriteResolver if resolverUtil == None: resolverUtil = "auto" - popup.addfstr(5, 2, "u: resolving utility (%s)" % resolverUtil) + popup.addfstr(3, 41, "u: resolving utility (%s)" % resolverUtil) pageOverrideKeys = (ord('l'), ord('s'), ord('u')) elif page == 3:

1 0

r24545: {arm} Last change accidently broke torrc validation - fixing this. (in arm/trunk/src: interface util)
by Damian Johnson 03 Apr '11

03 Apr '11

Author: atagar Date: 2011-04-03 06:12:59 +0000 (Sun, 03 Apr 2011) New Revision: 24545 Modified: arm/trunk/src/interface/controller.py arm/trunk/src/util/torConfig.py Log: Last change accidently broke torrc validation - fixing this. Modified: arm/trunk/src/interface/controller.py =================================================================== --- arm/trunk/src/interface/controller.py 2011-04-03 05:55:58 UTC (rev 24544) +++ arm/trunk/src/interface/controller.py 2011-04-03 06:12:59 UTC (rev 24545) @@ -1675,7 +1675,7 @@ # display a popup for saving the current configuration panel.CURSES_LOCK.acquire() try: - configLines = torConfig.getCustomOptions() + configLines = torConfig.getCustomOptions(True) # lists event types popup = panels["popup"] Modified: arm/trunk/src/util/torConfig.py =================================================================== --- arm/trunk/src/util/torConfig.py 2011-04-03 05:55:58 UTC (rev 24544) +++ arm/trunk/src/util/torConfig.py 2011-04-03 06:12:59 UTC (rev 24545) @@ -351,9 +351,13 @@ return tuple(MULTILINE_PARAM) -def getCustomOptions(): +def getCustomOptions(includeValue = False): """ Provides the torrc parameters that differ from their defaults. + + Arguments: + includeValue - provides the current value with results if true, otherwise + this just contains the options """ configText = torTools.getConn().getInfo("config-text", "").strip() @@ -370,7 +374,8 @@ try: configLines.remove("Log notice stdout") except ValueError: pass - return configLines + if includeValue: return configLines + else: return [line[:line.find(" ")] for line in configLines] def validate(contents = None): """

1 0

r24544: {arm} Dropping the pointless "Log notice stdout" config-text entry (in arm/trunk/src: interface util)
by Damian Johnson 03 Apr '11

03 Apr '11

Author: atagar Date: 2011-04-03 05:55:58 +0000 (Sun, 03 Apr 2011) New Revision: 24544 Modified: arm/trunk/src/interface/controller.py arm/trunk/src/util/torConfig.py Log: Dropping the pointless "Log notice stdout" config-text entry when writting the custom torrc. Modified: arm/trunk/src/interface/controller.py =================================================================== --- arm/trunk/src/interface/controller.py 2011-04-03 05:23:06 UTC (rev 24543) +++ arm/trunk/src/interface/controller.py 2011-04-03 05:55:58 UTC (rev 24544) @@ -1675,8 +1675,7 @@ # display a popup for saving the current configuration panel.CURSES_LOCK.acquire() try: - configText = torTools.getConn().getInfo("config-text", "").strip() - configLines = configText.split("\n") + configLines = torConfig.getCustomOptions() # lists event types popup = panels["popup"] @@ -1774,7 +1773,7 @@ # saves the configuration to the file configFile = open(configLocation, "w") - configFile.write(configText) + configFile.write("\n".join(configLines)) configFile.close() # reloads the cached torrc if overwriting it Modified: arm/trunk/src/util/torConfig.py =================================================================== --- arm/trunk/src/util/torConfig.py 2011-04-03 05:23:06 UTC (rev 24543) +++ arm/trunk/src/util/torConfig.py 2011-04-03 05:55:58 UTC (rev 24544) @@ -353,17 +353,24 @@ def getCustomOptions(): """ - Provides the set of torrc parameters that differ from their defaults. + Provides the torrc parameters that differ from their defaults. """ - customOptions, conn = set(), torTools.getConn() - configTextQuery = conn.getInfo("config-text", "").strip().split("\n") + configText = torTools.getConn().getInfo("config-text", "").strip() + configLines = configText.split("\n") - for entry in configTextQuery: - # tor provides a Log entry even if it matches the default - if entry != "Log notice stdout": - customOptions.add(entry[:entry.find(" ")]) - return customOptions + # removes any duplicates + configLines = list(set(configLines)) + + # The "GETINFO config-text" query only provides options that differ + # from Tor's defaults with the exception of its Log entry which, even + # if undefined, returns "Log notice stdout" as per: + # https://trac.torproject.org/projects/tor/ticket/2362 + + try: configLines.remove("Log notice stdout") + except ValueError: pass + + return configLines def validate(contents = None): """

1 0

r24543: {arm} Correcting page numbering for when either the new and/or old (arm/trunk/src/interface)
by Damian Johnson 03 Apr '11

03 Apr '11

Author: atagar Date: 2011-04-03 05:23:06 +0000 (Sun, 03 Apr 2011) New Revision: 24543 Modified: arm/trunk/src/interface/controller.py Log: Correcting page numbering for when either the new and/or old connection panel aren't being displayed. Modified: arm/trunk/src/interface/controller.py =================================================================== --- arm/trunk/src/interface/controller.py 2011-04-03 05:04:10 UTC (rev 24542) +++ arm/trunk/src/interface/controller.py 2011-04-03 05:23:06 UTC (rev 24543) @@ -120,7 +120,11 @@ currentPage = self.page pageCount = len(PAGES) - if self.isBlindMode: + if not CONFIG["features.connection.newPanel"]: + if currentPage >= 3: currentPage -= 1 + pageCount -= 1 + + if self.isBlindMode or not CONFIG["features.connection.oldPanel"]: if currentPage >= 2: currentPage -= 1 pageCount -= 1

1 0