- tor-commits - lists.torproject.org

[metrics-db/master] Download all server/extra-info descriptors once per day.
by karsten＠torproject.org 03 Apr '11

03 Apr '11

commit be6e7c3d4f9f9acdf5e2f459453c23a230c129d0 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Apr 1 15:43:38 2011 +0200 Download all server/extra-info descriptors once per day. The relay descriptor downloader so far only downloads server and extra-info descriptors that are referenced in other directory documents and that we are missing. With this patch we also download /tor/server/all and /tor/extra/all from the directories once per day to learn about server and extra-info descriptors that are not referenced anywhere. This patch further requests the compressed versions of descriptors by adding ".z" to the URLs in order to save bandwidth. This patch is a 50% rewrite of the relay descriptor downloader which became too complex over time. --- config.template | 27 +- src/org/torproject/ernie/db/Configuration.java | 56 ++- src/org/torproject/ernie/db/Main.java | 15 +- .../ernie/db/RelayDescriptorDownloader.java | 857 ++++++++++++-------- 4 files changed, 600 insertions(+), 355 deletions(-) diff --git a/config.template b/config.template index 47a8c8d..d744f25 100644 --- a/config.template +++ b/config.template @@ -24,7 +24,32 @@ # ## Comma separated list of directory authority addresses (IP[:port]) to ## download missing relay descriptors from -#DownloadFromDirectoryAuthorities 86.59.21.38,194.109.206.212,212.112.245.170 +#DownloadFromDirectoryAuthorities 86.59.21.38,216.224.124.114:9030,213.115.239.118:443,193.23.244.244,208.83.223.34:443,128.31.0.34:9131,194.109.206.212,212.112.245.170 +# +## Download the current consensus (only if DownloadRelayDescriptors is 1) +#DownloadCurrentConsensus 1 +# +## Download current votes (only if DownloadRelayDescriptors is 1) +#DownloadCurrentVotes 1 +# +## Download missing server descriptors (only if DownloadRelayDescriptors +## is 1) +#DownloadMissingServerDescriptors 1 +# +## Download missing extra-info descriptors (only if +## DownloadRelayDescriptors is 1) +#DownloadMissingExtraInfoDescriptors 1 +# +## Download all server descriptors from the directory authorities at most +## once a day (only if DownloadRelayDescriptors is 1) +#DownloadAllServerDescriptors 0 +# +## Download all extra-info descriptors from the directory authorities at +## most once a day (only if DownloadRelayDescriptors is 1) +#DownloadAllExtraInfoDescriptors 0 +# +## Compress relay descriptors downloads by adding .z to the URLs +#CompressRelayDescriptorDownloads 0 # ## Import bridge snapshots from disk, if available #ImportBridgeSnapshots 0 diff --git a/src/org/torproject/ernie/db/Configuration.java b/src/org/torproject/ernie/db/Configuration.java index ae24175..b4e6f8b 100644 --- a/src/org/torproject/ernie/db/Configuration.java +++ b/src/org/torproject/ernie/db/Configuration.java @@ -30,8 +30,17 @@ public class Configuration { private boolean importBridgeSnapshots = false; private String bridgeSnapshotsDirectory = "bridge-directories/"; private boolean downloadRelayDescriptors = false; - private List<String> downloadFromDirectoryAuthorities = Arrays.asList( - "86.59.21.38,194.109.206.212,80.190.246.100:8180".split(",")); + private List<String> downloadFromDirectoryAuthorities = Arrays.asList(( + "86.59.21.38,216.224.124.114:9030,213.115.239.118:443," + + "193.23.244.244,208.83.223.34:443,128.31.0.34:9131," + + "194.109.206.212,212.112.245.170").split(",")); + private boolean downloadCurrentConsensus = true; + private boolean downloadCurrentVotes = true; + private boolean downloadMissingServerDescriptors = true; + private boolean downloadMissingExtraInfoDescriptors = true; + private boolean downloadAllServerDescriptors = false; + private boolean downloadAllExtraInfoDescriptors = false; + private boolean compressRelayDescriptorDownloads; private boolean downloadGetTorStats = false; private String getTorStatsUrl = "http://gettor.torproject.org:8080/" + "~gettor/gettor_stats.txt"; @@ -117,6 +126,28 @@ public class Configuration { new URL("http://" + dir + "/"); this.downloadFromDirectoryAuthorities.add(dir); } + } else if (line.startsWith("DownloadCurrentConsensus")) { + this.downloadCurrentConsensus = Integer.parseInt( + line.split(" ")[1]) != 0; + } else if (line.startsWith("DownloadCurrentVotes")) { + this.downloadCurrentVotes = Integer.parseInt( + line.split(" ")[1]) != 0; + } else if (line.startsWith("DownloadMissingServerDescriptors")) { + this.downloadMissingServerDescriptors = Integer.parseInt( + line.split(" ")[1]) != 0; + } else if (line.startsWith( + "DownloadMissingExtraInfoDescriptors")) { + this.downloadMissingExtraInfoDescriptors = Integer.parseInt( + line.split(" ")[1]) != 0; + } else if (line.startsWith("DownloadAllServerDescriptors")) { + this.downloadAllServerDescriptors = Integer.parseInt( + line.split(" ")[1]) != 0; + } else if (line.startsWith("DownloadAllExtraInfoDescriptors")) { + this.downloadAllExtraInfoDescriptors = Integer.parseInt( + line.split(" ")[1]) != 0; + } else if (line.startsWith("CompressRelayDescriptorDownloads")) { + this.compressRelayDescriptorDownloads = Integer.parseInt( + line.split(" ")[1]) != 0; } else if (line.startsWith("DownloadGetTorStats")) { this.downloadGetTorStats = Integer.parseInt( line.split(" ")[1]) != 0; @@ -244,6 +275,27 @@ public class Configuration { public List<String> getDownloadFromDirectoryAuthorities() { return this.downloadFromDirectoryAuthorities; } + public boolean getDownloadCurrentConsensus() { + return this.downloadCurrentConsensus; + } + public boolean getDownloadCurrentVotes() { + return this.downloadCurrentVotes; + } + public boolean getDownloadMissingServerDescriptors() { + return this.downloadMissingServerDescriptors; + } + public boolean getDownloadMissingExtraInfoDescriptors() { + return this.downloadMissingExtraInfoDescriptors; + } + public boolean getDownloadAllServerDescriptors() { + return this.downloadAllServerDescriptors; + } + public boolean getDownloadAllExtraInfoDescriptors() { + return this.downloadAllExtraInfoDescriptors; + } + public boolean getCompressRelayDescriptorDownloads() { + return this.compressRelayDescriptorDownloads; + } public boolean getDownloadGetTorStats() { return this.downloadGetTorStats; } diff --git a/src/org/torproject/ernie/db/Main.java b/src/org/torproject/ernie/db/Main.java index 50c06bb..5a06900 100644 --- a/src/org/torproject/ernie/db/Main.java +++ b/src/org/torproject/ernie/db/Main.java @@ -49,13 +49,14 @@ public class Main { if (config.getDownloadRelayDescriptors()) { List<String> dirSources = config.getDownloadFromDirectoryAuthorities(); - boolean downloadCurrentConsensus = aw != null; - boolean downloadCurrentVotes = aw != null; - boolean downloadAllServerDescriptors = aw != null; - boolean downloadAllExtraInfos = aw != null; rdd = new RelayDescriptorDownloader(rdp, dirSources, - downloadCurrentConsensus, downloadCurrentVotes, - downloadAllServerDescriptors, downloadAllExtraInfos); + config.getDownloadCurrentConsensus(), + config.getDownloadCurrentVotes(), + config.getDownloadMissingServerDescriptors(), + config.getDownloadMissingExtraInfoDescriptors(), + config.getDownloadAllServerDescriptors(), + config.getDownloadAllExtraInfoDescriptors(), + config.getCompressRelayDescriptorDownloads()); rdp.setRelayDescriptorDownloader(rdd); } if (config.getImportCachedRelayDescriptors()) { @@ -77,7 +78,7 @@ public class Main { } } if (rdd != null) { - rdd.downloadMissingDescriptors(); + rdd.downloadDescriptors(); rdd.writeFile(); rdd = null; if (aw != null) { diff --git a/src/org/torproject/ernie/db/RelayDescriptorDownloader.java b/src/org/torproject/ernie/db/RelayDescriptorDownloader.java index 56d7e0d..ebbf761 100644 --- a/src/org/torproject/ernie/db/RelayDescriptorDownloader.java +++ b/src/org/torproject/ernie/db/RelayDescriptorDownloader.java @@ -7,27 +7,29 @@ import java.net.*; import java.text.*; import java.util.*; import java.util.logging.*; +import java.util.zip.*; import org.apache.commons.codec.digest.*; import org.apache.commons.codec.binary.*; /** - * Downloads missing relay descriptors from the directories via HTTP. + * Downloads relay descriptors from the directory authorities via HTTP. * Keeps a list of missing descriptors that gets updated by parse results - * from <code>RelayDescriptorParser</code>. Only descriptors on that - * missing list that we think might be available on the directories are - * downloaded. + * from <code>RelayDescriptorParser</code> and downloads all missing + * descriptors that have been published in the last 24 hours. Also + * downloads all server and extra-info descriptors known to a directory + * authority at most once a day. */ public class RelayDescriptorDownloader { /** * Text file containing the descriptors that we are missing and that we - * want to download in <code>downloadMissingDescriptors</code>. - * Lines are formatted as: + * want to download. Lines are formatted as: + * * - "consensus,<validafter>,<parsed>", * - "vote,<validafter>,<fingerprint>,<parsed>", * - "server,<published>,<relayid>,<descid>,<parsed>", or - * - "extra,<published>,<relayid>,<descid><parsed>". + * - "extra,<published>,<relayid>,<descid>,<parsed>". */ private File missingDescriptorsFile; @@ -35,25 +37,36 @@ public class RelayDescriptorDownloader { * Relay descriptors that we are missing and that we want to download * either in this execution or write to disk and try next time. Map keys * contain comma-separated values as in the missing descriptors files - * without the parsed column. Map values contain the parsed column. + * without the "parsed" column. Map values contain the "parsed" column. */ private SortedMap<String, String> missingDescriptors; /** - * <code>RelayDescriptorParser</code> that we will hand over the - * downloaded descriptors for parsing. + * Text file containing the IP addresses (and Dir ports if not 80) of + * directory authorities and when we last downloaded all server and + * extra-info descriptors from them, so that we can avoid downloading + * them too often. */ - private RelayDescriptorParser rdp; + private File lastDownloadedAllDescriptorsFile; + + /** + * Map of directory authorities and when we last downloaded all server + * and extra-info descriptors from them. Map keys are IP addresses (and + * Dir ports if not 80), map values are timestamps. + */ + private Map<String, String> lastDownloadedAllDescriptors; /** - * Directories that we will try to download missing descriptors from. + * <code>RelayDescriptorParser</code> that we will hand over the + * downloaded descriptors for parsing. */ - private List<String> dirSources; + private RelayDescriptorParser rdp; /** - * Number of descriptors requested by directory to be included in logs. + * Directory authorities that we will try to download missing + * descriptors from. */ - private Map<String, Integer> dirRequests; + private List<String> authorities; /** * Should we try to download the current consensus if we don't have it? @@ -66,22 +79,41 @@ public class RelayDescriptorDownloader { private boolean downloadCurrentVotes; /** - * Should we try to download all missing server descriptors that have + * Should we try to download missing server descriptors that have been + * published within the past 24 hours? + */ + private boolean downloadMissingServerDescriptors; + + /** + * Should we try to download missing extra-info descriptors that have * been published within the past 24 hours? */ + private boolean downloadMissingExtraInfos; + + /** + * Should we try to download all server descriptors from the authorities + * once every 24 hours? + */ private boolean downloadAllServerDescriptors; /** - * Should we try to download all missing extra-info descriptors that - * have been published within the past 24 hours? + * Should we try to download all extra-info descriptors from the + * authorities once every 24 hours? */ private boolean downloadAllExtraInfos; /** + * Should we download zlib-compressed versions of descriptors by adding + * ".z" to URLs? + */ + private boolean downloadCompressed; + + /** * valid-after time that we expect the current consensus and votes to * have, formatted "yyyy-MM-dd HH:mm:ss". We only expect to find - * consensuses and votes with this valid-after time on the directories. - * This time is initialized as the beginning of the current hour. + * consensuses and votes with this valid-after time on the directory + * authorities. This time is initialized as the beginning of the current + * hour. */ private String currentValidAfter; @@ -93,77 +125,115 @@ public class RelayDescriptorDownloader { private String descriptorCutOff; /** + * Cut-off time for downloading all server and extra-info descriptors + * from the directory authorities, formatted "yyyy-MM-dd HH:mm:ss". This + * time is initialized as the current system time minus 23:30 hours. + */ + private String downloadAllDescriptorsCutOff; + + /** + * Directory authorities that we plan to download all server and + * extra-info descriptors from in this execution. + */ + private Set<String> downloadAllDescriptorsFromAuthorities; + + /** * Current timestamp that is written to the missing list for descriptors - * that we parsed in this execution. This timestamp is most useful for - * debugging purposes when looking at the missing list. For execution it - * only matters whether the parsed time is "NA" or has some other value. + * that we parsed in this execution and for authorities that we + * downloaded all server and extra-info descriptors from. */ - private String parsedTimestampString; + private String currentTimestamp; /** * Logger for this class. */ private Logger logger; - private StringBuilder dumpStats; - private int newMissingConsensuses = 0, newMissingVotes = 0, + /** + * Number of descriptors requested by directory authority to be included + * in logs. + */ + private Map<String, Integer> requestsByAuthority; + + /** + * Counters for descriptors that we had on the missing list at the + * beginning of the execution, that we added to the missing list, + * that we requested, and that we successfully downloaded in this + * execution. + */ + private int oldMissingConsensuses = 0, oldMissingVotes = 0, + oldMissingServerDescriptors = 0, oldMissingExtraInfoDescriptors = 0, + newMissingConsensuses = 0, newMissingVotes = 0, newMissingServerDescriptors = 0, newMissingExtraInfoDescriptors = 0, - triedConsensuses = 0, triedVotes = 0, triedServerDescriptors = 0, - triedExtraInfoDescriptors = 0, downloadedConsensuses = 0, - downloadedVotes = 0, downloadedServerDescriptors = 0, - downloadedExtraInfoDescriptors = 0; + requestedConsensuses = 0, requestedVotes = 0, + requestedMissingServerDescriptors = 0, + requestedAllServerDescriptors = 0, + requestedMissingExtraInfoDescriptors = 0, + requestedAllExtraInfoDescriptors = 0, downloadedConsensuses = 0, + downloadedVotes = 0, downloadedMissingServerDescriptors = 0, + downloadedAllServerDescriptors = 0, + downloadedMissingExtraInfoDescriptors = 0, + downloadedAllExtraInfoDescriptors = 0; + /** * Initializes this class, including reading in missing descriptors from - * <code>stats/missing-relay-descriptors</code>. + * <code>stats/missing-relay-descriptors</code> and the times when we + * last downloaded all server and extra-info descriptors from + * <code>stats/last-downloaded-all-descriptors</code>. */ public RelayDescriptorDownloader(RelayDescriptorParser rdp, - List<String> dirSources, boolean downloadCurrentConsensus, - boolean downloadCurrentVotes, boolean downloadAllServerDescriptors, - boolean downloadAllExtraInfos) { + List<String> authorities, boolean downloadCurrentConsensus, + boolean downloadCurrentVotes, + boolean downloadMissingServerDescriptors, + boolean downloadMissingExtraInfos, + boolean downloadAllServerDescriptors, boolean downloadAllExtraInfos, + boolean downloadCompressed) { /* Memorize argument values. */ this.rdp = rdp; - this.dirSources = dirSources; + this.authorities = new ArrayList<String>(authorities); this.downloadCurrentConsensus = downloadCurrentConsensus; this.downloadCurrentVotes = downloadCurrentVotes; + this.downloadMissingServerDescriptors = + downloadMissingServerDescriptors; + this.downloadMissingExtraInfos = downloadMissingExtraInfos; this.downloadAllServerDescriptors = downloadAllServerDescriptors; this.downloadAllExtraInfos = downloadAllExtraInfos; + this.downloadCompressed = downloadCompressed; + + /* Shuffle list of authorities for better load balancing over time. */ + Collections.shuffle(this.authorities); /* Initialize logger. */ - this.logger = Logger.getLogger(RelayDescriptorParser.class.getName()); + this.logger = Logger.getLogger( + RelayDescriptorDownloader.class.getName()); - /* Prepare cut-off times and timestamp for missing descriptors - * list. */ + /* Prepare cut-off times and timestamp for the missing descriptors + * list and the list of authorities to download all server and + * extra-info descriptors from. */ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); format.setTimeZone(TimeZone.getTimeZone("UTC")); long now = System.currentTimeMillis(); this.currentValidAfter = format.format((now / (60L * 60L * 1000L)) * (60L * 60L * 1000L)); this.descriptorCutOff = format.format(now - 24L * 60L * 60L * 1000L); - this.parsedTimestampString = format.format(now); - - /* Initialize missing list and put current consensus on it if we want - * it. */ - this.missingDescriptors = new TreeMap<String, String>(); - if (this.downloadCurrentConsensus) { - this.missingDescriptors.put("consensus," + this.currentValidAfter, - "NA"); - } + this.currentTimestamp = format.format(now); + this.downloadAllDescriptorsCutOff = format.format(now + - 23L * 60L * 60L * 1000L - 30L * 60L * 1000L); /* Read list of missing descriptors from disk and memorize those that * we are interested in and that are likely to be found on the - * directory servers. */ + * directory authorities. */ + this.missingDescriptors = new TreeMap<String, String>(); this.missingDescriptorsFile = new File( "stats/missing-relay-descriptors"); - int missingConsensuses = 0, missingVotes = 0, - missingServerDescriptors = 0, missingExtraInfoDescriptors = 0; if (this.missingDescriptorsFile.exists()) { try { this.logger.fine("Reading file " + this.missingDescriptorsFile.getAbsolutePath() + "..."); BufferedReader br = new BufferedReader(new FileReader( this.missingDescriptorsFile)); - String line = null; + String line; while ((line = br.readLine()) != null) { if (line.split(",").length > 2) { String published = line.split(",")[1]; @@ -172,17 +242,17 @@ public class RelayDescriptorDownloader { this.currentValidAfter.equals(published)) || ((line.startsWith("server,") || line.startsWith("extra,")) && - this.descriptorCutOff.compareTo(published) <= 0)) { + this.descriptorCutOff.compareTo(published) < 0)) { if (!line.endsWith("NA")) { /* Not missing. */ } else if (line.startsWith("consensus,")) { - missingConsensuses++; + oldMissingConsensuses++; } else if (line.startsWith("vote,")) { - missingVotes++; + oldMissingVotes++; } else if (line.startsWith("server,")) { - missingServerDescriptors++; + oldMissingServerDescriptors++; } else if (line.startsWith("extra,")) { - missingExtraInfoDescriptors++; + oldMissingExtraInfoDescriptors++; } int separateAt = line.lastIndexOf(","); this.missingDescriptors.put(line.substring(0, @@ -205,86 +275,131 @@ public class RelayDescriptorDownloader { } } - dumpStats = new StringBuilder(); - dumpStats.append("Finished downloading relay descriptors from the " - + "directory authorities:\nAt the beginning of this execution, " - + "we were missing " + missingConsensuses + " consensus(es), " - + missingVotes + " vote(s), " + missingServerDescriptors - + " server descriptor(s), and " + missingExtraInfoDescriptors - + " extra-info descriptor(s).\n"); - - dirRequests = new HashMap<String, Integer>(); - for (String dirSource : dirSources) { - dirRequests.put(dirSource, 0); + /* Put the current consensus on the missing list, unless we already + * have it. */ + String consensusKey = "consensus," + this.currentValidAfter; + if (!this.missingDescriptors.containsKey(consensusKey)) { + this.missingDescriptors.put(consensusKey, "NA"); + this.newMissingConsensuses++; + } + + /* Read list of directory authorities and when we last downloaded all + * server and extra-info descriptors from them. */ + this.lastDownloadedAllDescriptors = new HashMap<String, String>(); + this.lastDownloadedAllDescriptorsFile = new File( + "stats/last-downloaded-all-descriptors"); + if (this.lastDownloadedAllDescriptorsFile.exists()) { + try { + this.logger.fine("Reading file " + + this.lastDownloadedAllDescriptorsFile.getAbsolutePath() + + "..."); + BufferedReader br = new BufferedReader(new FileReader( + this.lastDownloadedAllDescriptorsFile)); + String line; + while ((line = br.readLine()) != null) { + if (line.split(",").length != 2) { + this.logger.fine("Invalid line '" + line + "' in " + + this.lastDownloadedAllDescriptorsFile.getAbsolutePath() + + ". Ignoring."); + } else { + String[] parts = line.split(","); + String authority = parts[0]; + String lastDownloaded = parts[1]; + this.lastDownloadedAllDescriptors.put(authority, + lastDownloaded); + } + } + br.close(); + this.logger.fine("Finished reading file " + + this.lastDownloadedAllDescriptorsFile.getAbsolutePath() + + "."); + } catch (IOException e) { + this.logger.log(Level.WARNING, "Failed to read file " + + this.lastDownloadedAllDescriptorsFile.getAbsolutePath() + + "! This means that we might download all server and " + + "extra-info descriptors more often than we should.", e); + } + } + + /* Make a list of directory authorities that we want to download all + * server and extra-info descriptors from. */ + this.downloadAllDescriptorsFromAuthorities = new HashSet<String>(); + for (String authority : this.authorities) { + if (!this.lastDownloadedAllDescriptors.containsKey(authority) || + this.lastDownloadedAllDescriptors.get(authority).compareTo( + this.downloadAllDescriptorsCutOff) < 0) { + this.downloadAllDescriptorsFromAuthorities.add(authority); + } + } + + /* Prepare statistics on this execution. */ + this.requestsByAuthority = new HashMap<String, Integer>(); + for (String authority : this.authorities) { + this.requestsByAuthority.put(authority, 0); } } /** * We have parsed a consensus. Take this consensus off the missing list - * and add the votes created by the given <code>dirSources</code> and - * the <code>serverDescriptors</code> in the format + * and add the votes created by the given <code>authorities</code> and + * the <code>serverDescriptors</code> which are in the format * "<published>,<relayid>,<descid>" to that list. */ public void haveParsedConsensus(String validAfter, - Set<String> dirSources, Set<String> serverDescriptors) { + Set<String> authorities, Set<String> serverDescriptors) { /* Mark consensus as parsed. */ if (this.currentValidAfter.equals(validAfter)) { String consensusKey = "consensus," + validAfter; - this.missingDescriptors.put(consensusKey, - this.parsedTimestampString); + this.missingDescriptors.put(consensusKey, this.currentTimestamp); /* Add votes to missing list. */ - if (this.downloadCurrentVotes) { - for (String dirSource : dirSources) { - String voteKey = "vote," + validAfter + "," + dirSource; - if (!this.missingDescriptors.containsKey(voteKey)) { - this.missingDescriptors.put(voteKey, "NA"); - this.newMissingVotes++; - } + for (String authority : authorities) { + String voteKey = "vote," + validAfter + "," + authority; + if (!this.missingDescriptors.containsKey(voteKey)) { + this.missingDescriptors.put(voteKey, "NA"); + this.newMissingVotes++; } } } /* Add server descriptors to missing list. */ - if (this.downloadAllServerDescriptors) { - for (String serverDescriptor : serverDescriptors) { - String published = serverDescriptor.split(",")[0]; - if (this.descriptorCutOff.compareTo(published) <= 0) { - String serverDescriptorKey = "server," + serverDescriptor; - if (!this.missingDescriptors.containsKey( - serverDescriptorKey)) { - this.missingDescriptors.put(serverDescriptorKey, "NA"); - this.newMissingServerDescriptors++; - } + for (String serverDescriptor : serverDescriptors) { + String published = serverDescriptor.split(",")[0]; + if (this.descriptorCutOff.compareTo(published) < 0) { + String serverDescriptorKey = "server," + serverDescriptor; + if (!this.missingDescriptors.containsKey( + serverDescriptorKey)) { + this.missingDescriptors.put(serverDescriptorKey, "NA"); + this.newMissingServerDescriptors++; } } } } /** - * We have parsed a vote. Take this vote off the missing list. + * We have parsed a vote. Take this vote off the missing list and add + * the <code>serverDescriptors</code> which are in the format + * "<published>,<relayid>,<descid>" to that list. */ public void haveParsedVote(String validAfter, String fingerprint, Set<String> serverDescriptors) { - /* Mark consensus as parsed. */ + /* Mark vote as parsed. */ if (this.currentValidAfter.equals(validAfter)) { String voteKey = "vote," + validAfter + "," + fingerprint; - this.missingDescriptors.put(voteKey, this.parsedTimestampString); + this.missingDescriptors.put(voteKey, this.currentTimestamp); } /* Add server descriptors to missing list. */ - if (this.downloadAllServerDescriptors) { - for (String serverDescriptor : serverDescriptors) { - String published = serverDescriptor.split(",")[0]; - if (this.descriptorCutOff.compareTo(published) < 0) { - String serverDescriptorKey = "server," + serverDescriptor; - if (!this.missingDescriptors.containsKey( - serverDescriptorKey)) { - this.missingDescriptors.put(serverDescriptorKey, "NA"); - this.newMissingServerDescriptors++; - } + for (String serverDescriptor : serverDescriptors) { + String published = serverDescriptor.split(",")[0]; + if (this.descriptorCutOff.compareTo(published) < 0) { + String serverDescriptorKey = "server," + serverDescriptor; + if (!this.missingDescriptors.containsKey( + serverDescriptorKey)) { + this.missingDescriptors.put(serverDescriptorKey, "NA"); + this.newMissingServerDescriptors++; } } } @@ -300,14 +415,14 @@ public class RelayDescriptorDownloader { String extraInfoDigest) { /* Mark server descriptor as parsed. */ - if (this.descriptorCutOff.compareTo(published) <= 0) { + if (this.descriptorCutOff.compareTo(published) < 0) { String serverDescriptorKey = "server," + published + "," + relayIdentity + "," + serverDescriptorDigest; this.missingDescriptors.put(serverDescriptorKey, - this.parsedTimestampString); + this.currentTimestamp); /* Add extra-info descriptor to missing list. */ - if (extraInfoDigest != null && this.downloadAllExtraInfos) { + if (extraInfoDigest != null) { String extraInfoKey = "extra," + published + "," + relayIdentity + "," + extraInfoDigest; if (!this.missingDescriptors.containsKey(extraInfoKey)) { @@ -324,253 +439,261 @@ public class RelayDescriptorDownloader { */ public void haveParsedExtraInfoDescriptor(String published, String relayIdentity, String extraInfoDigest) { - if (this.descriptorCutOff.compareTo(published) <= 0) { + if (this.descriptorCutOff.compareTo(published) < 0) { String extraInfoKey = "extra," + published + "," + relayIdentity + "," + extraInfoDigest; - this.missingDescriptors.put(extraInfoKey, - this.parsedTimestampString); + this.missingDescriptors.put(extraInfoKey, this.currentTimestamp); } } /** * Downloads missing descriptors that we think might still be available - * on the directories. + * on the directory authorities as well as all server and extra-info + * descriptors once per day. */ - public void downloadMissingDescriptors() { - - /* Update cut-off times to reflect that execution so far might have - * taken a few minutes and that some descriptors aren't available on - * the directories anymore. */ - SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - format.setTimeZone(TimeZone.getTimeZone("UTC")); - long now = System.currentTimeMillis(); - this.currentValidAfter = format.format((now / (60L * 60L * 1000L)) * - (60L * 60L * 1000L)); - this.descriptorCutOff = format.format(now - 24L * 60L * 60L * 1000L); + public void downloadDescriptors() { + + /* Download descriptors from authorities which are in random order, so + * that we distribute the load somewhat fairly over time. */ + for (String authority : authorities) { + + /* Make all requests to an authority in a single try block. If + * something goes wrong with this authority, we give up on all + * downloads and continue with the next authority. */ + /* TODO Some authorities provide very little bandwidth and could + * slow down the entire download process. Ponder adding a timeout of + * 3 or 5 minutes per authority to avoid getting in the way of the + * next execution. */ + try { - /* Remember which directories remain as source for downloading - * descriptors. */ - List<String> remainingDirSources = - new ArrayList<String>(this.dirSources); - - /* URLs of descriptors we want to download. */ - SortedSet<String> urls = new TreeSet<String>(); - - /* Complete URLs of authorities and descriptors we have downloaded or - * tried to download. Ensures that we're not attempting to download - * the same thing from an authority more than once. There are edge - * cases when an authority returns a valid response to something we - * asked for, but which is not what we wanted (e.g. old consensus). */ - SortedSet<String> downloaded = new TreeSet<String>(); - - /* We might need more than one iteration for downloading descriptors, - * because we might learn about descriptors while parsing those that - * we got. In every iteration, compile a new list of URLs, remove - * those that we tried before, and download the remaining ones. Stop - * when there are no new URLs anymore. */ - while (true) { - - /* Compile list of URLs to download in this iteration. */ - urls.clear(); - for (Map.Entry<String, String> e : - this.missingDescriptors.entrySet()) { - if (e.getValue().equals("NA")) { - String[] parts = e.getKey().split(","); - if (parts[0].equals("consensus") && - this.downloadCurrentConsensus && - this.currentValidAfter.equals(parts[1])) { - urls.add("/tor/status-vote/current/consensus"); - } else if (parts[0].equals("vote") && - this.downloadCurrentVotes && - this.currentValidAfter.equals(parts[1])) { - urls.add("/tor/status-vote/current/" + parts[2]); - } else if (parts[0].equals("server") && - this.downloadAllServerDescriptors && - this.descriptorCutOff.compareTo(parts[1]) <= 0) { - urls.add("/tor/server/d/" + parts[3]); - } else if (parts[0].equals("extra") && - this.downloadAllExtraInfos && - this.descriptorCutOff.compareTo(parts[1]) <= 0) { - urls.add("/tor/extra/d/" + parts[3]); + /* Start with downloading the current consensus, unless we already + * have it. */ + if (downloadCurrentConsensus) { + String consensusKey = "consensus," + this.currentValidAfter; + if (this.missingDescriptors.containsKey(consensusKey) && + this.missingDescriptors.get(consensusKey).equals("NA")) { + this.requestedConsensuses++; + this.downloadedConsensuses += + this.downloadResourceFromAuthority(authority, + "/tor/status-vote/current/consensus"); } } - } - /* Stop if we don't have (new) URLs to download. */ - if (urls.isEmpty()) { - break; - } - - for (String url : urls) { - if (url.endsWith("consensus")) { - this.triedConsensuses++; - } else if (url.contains("status-vote")) { - this.triedVotes++; - } else if (url.contains("server")) { - this.triedServerDescriptors++; - } else if (url.contains("extra")) { - this.triedExtraInfoDescriptors++; - } - } - - /* Log what we're downloading. */ - StringBuilder sb = new StringBuilder("Downloading " + urls.size() - + " descriptors:"); - for (String url : urls) { - sb.append("\n" + url); - } - this.logger.fine(sb.toString()); - - /* We are trying to download these descriptors from each directory - * source one after the other until we got it from one. For each - * directory source we are removing the URLs from urls and putting - * those the we want to retry into retryUrls. Once we are done, we - * move the URLs back to urls and try the next directory source. */ - SortedSet<String> currentDirSources = - new TreeSet<String>(remainingDirSources); - SortedSet<String> retryUrls = new TreeSet<String>(); - int numDownloaded = 0; - while (!currentDirSources.isEmpty() && !urls.isEmpty()) { - String authority = currentDirSources.first(); - String url = urls.first(); - urls.remove(url); - Set<String> requestedUrls = new HashSet<String>(); - requestedUrls.add(url); - if (url.contains("server")) { - StringBuilder combinedUrl = new StringBuilder(url); - int descriptors = 1; - while (descriptors++ <= 96 && !urls.isEmpty() && - urls.first().contains("server")) { - url = urls.first(); - urls.remove(url); - requestedUrls.add(url); - combinedUrl.append("+" + url.substring("/tor/server/d/". - length())); + /* Next, try to download current votes that we're missing. */ + if (downloadCurrentVotes) { + String voteKeyPrefix = "vote," + this.currentValidAfter; + SortedSet<String> fingerprints = new TreeSet<String>(); + for (Map.Entry<String, String> e : + this.missingDescriptors.entrySet()) { + if (e.getValue().equals("NA") && + e.getKey().startsWith(voteKeyPrefix)) { + String fingerprint = e.getKey().split(",")[2]; + fingerprints.add(fingerprint); + } } - url = combinedUrl.toString(); - } else if (url.contains("extra")) { - StringBuilder combinedUrl = new StringBuilder(url); - int descriptors = 1; - while (descriptors++ <= 96 && !urls.isEmpty() && - urls.first().contains("extra")) { - url = urls.first(); - urls.remove(url); - requestedUrls.add(url); - combinedUrl.append("+" + url.substring("/tor/extra/d/". - length())); + for (String fingerprint : fingerprints) { + this.requestedVotes++; + this.downloadedVotes += + this.downloadResourceFromAuthority(authority, + "/tor/status-vote/current/" + fingerprint); } - url = combinedUrl.toString(); } - String fullUrl = "http://" + authority + url; - byte[] allData = null; - if (!downloaded.contains(fullUrl)) { - downloaded.add(fullUrl); - numDownloaded++; - this.dirRequests.put(authority, dirRequests.get(authority) + 1); - try { - URL u = new URL(fullUrl); - HttpURLConnection huc = - (HttpURLConnection) u.openConnection(); - huc.setRequestMethod("GET"); - huc.connect(); - int response = huc.getResponseCode(); - logger.fine("Downloading http://" + authority + url + " -> " - + response); - if (response == 200) { - BufferedInputStream in = new BufferedInputStream( - huc.getInputStream()); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - int len; - byte[] data = new byte[1024]; - while ((len = in.read(data, 0, 1024)) >= 0) { - baos.write(data, 0, len); - } - in.close(); - allData = baos.toByteArray(); - } - } catch (IOException e) { - remainingDirSources.remove(authority); - currentDirSources.remove(authority); - if (!remainingDirSources.isEmpty()) { - logger.log(Level.FINE, "Failed downloading from " - + authority + "!", e); + + /* Download either all server and extra-info descriptors or only + * those that we're missing. Start with server descriptors, then + * request extra-info descriptors. */ + List<String> types = new ArrayList<String>(Arrays.asList( + "server,extra".split(","))); + for (String type : types) { + + /* Download all server or extra-info descriptors from this + * authority if we haven't done so for 24 hours and if we're + * configured to do so. */ + /* TODO Distribute downloads of all descriptors over the day for + * different authorities. Maybe limit the number of these + * downloads to 1 or 2 per execution. */ + if (this.downloadAllDescriptorsFromAuthorities.contains( + authority) && ((type.equals("server") && + this.downloadAllServerDescriptors) || + (type.equals("extra") && this.downloadAllExtraInfos))) { + int downloadedAllDescriptors = + this.downloadResourceFromAuthority(authority, "/tor/" + + type + "/all"); + if (type.equals("server")) { + this.requestedAllServerDescriptors++; + this.downloadedAllServerDescriptors += + downloadedAllDescriptors; } else { - logger.log(Level.WARNING, "Failed downloading from " - + authority + "! We have no authorities left to download " - + "from!", e); + this.requestedAllExtraInfoDescriptors++; + this.downloadedAllExtraInfoDescriptors += + downloadedAllDescriptors; } - } - } - if (allData != null) { - if (url.endsWith("consensus")) { - this.rdp.parse(allData); - this.downloadedConsensuses++; - } else if (url.contains("status-vote")) { - this.rdp.parse(allData); - this.downloadedVotes++; - } else if (url.contains("server") || - url.contains("extra")) { - String ascii = null; - try { - ascii = new String(allData, "US-ASCII"); - } catch (UnsupportedEncodingException e) { - } - int start = -1, sig = -1, end = -1; - String startToken = url.contains("server") ? - "router " : "extra-info "; - String sigToken = "\nrouter-signature\n"; - String endToken = "\n-----END SIGNATURE-----\n"; - while (end < ascii.length()) { - start = ascii.indexOf(startToken, end); - if (start < 0) { - break; - } - sig = ascii.indexOf(sigToken, start); - if (sig < 0) { - break; + + /* Download missing server or extra-info descriptors if we're + * configured to do so. */ + } else if ((type.equals("server") && + this.downloadMissingServerDescriptors) || + (type.equals("extra") && this.downloadMissingExtraInfos)) { + + /* Go through the list of missing descriptors of this type + * and combine the descriptor identifiers to a URL of up to + * 96 descriptors that we can download at once. */ + SortedSet<String> descriptorIdentifiers = + new TreeSet<String>(); + for (Map.Entry<String, String> e : + this.missingDescriptors.entrySet()) { + if (e.getValue().equals("NA") && + e.getKey().startsWith(type + ",") && + this.descriptorCutOff.compareTo( + e.getKey().split(",")[1]) < 0) { + String descriptorIdentifier = e.getKey().split(",")[3]; + descriptorIdentifiers.add(descriptorIdentifier); } - sig += sigToken.length(); - end = ascii.indexOf(endToken, sig); - if (end < 0) { - break; + } + StringBuilder combinedResource = null; + int descriptorsInCombinedResource = 0, + requestedDescriptors = 0, downloadedDescriptors = 0; + for (String descriptorIdentifier : descriptorIdentifiers) { + if (descriptorsInCombinedResource >= 96) { + requestedDescriptors += descriptorsInCombinedResource; + downloadedDescriptors += + this.downloadResourceFromAuthority(authority, + combinedResource.toString()); + combinedResource = null; + descriptorsInCombinedResource = 0; } - end += endToken.length(); - byte[] descBytes = new byte[end - start]; - System.arraycopy(allData, start, descBytes, 0, end - start); - String digest = Hex.encodeHexString(DigestUtils.sha( - descBytes)); - this.rdp.parse(descBytes); - if (url.contains("server")) { - this.downloadedServerDescriptors++; - requestedUrls.remove("/tor/server/d/" + digest); + if (descriptorsInCombinedResource == 0) { + combinedResource = new StringBuilder("/tor/" + type + + "/d/" + descriptorIdentifier); } else { - this.downloadedExtraInfoDescriptors++; - requestedUrls.remove("/tor/extra/d/" + digest); + combinedResource.append("+" + descriptorIdentifier); } + descriptorsInCombinedResource++; + } + if (descriptorsInCombinedResource > 0) { + requestedDescriptors += descriptorsInCombinedResource; + downloadedDescriptors += + this.downloadResourceFromAuthority(authority, + combinedResource.toString()); + } + if (type.equals("server")) { + this.requestedMissingServerDescriptors += + requestedDescriptors; + this.downloadedMissingServerDescriptors += + downloadedDescriptors; + } else { + this.requestedMissingExtraInfoDescriptors += + requestedDescriptors; + this.downloadedMissingExtraInfoDescriptors += + downloadedDescriptors; } } } - if (requestedUrls.isEmpty()) { - retryUrls.addAll(requestedUrls); - logger.fine("Retrying " + requestedUrls.size() + " URLs for " - + "which we didn't receive descriptors."); - } - if (urls.isEmpty()) { - currentDirSources.remove(authority); - urls.addAll(retryUrls); - retryUrls.clear(); - } + + /* If a download failed, stop requesting descriptors from this + * authority and move on to the next. */ + } catch (IOException e) { + logger.log(Level.FINE, "Failed downloading from " + authority + + "!", e); } + } + } - /* If we haven't downloaded a single descriptor in this iteration, - * we cannot have learned something new, so we're done. */ - if (numDownloaded < 1) { - break; + /** + * Attempts to download one or more descriptors identified by a resource + * string from a directory authority and passes the returned + * descriptor(s) to the <code>RelayDescriptorParser</code> upon success. + * Returns the number of descriptors contained in the reply. Throws an + * <code>IOException</code> if something goes wrong while downloading. + */ + private int downloadResourceFromAuthority(String authority, + String resource) throws IOException { + byte[] allData = null; + this.requestsByAuthority.put(authority, + this.requestsByAuthority.get(authority) + 1); + /* TODO Disable compressed downloads for extra-info descriptors, + * because zlib decompression doesn't work correctly. Figure out why + * this is and fix it. */ + String fullUrl = "http://" + authority + resource + + (this.downloadCompressed && !resource.startsWith("/tor/extra/") + ? ".z" : ""); + URL u = new URL(fullUrl); + HttpURLConnection huc = (HttpURLConnection) u.openConnection(); + huc.setRequestMethod("GET"); + huc.connect(); + int response = huc.getResponseCode(); + if (response == 200) { + BufferedInputStream in = downloadCompressed + ? new BufferedInputStream(new InflaterInputStream( + huc.getInputStream())) + : new BufferedInputStream(huc.getInputStream()); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + int len; + byte[] data = new byte[1024]; + while ((len = in.read(data, 0, 1024)) >= 0) { + baos.write(data, 0, len); + } + in.close(); + allData = baos.toByteArray(); + } + logger.fine("Downloaded " + fullUrl + " -> " + response + " (" + + (allData == null ? 0 : allData.length) + " bytes)"); + int receivedDescriptors = 0; + if (allData != null) { + if (resource.startsWith("/tor/status-vote/current/")) { + this.rdp.parse(allData); + receivedDescriptors = 1; + } else if (resource.startsWith("/tor/server/") || + resource.startsWith("/tor/extra/")) { + if (resource.equals("/tor/server/all")) { + this.lastDownloadedAllDescriptors.put(authority, + this.currentTimestamp); + } + String ascii = null; + try { + ascii = new String(allData, "US-ASCII"); + } catch (UnsupportedEncodingException e) { + /* No way that US-ASCII is not supported. */ + } + int start = -1, sig = -1, end = -1; + String startToken = resource.startsWith("/tor/server/") ? + "router " : "extra-info "; + String sigToken = "\nrouter-signature\n"; + String endToken = "\n-----END SIGNATURE-----\n"; + while (end < ascii.length()) { + start = ascii.indexOf(startToken, end); + if (start < 0) { + break; + } + sig = ascii.indexOf(sigToken, start); + if (sig < 0) { + break; + } + sig += sigToken.length(); + end = ascii.indexOf(endToken, sig); + if (end < 0) { + break; + } + end += endToken.length(); + byte[] descBytes = new byte[end - start]; + System.arraycopy(allData, start, descBytes, 0, end - start); + String digest = Hex.encodeHexString(DigestUtils.sha( + descBytes)); + this.rdp.parse(descBytes); + receivedDescriptors++; + } } } + return receivedDescriptors; } + /** + * Writes status files to disk and logs statistics about downloading + * relay descriptors in this execution. + */ public void writeFile() { + + /* Write missing descriptors file to disk. */ int missingConsensuses = 0, missingVotes = 0, missingServerDescriptors = 0, missingExtraInfoDescriptors = 0; try { @@ -581,8 +704,8 @@ public class RelayDescriptorDownloader { this.missingDescriptorsFile)); for (Map.Entry<String, String> e : this.missingDescriptors.entrySet()) { - String key = e.getKey(); - if (!e.getValue().equals("NA")) { + String key = e.getKey(), value = e.getValue(); + if (!value.equals("NA")) { /* Not missing. */ } else if (key.startsWith("consensus,")) { missingConsensuses++; @@ -593,7 +716,7 @@ public class RelayDescriptorDownloader { } else if (key.startsWith("extra,")) { missingExtraInfoDescriptors++; } - bw.write(e.getKey() + "," + e.getValue() + "\n"); + bw.write(key + "," + value + "\n"); } bw.close(); this.logger.fine("Finished writing file " @@ -603,35 +726,79 @@ public class RelayDescriptorDownloader { + this.missingDescriptorsFile.getAbsolutePath() + "!", e); } - dumpStats.append("During this execution, we added " + /* Write text file containing the directory authorities and when we + * last downloaded all server and extra-info descriptors from them to + * disk. */ + try { + this.logger.fine("Writing file " + + this.lastDownloadedAllDescriptorsFile.getAbsolutePath() + + "..."); + this.lastDownloadedAllDescriptorsFile.getParentFile().mkdirs(); + BufferedWriter bw = new BufferedWriter(new FileWriter( + this.lastDownloadedAllDescriptorsFile)); + for (Map.Entry<String, String> e : + this.lastDownloadedAllDescriptors.entrySet()) { + String authority = e.getKey(); + String lastDownloaded = e.getValue(); + bw.write(authority + "," + lastDownloaded + "\n"); + } + bw.close(); + this.logger.fine("Finished writing file " + + this.lastDownloadedAllDescriptorsFile.getAbsolutePath() + + "."); + } catch (IOException e) { + this.logger.log(Level.WARNING, "Failed writing " + + this.lastDownloadedAllDescriptorsFile.getAbsolutePath() + "!", + e); + } + + /* Log statistics about this execution. */ + this.logger.info("Finished downloading relay descriptors from the " + + "directory authorities."); + this.logger.info("At the beginning of this execution, we were " + + "missing " + oldMissingConsensuses + " consensus(es), " + + oldMissingVotes + " vote(s), " + oldMissingServerDescriptors + + " server descriptor(s), and " + oldMissingExtraInfoDescriptors + + " extra-info descriptor(s)."); + this.logger.info("During this execution, we added " + this.newMissingConsensuses + " consensus(es), " + this.newMissingVotes + " vote(s), " + this.newMissingServerDescriptors + " server descriptor(s), and " + this.newMissingExtraInfoDescriptors + " extra-info " - + "descriptor(s) to the missing list.\n"); - dumpStats.append("We attempted to download " + this.triedConsensuses - + " consensus(es), " + this.triedVotes + " vote(s), " - + this.triedServerDescriptors + " server descriptor(s), and " - + this.triedExtraInfoDescriptors + " extra-info descriptor(s) " - + "from the directory authorities.\n"); - dumpStats.append("Requests were sent to these directory " - + "authorities:"); - for (String dirSource : this.dirSources) { - dumpStats.append(" " + dirSource + "=" - + this.dirRequests.get(dirSource)); + + "descriptor(s) to the missing list, some of which we also " + + "requested and removed from the list again."); + this.logger.info("We requested " + this.requestedConsensuses + + " consensus(es), " + this.requestedVotes + " vote(s), " + + this.requestedMissingServerDescriptors + " missing server " + + "descriptor(s), " + this.requestedAllServerDescriptors + + " times all server descriptors, " + + this.requestedMissingExtraInfoDescriptors + " missing " + + "extra-info descriptor(s), and " + + this.requestedAllExtraInfoDescriptors + " times all extra-info " + + "descriptors from the directory authorities."); + StringBuilder sb = new StringBuilder(); + for (String authority : this.authorities) { + sb.append(" " + authority + "=" + + this.requestsByAuthority.get(authority)); } - dumpStats.append("\nWe successfully downloaded " + this.logger.info("We sent these numbers of requests to the directory " + + "authorities:" + sb.toString()); + this.logger.info("We successfully downloaded " + this.downloadedConsensuses + " consensus(es), " + this.downloadedVotes + " vote(s), " - + this.downloadedServerDescriptors + " server descriptor(s), and " - + this.downloadedExtraInfoDescriptors + " extra-info " - + "descriptor(s) from the directory authorities.\n"); - dumpStats.append("At the end of this execution, " - + "we are missing " + missingConsensuses + " consensus(es), " - + missingVotes + " vote(s), " + missingServerDescriptors - + " server descriptor(s), and " + missingExtraInfoDescriptors + + this.downloadedMissingServerDescriptors + " missing server " + + "descriptor(s), " + this.downloadedAllServerDescriptors + + " server descriptor(s) when downloading all descriptors, " + + this.downloadedMissingExtraInfoDescriptors + " missing " + + "extra-info descriptor(s) and " + + this.downloadedAllExtraInfoDescriptors + " extra-info " + + "descriptor(s) when downloading all descriptors."); + this.logger.info("At the end of this execution, we are missing " + + missingConsensuses + " consensus(es), " + missingVotes + + " vote(s), " + missingServerDescriptors + " server " + + "descriptor(s), and " + missingExtraInfoDescriptors + " extra-info descriptor(s), some of which we may try in the next " + "execution."); - this.logger.info(dumpStats.toString()); } } +

1 0

r24548: {arm} Using both the IP and port as sort attibutes when sorting by (arm/trunk/src/interface/connections)
by Damian Johnson 03 Apr '11

03 Apr '11

Author: atagar Date: 2011-04-03 06:36:26 +0000 (Sun, 03 Apr 2011) New Revision: 24548 Modified: arm/trunk/src/interface/connections/entries.py Log: Using both the IP and port as sort attibutes when sorting by listed IP entries. Modified: arm/trunk/src/interface/connections/entries.py =================================================================== --- arm/trunk/src/interface/connections/entries.py 2011-04-03 06:28:09 UTC (rev 24547) +++ arm/trunk/src/interface/connections/entries.py 2011-04-03 06:36:26 UTC (rev 24548) @@ -18,6 +18,9 @@ SortAttr.FINGERPRINT: "cyan", SortAttr.NICKNAME: "cyan", SortAttr.COUNTRY: "blue"} +# maximum number of ports a system can have +PORT_COUNT = 65536 + class ConnectionPanelEntry: """ Common parent for connection panel entries. This consists of a list of lines @@ -73,7 +76,10 @@ if attr == SortAttr.LISTING: if listingType == ListingType.IP_ADDRESS: - return self.getSortValue(SortAttr.IP_ADDRESS, listingType) + # uses the IP address as the primary value, and port as secondary + sortValue = self.getSortValue(SortAttr.IP_ADDRESS, listingType) * PORT_COUNT + sortValue += self.getSortValue(SortAttr.PORT, listingType) + return sortValue elif listingType == ListingType.HOSTNAME: return self.getSortValue(SortAttr.HOSTNAME, listingType) elif listingType == ListingType.FINGERPRINT:

1 0

r24547: {arm} Swapping the external and internal src address positions for (arm/trunk/src/interface/connections)
by Damian Johnson 03 Apr '11

03 Apr '11

Author: atagar Date: 2011-04-03 06:28:09 +0000 (Sun, 03 Apr 2011) New Revision: 24547 Modified: arm/trunk/src/interface/connections/connEntry.py Log: Swapping the external and internal src address positions for expanded entries (it was "<foreign> --> <internal> --> <external>" which is, of course, wrong) Modified: arm/trunk/src/interface/connections/connEntry.py =================================================================== --- arm/trunk/src/interface/connections/connEntry.py 2011-04-03 06:15:38 UTC (rev 24546) +++ arm/trunk/src/interface/connections/connEntry.py 2011-04-03 06:28:09 UTC (rev 24547) @@ -597,7 +597,14 @@ if addrDiffer and isExpansionType and isExpandedAddrVisible and self.includeExpandedIpAddr and CONFIG["features.connection.showColumn.expandedIp"]: # include the internal address in the src (extra 28 characters) internalAddress = self.local.getIpAddr() + localPort - src = "%-21s --> %s" % (internalAddress, src) + + # If this is an inbound connection then reverse ordering so it's: + # <foreign> --> <external> --> <internal> + # when the src and dst are swapped later + + if myType == Category.INBOUND: src = "%-21s --> %s" % (src, internalAddress) + else: src = "%-21s --> %s" % (internalAddress, src) + usedSpace += 28 etc = self.getEtcContent(width - usedSpace, listingType)

1 0

r24546: {arm} Rearanging config options to fill temporary gap in the help (arm/trunk/src/interface)
by Damian Johnson 03 Apr '11

03 Apr '11

Author: atagar Date: 2011-04-03 06:15:38 +0000 (Sun, 03 Apr 2011) New Revision: 24546 Modified: arm/trunk/src/interface/controller.py Log: Rearanging config options to fill temporary gap in the help dialog (the descriptor popup won't be available until 1.4.3). Modified: arm/trunk/src/interface/controller.py =================================================================== --- arm/trunk/src/interface/controller.py 2011-04-03 06:12:59 UTC (rev 24545) +++ arm/trunk/src/interface/controller.py 2011-04-03 06:15:38 UTC (rev 24546) @@ -1016,7 +1016,7 @@ resolverUtil = connections.getResolver("tor").overwriteResolver if resolverUtil == None: resolverUtil = "auto" - popup.addfstr(5, 2, "u: resolving utility (%s)" % resolverUtil) + popup.addfstr(3, 41, "u: resolving utility (%s)" % resolverUtil) pageOverrideKeys = (ord('l'), ord('s'), ord('u')) elif page == 3:

1 0

r24545: {arm} Last change accidently broke torrc validation - fixing this. (in arm/trunk/src: interface util)
by Damian Johnson 03 Apr '11

03 Apr '11

Author: atagar Date: 2011-04-03 06:12:59 +0000 (Sun, 03 Apr 2011) New Revision: 24545 Modified: arm/trunk/src/interface/controller.py arm/trunk/src/util/torConfig.py Log: Last change accidently broke torrc validation - fixing this. Modified: arm/trunk/src/interface/controller.py =================================================================== --- arm/trunk/src/interface/controller.py 2011-04-03 05:55:58 UTC (rev 24544) +++ arm/trunk/src/interface/controller.py 2011-04-03 06:12:59 UTC (rev 24545) @@ -1675,7 +1675,7 @@ # display a popup for saving the current configuration panel.CURSES_LOCK.acquire() try: - configLines = torConfig.getCustomOptions() + configLines = torConfig.getCustomOptions(True) # lists event types popup = panels["popup"] Modified: arm/trunk/src/util/torConfig.py =================================================================== --- arm/trunk/src/util/torConfig.py 2011-04-03 05:55:58 UTC (rev 24544) +++ arm/trunk/src/util/torConfig.py 2011-04-03 06:12:59 UTC (rev 24545) @@ -351,9 +351,13 @@ return tuple(MULTILINE_PARAM) -def getCustomOptions(): +def getCustomOptions(includeValue = False): """ Provides the torrc parameters that differ from their defaults. + + Arguments: + includeValue - provides the current value with results if true, otherwise + this just contains the options """ configText = torTools.getConn().getInfo("config-text", "").strip() @@ -370,7 +374,8 @@ try: configLines.remove("Log notice stdout") except ValueError: pass - return configLines + if includeValue: return configLines + else: return [line[:line.find(" ")] for line in configLines] def validate(contents = None): """

1 0

r24544: {arm} Dropping the pointless "Log notice stdout" config-text entry (in arm/trunk/src: interface util)
by Damian Johnson 03 Apr '11

03 Apr '11

Author: atagar Date: 2011-04-03 05:55:58 +0000 (Sun, 03 Apr 2011) New Revision: 24544 Modified: arm/trunk/src/interface/controller.py arm/trunk/src/util/torConfig.py Log: Dropping the pointless "Log notice stdout" config-text entry when writting the custom torrc. Modified: arm/trunk/src/interface/controller.py =================================================================== --- arm/trunk/src/interface/controller.py 2011-04-03 05:23:06 UTC (rev 24543) +++ arm/trunk/src/interface/controller.py 2011-04-03 05:55:58 UTC (rev 24544) @@ -1675,8 +1675,7 @@ # display a popup for saving the current configuration panel.CURSES_LOCK.acquire() try: - configText = torTools.getConn().getInfo("config-text", "").strip() - configLines = configText.split("\n") + configLines = torConfig.getCustomOptions() # lists event types popup = panels["popup"] @@ -1774,7 +1773,7 @@ # saves the configuration to the file configFile = open(configLocation, "w") - configFile.write(configText) + configFile.write("\n".join(configLines)) configFile.close() # reloads the cached torrc if overwriting it Modified: arm/trunk/src/util/torConfig.py =================================================================== --- arm/trunk/src/util/torConfig.py 2011-04-03 05:23:06 UTC (rev 24543) +++ arm/trunk/src/util/torConfig.py 2011-04-03 05:55:58 UTC (rev 24544) @@ -353,17 +353,24 @@ def getCustomOptions(): """ - Provides the set of torrc parameters that differ from their defaults. + Provides the torrc parameters that differ from their defaults. """ - customOptions, conn = set(), torTools.getConn() - configTextQuery = conn.getInfo("config-text", "").strip().split("\n") + configText = torTools.getConn().getInfo("config-text", "").strip() + configLines = configText.split("\n") - for entry in configTextQuery: - # tor provides a Log entry even if it matches the default - if entry != "Log notice stdout": - customOptions.add(entry[:entry.find(" ")]) - return customOptions + # removes any duplicates + configLines = list(set(configLines)) + + # The "GETINFO config-text" query only provides options that differ + # from Tor's defaults with the exception of its Log entry which, even + # if undefined, returns "Log notice stdout" as per: + # https://trac.torproject.org/projects/tor/ticket/2362 + + try: configLines.remove("Log notice stdout") + except ValueError: pass + + return configLines def validate(contents = None): """

1 0

r24543: {arm} Correcting page numbering for when either the new and/or old (arm/trunk/src/interface)
by Damian Johnson 03 Apr '11

03 Apr '11

Author: atagar Date: 2011-04-03 05:23:06 +0000 (Sun, 03 Apr 2011) New Revision: 24543 Modified: arm/trunk/src/interface/controller.py Log: Correcting page numbering for when either the new and/or old connection panel aren't being displayed. Modified: arm/trunk/src/interface/controller.py =================================================================== --- arm/trunk/src/interface/controller.py 2011-04-03 05:04:10 UTC (rev 24542) +++ arm/trunk/src/interface/controller.py 2011-04-03 05:23:06 UTC (rev 24543) @@ -120,7 +120,11 @@ currentPage = self.page pageCount = len(PAGES) - if self.isBlindMode: + if not CONFIG["features.connection.newPanel"]: + if currentPage >= 3: currentPage -= 1 + pageCount -= 1 + + if self.isBlindMode or not CONFIG["features.connection.oldPanel"]: if currentPage >= 2: currentPage -= 1 pageCount -= 1

1 0

r24542: {arm} Fetching the exit policy from the consensus if available, an (arm/trunk/src/interface/connections)
by Damian Johnson 03 Apr '11

03 Apr '11

Author: atagar Date: 2011-04-03 05:04:10 +0000 (Sun, 03 Apr 2011) New Revision: 24542 Modified: arm/trunk/src/interface/connections/connEntry.py Log: Fetching the exit policy from the consensus if available, and otherwise falls back to the descriptor. Without this arm was encountering a parsing error with older Tor versions (encountered on 0.2.1.29). Modified: arm/trunk/src/interface/connections/connEntry.py =================================================================== --- arm/trunk/src/interface/connections/connEntry.py 2011-04-03 04:16:00 UTC (rev 24541) +++ arm/trunk/src/interface/connections/connEntry.py 2011-04-03 05:04:10 UTC (rev 24542) @@ -702,14 +702,32 @@ _, nickname, _, _, pubDate, pubTime, _, orPort, dirPort = firstLineComp[:9] else: nickname, pubDate, pubTime, orPort, dirPort = "", "", "", "", "" - flags = nsLines[1][2:] - microExit = nsLines[3][2:] + flags = "unknown" + if len(nsLines) >= 2 and nsLines[1].startswith("s "): + flags = nsLines[1][2:] + # The network status exit policy doesn't exist for older tor versions. + # If unavailble we'll need the full exit policy which is on the + # descriptor (if that's available). + + exitPolicy = "unknown" + if len(nsLines) >= 4 and nsLines[3].startswith("p "): + exitPolicy = nsLines[3][2:].replace(",", ", ") + elif descEntry: + # the descriptor has an individual line for each entry in the exit policy + exitPolicyEntries = [] + + for line in descEntry.split("\n"): + if line.startswith("accept") or line.startswith("reject"): + exitPolicyEntries.append(line.strip()) + + exitPolicy = ", ".join(exitPolicyEntries) + dirPortLabel = "" if dirPort == "0" else "dirport: %s" % dirPort lines[2] = "nickname: %-25s orport: %-10s %s" % (nickname, orPort, dirPortLabel) lines[3] = "published: %s %s" % (pubDate, pubTime) lines[4] = "flags: %s" % flags.replace(" ", ", ") - lines[5] = "exit policy: %s" % microExit.replace(",", ", ") + lines[5] = "exit policy: %s" % exitPolicy if descEntry: torVersion, platform, contact = "", "", ""

1 0

r24541: {arm} Only flagging inbound connections as possibly being private (arm/trunk/src/interface/connections)
by Damian Johnson 03 Apr '11

03 Apr '11

Author: atagar Date: 2011-04-03 04:16:00 +0000 (Sun, 03 Apr 2011) New Revision: 24541 Modified: arm/trunk/src/interface/connections/connEntry.py Log: Only flagging inbound connections as possibly being private if we're a guard or bridge. Modified: arm/trunk/src/interface/connections/connEntry.py =================================================================== --- arm/trunk/src/interface/connections/connEntry.py 2011-04-03 04:00:48 UTC (rev 24540) +++ arm/trunk/src/interface/connections/connEntry.py 2011-04-03 04:16:00 UTC (rev 24541) @@ -361,12 +361,13 @@ myType = self.getType() if myType == Category.INBOUND: - # if the connection doesn't belong to a known relay then it might be - # client traffic + # if we're a guard or bridge and the connection doesn't belong to a + # known relay then it might be client traffic conn = torTools.getConn() - allMatches = conn.getRelayFingerprint(self.foreign.getIpAddr(), getAllMatches = True) - return allMatches == [] + if "Guard" in conn.getMyFlags() or conn.getOption("BridgeRelay") == "1": + allMatches = conn.getRelayFingerprint(self.foreign.getIpAddr(), getAllMatches = True) + return allMatches == [] elif myType == Category.EXIT: # DNS connections exiting us aren't private (since they're hitting our # resolvers). Everything else, however, is.

1 0

r24540: {arm} Giving a warning and prematurely aborting geoip lookups when (in arm/trunk: . src/util)
by Damian Johnson 03 Apr '11

03 Apr '11

Author: atagar Date: 2011-04-03 04:00:48 +0000 (Sun, 03 Apr 2011) New Revision: 24540 Modified: arm/trunk/armrc.sample arm/trunk/src/util/torTools.py Log: Giving a warning and prematurely aborting geoip lookups when the database is unavailble. Modified: arm/trunk/armrc.sample =================================================================== --- arm/trunk/armrc.sample 2011-04-02 23:51:15 UTC (rev 24539) +++ arm/trunk/armrc.sample 2011-04-03 04:00:48 UTC (rev 24540) @@ -271,6 +271,7 @@ log.cursesColorSupport INFO log.bsdJailFound INFO log.unknownBsdJailId WARN +log.geoipUnavailable WARN log.stats.failedProcResolution DEBUG log.stats.procResolutionFailover INFO log.stats.failedPsResolution INFO Modified: arm/trunk/src/util/torTools.py =================================================================== --- arm/trunk/src/util/torTools.py 2011-04-02 23:51:15 UTC (rev 24539) +++ arm/trunk/src/util/torTools.py 2011-04-03 04:00:48 UTC (rev 24540) @@ -90,6 +90,10 @@ "NS": "information related to the consensus will grow stale", "NEWCONSENSUS": "information related to the consensus will grow stale"} +# number of sequential attempts before we decide that the Tor geoip database +# is unavailable +GEOIP_FAILURE_THRESHOLD = 5 + # provides int -> str mappings for torctl event runlevels TORCTL_RUNLEVELS = dict([(val, key) for (key, val) in TorUtil.loglevels.items()]) @@ -310,6 +314,9 @@ # directs TorCtl to notify us of events TorUtil.logger = self TorUtil.loglevel = "DEBUG" + + # tracks the number of sequential geoip lookup failures + self.geoipFailureCount = 0 def init(self, conn=None): """ @@ -445,6 +452,8 @@ self.connLock.acquire() + isGeoipRequest = param.startswith("ip-to-country/") + # checks if this is an arg caching covers isCacheArg = param in CACHE_ARGS @@ -462,13 +471,23 @@ if isCacheArg and cachedValue: result = cachedValue isFromCache = True + elif isGeoipRequest and self.geoipFailureCount == GEOIP_FAILURE_THRESHOLD: + # the geoip database aleady looks to be unavailable - abort the request + raisedExc = TorCtl.ErrorReply("Tor geoip database is unavailable.") else: try: getInfoVal = self.conn.get_info(param)[param] if getInfoVal != None: result = getInfoVal + if isGeoipRequest: self.geoipFailureCount = 0 except (socket.error, TorCtl.ErrorReply, TorCtl.TorCtlClosed), exc: if type(exc) == TorCtl.TorCtlClosed: self.close() raisedExc = exc + + if isGeoipRequest: + self.geoipFailureCount += 1 + + if self.geoipFailureCount == GEOIP_FAILURE_THRESHOLD: + log.log(CONFIG["log.geoipUnavailable"], "Tor geoip database is unavailable.") if isCacheArg and result and not isFromCache: self._cachedParam[param] = result

1 0