[or-cvs] [ernie/master 1/2] Fix relay descriptor download logic.
karsten at torproject.org
karsten at torproject.org
Tue Apr 20 13:51:13 UTC 2010
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Tue, 20 Apr 2010 14:20:49 +0200
Subject: Fix relay descriptor download logic.
Commit: c5910c724ec67438813fffe8bf837a308d43ad01
---
src/Main.java | 3 +-
src/RelayDescriptorDownloader.java | 117 ++++++++++++++++++++----------------
2 files changed, 68 insertions(+), 52 deletions(-)
diff --git a/src/Main.java b/src/Main.java
index e7adcb5..0d01f03 100644
--- a/src/Main.java
+++ b/src/Main.java
@@ -79,7 +79,8 @@ public class Main {
boolean downloadAllServerDescriptors = aw != null ||
sdsfh != null || rddi != null;
boolean downloadAllExtraInfos = aw != null;
- Set<String> downloadDescriptorsForRelays = directories;
+ Set<String> downloadDescriptorsForRelays = bsfh != null ||
+ dsfh != null ? directories : new HashSet<String>();
rdd = new RelayDescriptorDownloader(rdp, dirSources,
downloadCurrentConsensus, downloadCurrentVotes,
downloadAllServerDescriptors, downloadAllExtraInfos,
diff --git a/src/RelayDescriptorDownloader.java b/src/RelayDescriptorDownloader.java
index f9182b3..5a5ac65 100644
--- a/src/RelayDescriptorDownloader.java
+++ b/src/RelayDescriptorDownloader.java
@@ -361,8 +361,11 @@ public class RelayDescriptorDownloader {
/* URLs of descriptors we want to download. */
SortedSet<String> urls = new TreeSet<String>();
- /* URLs of descriptors we have downloaded or at least tried to
- * download. */
+ /* Complete URLs of authorities and descriptors we have downloaded or
+ * tried to download. Ensures that we're not attempting to download
+ * the same thing from an authority more than once. There are edge
+ * cases when an authority returns a valid response to something we
+ * asked for, but which is not what we wanted (e.g. old consensus). */
SortedSet<String> downloaded = new TreeSet<String>();
/* We might need more than one iteration for downloading descriptors,
@@ -403,7 +406,6 @@ public class RelayDescriptorDownloader {
}
}
}
- urls.removeAll(downloaded);
/* Stop if we don't have (new) URLs to download. */
if (urls.isEmpty()) {
@@ -426,7 +428,7 @@ public class RelayDescriptorDownloader {
StringBuilder sb = new StringBuilder("Downloading " + urls.size()
+ " descriptors:");
for (String url : urls) {
- sb.append(url + "\n");
+ sb.append("\n" + url);
}
this.logger.fine(sb.toString());
@@ -438,62 +440,75 @@ public class RelayDescriptorDownloader {
SortedSet<String> currentDirSources =
new TreeSet<String>(remainingDirSources);
SortedSet<String> retryUrls = new TreeSet<String>();
+ int numDownloaded = 0;
while (!currentDirSources.isEmpty() && !urls.isEmpty()) {
String authority = currentDirSources.first();
String url = urls.first();
- try {
- URL u = new URL("http://" + authority + url);
- HttpURLConnection huc =
- (HttpURLConnection) u.openConnection();
- huc.setRequestMethod("GET");
- huc.connect();
- int response = huc.getResponseCode();
- logger.finer("Downloading http://" + authority + url + " -> "
- + response);
- if (response == 200) {
- BufferedInputStream in = new BufferedInputStream(
- huc.getInputStream());
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- int len;
- byte[] data = new byte[1024];
- while ((len = in.read(data, 0, 1024)) >= 0) {
- baos.write(data, 0, len);
- }
- in.close();
- byte[] allData = baos.toByteArray();
- rdp.parse(allData);
- if (url.endsWith("consensus")) {
- this.downloadedConsensuses++;
- } else if (url.contains("status-vote")) {
- this.downloadedVotes++;
- } else if (url.contains("server")) {
- this.downloadedServerDescriptors++;
- } else if (url.contains("extra")) {
- this.downloadedExtraInfoDescriptors++;
+ String fullUrl = "http://" + authority + url;
+ byte[] allData = null;
+ if (!downloaded.contains(fullUrl)) {
+ downloaded.add(fullUrl);
+ numDownloaded++;
+ try {
+ URL u = new URL(fullUrl);
+ HttpURLConnection huc =
+ (HttpURLConnection) u.openConnection();
+ huc.setRequestMethod("GET");
+ huc.connect();
+ int response = huc.getResponseCode();
+ logger.fine("Downloading http://" + authority + url + " -> "
+ + response);
+ if (response == 200) {
+ BufferedInputStream in = new BufferedInputStream(
+ huc.getInputStream());
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ int len;
+ byte[] data = new byte[1024];
+ while ((len = in.read(data, 0, 1024)) >= 0) {
+ baos.write(data, 0, len);
+ }
+ in.close();
+ allData = baos.toByteArray();
+ rdp.parse(allData);
+ if (url.endsWith("consensus")) {
+ this.downloadedConsensuses++;
+ } else if (url.contains("status-vote")) {
+ this.downloadedVotes++;
+ } else if (url.contains("server")) {
+ this.downloadedServerDescriptors++;
+ } else if (url.contains("extra")) {
+ this.downloadedExtraInfoDescriptors++;
+ }
}
- } else {
- retryUrls.add(url);
- }
- urls.remove(url);
- if (urls.isEmpty()) {
+ } catch (IOException e) {
+ remainingDirSources.remove(authority);
currentDirSources.remove(authority);
- urls.addAll(retryUrls);
- retryUrls.clear();
+ if (!remainingDirSources.isEmpty()) {
+ logger.log(Level.FINE, "Failed downloading from "
+ + authority + "!", e);
+ } else {
+ logger.log(Level.WARNING, "Failed downloading from "
+ + authority + "! We have no authorities left to download "
+ + "from!", e);
+ }
}
- } catch (IOException e) {
- remainingDirSources.remove(authority);
+ }
+ if (allData == null) {
+ retryUrls.add(url);
+ }
+ urls.remove(url);
+ if (urls.isEmpty()) {
currentDirSources.remove(authority);
- if (!remainingDirSources.isEmpty()) {
- logger.log(Level.FINE, "Failed downloading from "
- + authority + "!", e);
- } else {
- logger.log(Level.WARNING, "Failed downloading from "
- + authority + "! We have no authorities left to download "
- + "from!", e);
- }
+ urls.addAll(retryUrls);
+ retryUrls.clear();
}
}
- downloaded.addAll(urls);
+
+ /* If we haven't downloaded a single descriptor in this iteration,
+ * we cannot have learned something new, so we're done. */
+ if (numDownloaded < 1) {
+ break;
+ }
}
}
--
1.6.5
More information about the tor-commits
mailing list