commit 0c96c1a6410dd93545692de70f66be2357f0dcbd Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Mon Dec 12 11:29:03 2011 +0100 Use the new metrics-lib for downloading consensuses and votes. Don't use the lib for parsing consensuses and votes. That part is not implemented yet. --- build.xml | 1 + src/org/torproject/doctor/Downloader.java | 261 ++++++----------------------- 2 files changed, 52 insertions(+), 210 deletions(-) diff --git a/build.xml b/build.xml index 2db0db4..feedb12 100755 --- a/build.xml +++ b/build.xml @@ -5,6 +5,7 @@ <path id="classpath"> <pathelement path="${classes}"/> <pathelement location="${libs}/commons-codec-1.4.jar"/> + <pathelement location="${libs}/descriptor.jar"/> </path> <target name="init"> <mkdir dir="${classes}"/> diff --git a/src/org/torproject/doctor/Downloader.java b/src/org/torproject/doctor/Downloader.java index a2e6db5..a82ac8d 100755 --- a/src/org/torproject/doctor/Downloader.java +++ b/src/org/torproject/doctor/Downloader.java @@ -7,241 +7,82 @@ import java.net.*; import java.text.*; import java.util.*; import java.util.zip.*; +import org.torproject.descriptor.*; /* Download the latest network status consensus and corresponding * votes. */ public class Downloader { - /* List of directory authorities to download consensuses and votes - * from. */ - private SortedMap<String, String> authorities = - new TreeMap<String, String>(); - public Downloader() { - this.authorities.put("gabelmoo", "212.112.245.170"); - this.authorities.put("tor26", "86.59.21.38"); - this.authorities.put("ides", "216.224.124.114:9030"); - this.authorities.put("maatuska", "213.115.239.118:443"); - this.authorities.put("dannenberg", "193.23.244.244"); - this.authorities.put("urras", "208.83.223.34:443"); - this.authorities.put("moria1", "128.31.0.34:9131"); - this.authorities.put("dizum", "194.109.206.212"); - } - /* Download a new consensus and corresponding votes. */ public void downloadFromAuthorities() { - this.downloadConsensus(); - if (!this.downloadedConsensuses.isEmpty()) { - this.parseConsensusToFindReferencedVotes(); - this.downloadReferencedVotes(); - } - } - /* Download the most recent consensus from all authorities. */ - private List<Download> downloadedConsensuses = - new ArrayList<Download>(); - private void downloadConsensus() { - Map<String, String> urls = new HashMap<String, String>(); - for (Map.Entry<String, String> e : this.authorities.entrySet()) { - String nickname = e.getKey(); - String address = e.getValue(); - String resource = "/tor/status-vote/current/consensus.z"; - String fullUrl = "http://" + address + resource; - urls.put(fullUrl, nickname); - } - this.downloadedConsensuses = this.downloadFromAuthority(urls); - if (this.downloadedConsensuses.isEmpty()) { - System.err.println("Could not download consensus from any of the " - + "directory authorities. Ignoring."); - } - } + RelayDescriptorDownloader downloader = + DescriptorSourceFactory.createRelayDescriptorDownloader(); - /* Downloads a consensus or vote in a separate thread that can be - * interrupted after a timeout. */ - private static class DownloadRunnable implements Runnable { - Thread mainThread; - String nickname; - String url; - String response; - boolean finished = false; - long requestStart = 0L, requestEnd = 0L; - public DownloadRunnable(String nickname, String url) { - this.mainThread = Thread.currentThread(); - this.nickname = nickname; - this.url = url; - } - public void run() { - this.requestStart = System.currentTimeMillis(); - try { - URL u = new URL(this.url); - HttpURLConnection huc = (HttpURLConnection) u.openConnection(); - huc.setRequestMethod("GET"); - huc.connect(); - int responseCode = huc.getResponseCode(); - if (responseCode == 200) { - BufferedInputStream in = new BufferedInputStream( - new InflaterInputStream(huc.getInputStream())); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - int len; - byte[] data = new byte[1024]; - while (!this.finished && - (len = in.read(data, 0, 1024)) >= 0) { - baos.write(data, 0, len); - } - if (this.finished) { - return; - } - in.close(); - byte[] allData = baos.toByteArray(); - this.response = new String(allData); - this.finished = true; - this.mainThread.interrupt(); - } - } catch (IOException e) { - /* Can't do much except leaving this.response at null. */ - } - this.finished = true; - this.requestEnd = System.currentTimeMillis(); - } - public Download getDownload() { - Download result = null; - if (this.response != null) { - result = new Download(this.nickname, this.url, this.response, - this.requestStart, this.requestEnd - this.requestStart); - } - return result; - } - } + downloader.addDirectoryAuthority("gabelmoo", "212.112.245.170", 80); + downloader.addDirectoryAuthority("tor26", "86.59.21.38", 80); + downloader.addDirectoryAuthority("ides", "216.224.124.114", 9030); + downloader.addDirectoryAuthority("maatuska", "213.115.239.118", 443); + downloader.addDirectoryAuthority("dannenberg", "193.23.244.244", 80); + downloader.addDirectoryAuthority("urras", "208.83.223.34", 443); + downloader.addDirectoryAuthority("moria1", "128.31.0.34", 9131); + downloader.addDirectoryAuthority("dizum", "194.109.206.212", 80); - /* Download one or more consensuses or votes from one or more directory - * authorities using a timeout of 60 seconds. */ - private List<Download> downloadFromAuthority(Map<String, String> urls) { - Set<DownloadRunnable> downloadRunnables = - new HashSet<DownloadRunnable>(); - for (Map.Entry<String, String> e : urls.entrySet()) { - String url = e.getKey(); - String nickname = e.getValue(); - DownloadRunnable downloadRunnable = new DownloadRunnable(nickname, - url); - downloadRunnables.add(downloadRunnable); - new Thread(downloadRunnable).start(); - } - long started = System.currentTimeMillis(), sleep; - while ((sleep = started + 60L * 1000L - System.currentTimeMillis()) - > 0L) { - try { - Thread.sleep(sleep); - } catch (InterruptedException e) { - /* Do nothing. */ - } - boolean unfinished = false; - for (DownloadRunnable downloadRunnable : downloadRunnables) { - if (!downloadRunnable.finished) { - unfinished = true; - break; - } - } - if (!unfinished) { - break; - } - } - List<Download> responses = new ArrayList<Download>(); - for (DownloadRunnable downloadRunnable : downloadRunnables) { - Download download = downloadRunnable.getDownload(); - if (download != null) { - responses.add(download); - } - downloadRunnable.finished = true; - } - return responses; - } + downloader.setIncludeCurrentConsensusFromAllDirectoryAuthorities(); + downloader.setIncludeCurrentReferencedVotes(); - /* Date-time formats to parse and format timestamps. */ - private static SimpleDateFormat dateTimeFormat; - static { - dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - } + downloader.setRequestTimeout(60L * 1000L); - /* Parse the downloaded consensus to find fingerprints of directory - * authorities publishing the corresponding votes. */ - private SortedSet<String> fingerprints = new TreeSet<String>(); - private void parseConsensusToFindReferencedVotes() { - for (Download download : this.downloadedConsensuses) { - String downloadedConsensus = download.getResponseString(); - try { - BufferedReader br = new BufferedReader(new StringReader( - downloadedConsensus)); - String line; - while ((line = br.readLine()) != null) { - if (line.startsWith("valid-after ")) { - try { - long validAfterMillis = dateTimeFormat.parse(line.substring( - "valid-after ".length())).getTime(); - if (validAfterMillis + 60L * 60L * 1000L < - System.currentTimeMillis()) { - /* Consensus is more than 1 hour old. We won't be able to - * download the corresponding votes anymore. */ - break; - } - } catch (ParseException e) { - System.err.println("Could not parse valid-after timestamp " - + "in line '" + line + "' of a downloaded consensus. " - + "Not downloading votes."); - break; - } - } else if (line.startsWith("dir-source ")) { - String[] parts = line.split(" "); - if (parts.length < 3) { - System.err.println("Bad dir-source line '" + line - + "' in downloaded consensus. Skipping."); - continue; - } - String nickname = parts[1]; - if (nickname.endsWith("-legacy")) { - continue; - } - String fingerprint = parts[2]; - this.fingerprints.add(fingerprint); - } + Iterator<DescriptorRequest> descriptorRequests = + downloader.downloadDescriptors(); + while (descriptorRequests.hasNext()) { + DescriptorRequest request = descriptorRequests.next(); + String authority = request.getDirectoryNickname(); + String requestUrl = request.getRequestUrl(); + long requestStart = request.getRequestStart(); + long fetchTime = request.getRequestEnd() + - request.getRequestStart(); + if (request.globalTimeoutHasExpired()) { + System.err.println("Global timeout has expired. Exiting."); + System.exit(1); + } else if (!request.requestTimeoutHasExpired()) { + if (request.getDescriptors().isEmpty()) { + /* No response. We'll realize later on if we're missing a + * consensus or vote. */ + continue; + } else if (request.getDescriptors().size() > 1) { + System.out.println("Response contains more than 1 " + + "descriptor. Considering only the first."); } - br.close(); - } catch (IOException e) { - System.err.println("Could not parse consensus to find referenced " - + "votes in it. Skipping."); - } - } - } - - /* Download the votes published by directory authorities listed in the - * consensus. */ - private List<Download> downloadedVotes = new ArrayList<Download>(); - private void downloadReferencedVotes() { - for (String fingerprint : this.fingerprints) { - String downloadedVote = null; - List<String> authorities = new ArrayList<String>( - this.authorities.values()); - Collections.shuffle(authorities); - for (String authority : authorities) { - if (downloadedVote != null) { - break; + Descriptor downloadedDescriptor = request.getDescriptors().get(0); + String response = new String(request.getDescriptors().get(0). + getRawDescriptorBytes()); + Download download = new Download(authority, requestUrl, response, + requestStart, fetchTime); + if (downloadedDescriptor instanceof + RelayNetworkStatusConsensus) { + this.downloadedConsensuses.add(download); + } else if (downloadedDescriptor instanceof + RelayNetworkStatusVote) { + this.downloadedVotes.add(download); + } else { + System.err.println("Did not expect a descriptor of type " + + downloadedDescriptor.getClass() + ". Ignoring."); } - String resource = "/tor/status-vote/current/" + fingerprint - + ".z"; - String fullUrl = "http://" + authority + resource; - Map<String, String> urls = new HashMap<String, String>(); - urls.put(fullUrl, authority); - this.downloadedVotes.addAll(this.downloadFromAuthority(urls)); } } } /* Return the previously downloaded (unparsed) consensus string by * authority nickname. */ + private List<Download> downloadedConsensuses = + new ArrayList<Download>(); public List<Download> getConsensuses() { return this.downloadedConsensuses; } /* Return the previously downloaded (unparsed) vote strings. */ + private List<Download> downloadedVotes = new ArrayList<Download>(); public List<Download> getVotes() { return this.downloadedVotes; }