commit 45359de3263c928c6cd7e8f2e3916bb565eeafbd Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Wed Jul 23 18:03:28 2014 +0200 Make all inner classes first-class classes. --- .../onionoo/server/HttpServletRequestWrapper.java | 24 ++ .../onionoo/server/HttpServletResponseWrapper.java | 30 ++ src/org/torproject/onionoo/server/NodeIndex.java | 142 +++++++ src/org/torproject/onionoo/server/NodeIndexer.java | 134 ------- .../torproject/onionoo/server/ResourceServlet.java | 39 -- .../onionoo/updater/DescriptorDownloader.java | 178 +++++++++ .../onionoo/updater/DescriptorHistory.java | 12 + .../onionoo/updater/DescriptorQueue.java | 221 +++++++++++ .../onionoo/updater/DescriptorSource.java | 396 -------------------- .../onionoo/updater/RdnsLookupRequest.java | 43 +++ .../onionoo/updater/RdnsLookupWorker.java | 55 +++ .../onionoo/updater/ReverseDomainNameResolver.java | 87 +---- .../torproject/onionoo/ResourceServletTest.java | 4 +- 13 files changed, 715 insertions(+), 650 deletions(-) diff --git a/src/org/torproject/onionoo/server/HttpServletRequestWrapper.java b/src/org/torproject/onionoo/server/HttpServletRequestWrapper.java new file mode 100644 index 0000000..3349acd --- /dev/null +++ b/src/org/torproject/onionoo/server/HttpServletRequestWrapper.java @@ -0,0 +1,24 @@ +/* Copyright 2011, 2012 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.server; + +import java.util.Map; + +import javax.servlet.http.HttpServletRequest; + +public class HttpServletRequestWrapper { + private HttpServletRequest request; + protected HttpServletRequestWrapper(HttpServletRequest request) { + this.request = request; + } + protected String getRequestURI() { + return this.request.getRequestURI(); + } + @SuppressWarnings("rawtypes") + protected Map getParameterMap() { + return this.request.getParameterMap(); + } + protected String[] getParameterValues(String parameterKey) { + return this.request.getParameterValues(parameterKey); + } +} \ No newline at end of file diff --git a/src/org/torproject/onionoo/server/HttpServletResponseWrapper.java b/src/org/torproject/onionoo/server/HttpServletResponseWrapper.java new file mode 100644 index 0000000..58d9f03 --- /dev/null +++ b/src/org/torproject/onionoo/server/HttpServletResponseWrapper.java @@ -0,0 +1,30 @@ +/* Copyright 2011, 2012 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.server; + +import java.io.IOException; +import java.io.PrintWriter; + +import javax.servlet.http.HttpServletResponse; + +public class HttpServletResponseWrapper { + private HttpServletResponse response = null; + protected HttpServletResponseWrapper(HttpServletResponse response) { + this.response = response; + } + protected void sendError(int errorStatusCode) throws IOException { + this.response.sendError(errorStatusCode); + } + protected void setHeader(String headerName, String headerValue) { + this.response.setHeader(headerName, headerValue); + } + protected void setContentType(String contentType) { + this.response.setContentType(contentType); + } + protected void setCharacterEncoding(String characterEncoding) { + this.response.setCharacterEncoding(characterEncoding); + } + protected PrintWriter getWriter() throws IOException { + return this.response.getWriter(); + } +} \ No newline at end of file diff --git a/src/org/torproject/onionoo/server/NodeIndex.java b/src/org/torproject/onionoo/server/NodeIndex.java new file mode 100644 index 0000000..7b95d2e --- /dev/null +++ b/src/org/torproject/onionoo/server/NodeIndex.java @@ -0,0 +1,142 @@ +package org.torproject.onionoo.server; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; + +import org.torproject.onionoo.docs.SummaryDocument; + +class NodeIndex { + + private String relaysPublishedString; + public void setRelaysPublishedString(String relaysPublishedString) { + this.relaysPublishedString = relaysPublishedString; + } + public String getRelaysPublishedString() { + return relaysPublishedString; + } + + private String bridgesPublishedString; + public void setBridgesPublishedString(String bridgesPublishedString) { + this.bridgesPublishedString = bridgesPublishedString; + } + public String getBridgesPublishedString() { + return bridgesPublishedString; + } + + private List<String> relaysByConsensusWeight; + public void setRelaysByConsensusWeight( + List<String> relaysByConsensusWeight) { + this.relaysByConsensusWeight = relaysByConsensusWeight; + } + public List<String> getRelaysByConsensusWeight() { + return relaysByConsensusWeight; + } + + + private Map<String, SummaryDocument> relayFingerprintSummaryLines; + public void setRelayFingerprintSummaryLines( + Map<String, SummaryDocument> relayFingerprintSummaryLines) { + this.relayFingerprintSummaryLines = relayFingerprintSummaryLines; + } + public Map<String, SummaryDocument> getRelayFingerprintSummaryLines() { + return this.relayFingerprintSummaryLines; + } + + private Map<String, SummaryDocument> bridgeFingerprintSummaryLines; + public void setBridgeFingerprintSummaryLines( + Map<String, SummaryDocument> bridgeFingerprintSummaryLines) { + this.bridgeFingerprintSummaryLines = bridgeFingerprintSummaryLines; + } + public Map<String, SummaryDocument> getBridgeFingerprintSummaryLines() { + return this.bridgeFingerprintSummaryLines; + } + + private Map<String, Set<String>> relaysByCountryCode = null; + public void setRelaysByCountryCode( + Map<String, Set<String>> relaysByCountryCode) { + this.relaysByCountryCode = relaysByCountryCode; + } + public Map<String, Set<String>> getRelaysByCountryCode() { + return relaysByCountryCode; + } + + private Map<String, Set<String>> relaysByASNumber = null; + public void setRelaysByASNumber( + Map<String, Set<String>> relaysByASNumber) { + this.relaysByASNumber = relaysByASNumber; + } + public Map<String, Set<String>> getRelaysByASNumber() { + return relaysByASNumber; + } + + private Map<String, Set<String>> relaysByFlag = null; + public void setRelaysByFlag(Map<String, Set<String>> relaysByFlag) { + this.relaysByFlag = relaysByFlag; + } + public Map<String, Set<String>> getRelaysByFlag() { + return relaysByFlag; + } + + private Map<String, Set<String>> bridgesByFlag = null; + public void setBridgesByFlag(Map<String, Set<String>> bridgesByFlag) { + this.bridgesByFlag = bridgesByFlag; + } + public Map<String, Set<String>> getBridgesByFlag() { + return bridgesByFlag; + } + + private Map<String, Set<String>> relaysByContact = null; + public void setRelaysByContact( + Map<String, Set<String>> relaysByContact) { + this.relaysByContact = relaysByContact; + } + public Map<String, Set<String>> getRelaysByContact() { + return relaysByContact; + } + + private Map<String, Set<String>> relaysByFamily = null; + public void setRelaysByFamily(Map<String, Set<String>> relaysByFamily) { + this.relaysByFamily = relaysByFamily; + } + public Map<String, Set<String>> getRelaysByFamily() { + return this.relaysByFamily; + } + + private SortedMap<Integer, Set<String>> relaysByFirstSeenDays; + public void setRelaysByFirstSeenDays( + SortedMap<Integer, Set<String>> relaysByFirstSeenDays) { + this.relaysByFirstSeenDays = relaysByFirstSeenDays; + } + public SortedMap<Integer, Set<String>> getRelaysByFirstSeenDays() { + return relaysByFirstSeenDays; + } + + private SortedMap<Integer, Set<String>> bridgesByFirstSeenDays; + public void setBridgesByFirstSeenDays( + SortedMap<Integer, Set<String>> bridgesByFirstSeenDays) { + this.bridgesByFirstSeenDays = bridgesByFirstSeenDays; + } + public SortedMap<Integer, Set<String>> getBridgesByFirstSeenDays() { + return bridgesByFirstSeenDays; + } + + private SortedMap<Integer, Set<String>> relaysByLastSeenDays; + public void setRelaysByLastSeenDays( + SortedMap<Integer, Set<String>> relaysByLastSeenDays) { + this.relaysByLastSeenDays = relaysByLastSeenDays; + } + public SortedMap<Integer, Set<String>> getRelaysByLastSeenDays() { + return relaysByLastSeenDays; + } + + private SortedMap<Integer, Set<String>> bridgesByLastSeenDays; + public void setBridgesByLastSeenDays( + SortedMap<Integer, Set<String>> bridgesByLastSeenDays) { + this.bridgesByLastSeenDays = bridgesByLastSeenDays; + } + public SortedMap<Integer, Set<String>> getBridgesByLastSeenDays() { + return bridgesByLastSeenDays; + } +} \ No newline at end of file diff --git a/src/org/torproject/onionoo/server/NodeIndexer.java b/src/org/torproject/onionoo/server/NodeIndexer.java index b76f4c1..22d8608 100644 --- a/src/org/torproject/onionoo/server/NodeIndexer.java +++ b/src/org/torproject/onionoo/server/NodeIndexer.java @@ -23,140 +23,6 @@ import org.torproject.onionoo.util.ApplicationFactory; import org.torproject.onionoo.util.DateTimeHelper; import org.torproject.onionoo.util.Time; -class NodeIndex { - - private String relaysPublishedString; - public void setRelaysPublishedString(String relaysPublishedString) { - this.relaysPublishedString = relaysPublishedString; - } - public String getRelaysPublishedString() { - return relaysPublishedString; - } - - private String bridgesPublishedString; - public void setBridgesPublishedString(String bridgesPublishedString) { - this.bridgesPublishedString = bridgesPublishedString; - } - public String getBridgesPublishedString() { - return bridgesPublishedString; - } - - private List<String> relaysByConsensusWeight; - public void setRelaysByConsensusWeight( - List<String> relaysByConsensusWeight) { - this.relaysByConsensusWeight = relaysByConsensusWeight; - } - public List<String> getRelaysByConsensusWeight() { - return relaysByConsensusWeight; - } - - - private Map<String, SummaryDocument> relayFingerprintSummaryLines; - public void setRelayFingerprintSummaryLines( - Map<String, SummaryDocument> relayFingerprintSummaryLines) { - this.relayFingerprintSummaryLines = relayFingerprintSummaryLines; - } - public Map<String, SummaryDocument> getRelayFingerprintSummaryLines() { - return this.relayFingerprintSummaryLines; - } - - private Map<String, SummaryDocument> bridgeFingerprintSummaryLines; - public void setBridgeFingerprintSummaryLines( - Map<String, SummaryDocument> bridgeFingerprintSummaryLines) { - this.bridgeFingerprintSummaryLines = bridgeFingerprintSummaryLines; - } - public Map<String, SummaryDocument> getBridgeFingerprintSummaryLines() { - return this.bridgeFingerprintSummaryLines; - } - - private Map<String, Set<String>> relaysByCountryCode = null; - public void setRelaysByCountryCode( - Map<String, Set<String>> relaysByCountryCode) { - this.relaysByCountryCode = relaysByCountryCode; - } - public Map<String, Set<String>> getRelaysByCountryCode() { - return relaysByCountryCode; - } - - private Map<String, Set<String>> relaysByASNumber = null; - public void setRelaysByASNumber( - Map<String, Set<String>> relaysByASNumber) { - this.relaysByASNumber = relaysByASNumber; - } - public Map<String, Set<String>> getRelaysByASNumber() { - return relaysByASNumber; - } - - private Map<String, Set<String>> relaysByFlag = null; - public void setRelaysByFlag(Map<String, Set<String>> relaysByFlag) { - this.relaysByFlag = relaysByFlag; - } - public Map<String, Set<String>> getRelaysByFlag() { - return relaysByFlag; - } - - private Map<String, Set<String>> bridgesByFlag = null; - public void setBridgesByFlag(Map<String, Set<String>> bridgesByFlag) { - this.bridgesByFlag = bridgesByFlag; - } - public Map<String, Set<String>> getBridgesByFlag() { - return bridgesByFlag; - } - - private Map<String, Set<String>> relaysByContact = null; - public void setRelaysByContact( - Map<String, Set<String>> relaysByContact) { - this.relaysByContact = relaysByContact; - } - public Map<String, Set<String>> getRelaysByContact() { - return relaysByContact; - } - - private Map<String, Set<String>> relaysByFamily = null; - public void setRelaysByFamily(Map<String, Set<String>> relaysByFamily) { - this.relaysByFamily = relaysByFamily; - } - public Map<String, Set<String>> getRelaysByFamily() { - return this.relaysByFamily; - } - - private SortedMap<Integer, Set<String>> relaysByFirstSeenDays; - public void setRelaysByFirstSeenDays( - SortedMap<Integer, Set<String>> relaysByFirstSeenDays) { - this.relaysByFirstSeenDays = relaysByFirstSeenDays; - } - public SortedMap<Integer, Set<String>> getRelaysByFirstSeenDays() { - return relaysByFirstSeenDays; - } - - private SortedMap<Integer, Set<String>> bridgesByFirstSeenDays; - public void setBridgesByFirstSeenDays( - SortedMap<Integer, Set<String>> bridgesByFirstSeenDays) { - this.bridgesByFirstSeenDays = bridgesByFirstSeenDays; - } - public SortedMap<Integer, Set<String>> getBridgesByFirstSeenDays() { - return bridgesByFirstSeenDays; - } - - private SortedMap<Integer, Set<String>> relaysByLastSeenDays; - public void setRelaysByLastSeenDays( - SortedMap<Integer, Set<String>> relaysByLastSeenDays) { - this.relaysByLastSeenDays = relaysByLastSeenDays; - } - public SortedMap<Integer, Set<String>> getRelaysByLastSeenDays() { - return relaysByLastSeenDays; - } - - private SortedMap<Integer, Set<String>> bridgesByLastSeenDays; - public void setBridgesByLastSeenDays( - SortedMap<Integer, Set<String>> bridgesByLastSeenDays) { - this.bridgesByLastSeenDays = bridgesByLastSeenDays; - } - public SortedMap<Integer, Set<String>> getBridgesByLastSeenDays() { - return bridgesByLastSeenDays; - } -} - public class NodeIndexer implements ServletContextListener, Runnable { public void contextInitialized(ServletContextEvent contextEvent) { diff --git a/src/org/torproject/onionoo/server/ResourceServlet.java b/src/org/torproject/onionoo/server/ResourceServlet.java index 1e05d12..6f01448 100644 --- a/src/org/torproject/onionoo/server/ResourceServlet.java +++ b/src/org/torproject/onionoo/server/ResourceServlet.java @@ -43,45 +43,6 @@ public class ResourceServlet extends HttpServlet { } } - public static class HttpServletRequestWrapper { - private HttpServletRequest request; - protected HttpServletRequestWrapper(HttpServletRequest request) { - this.request = request; - } - protected String getRequestURI() { - return this.request.getRequestURI(); - } - @SuppressWarnings("rawtypes") - protected Map getParameterMap() { - return this.request.getParameterMap(); - } - protected String[] getParameterValues(String parameterKey) { - return this.request.getParameterValues(parameterKey); - } - } - - public static class HttpServletResponseWrapper { - private HttpServletResponse response = null; - protected HttpServletResponseWrapper(HttpServletResponse response) { - this.response = response; - } - protected void sendError(int errorStatusCode) throws IOException { - this.response.sendError(errorStatusCode); - } - protected void setHeader(String headerName, String headerValue) { - this.response.setHeader(headerName, headerValue); - } - protected void setContentType(String contentType) { - this.response.setContentType(contentType); - } - protected void setCharacterEncoding(String characterEncoding) { - this.response.setCharacterEncoding(characterEncoding); - } - protected PrintWriter getWriter() throws IOException { - return this.response.getWriter(); - } - } - public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { HttpServletRequestWrapper requestWrapper = diff --git a/src/org/torproject/onionoo/updater/DescriptorDownloader.java b/src/org/torproject/onionoo/updater/DescriptorDownloader.java new file mode 100644 index 0000000..60d8d45 --- /dev/null +++ b/src/org/torproject/onionoo/updater/DescriptorDownloader.java @@ -0,0 +1,178 @@ +package org.torproject.onionoo.updater; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.zip.GZIPInputStream; + +class DescriptorDownloader { + + private final String protocolHostNameResourcePrefix = + "https://collector.torproject.org/recent/"; + + private String directory; + + private final File inDir = new File("in/recent"); + + public DescriptorDownloader(DescriptorType descriptorType) { + switch (descriptorType) { + case RELAY_CONSENSUSES: + this.directory = "relay-descriptors/consensuses/"; + break; + case RELAY_SERVER_DESCRIPTORS: + this.directory = "relay-descriptors/server-descriptors/"; + break; + case RELAY_EXTRA_INFOS: + this.directory = "relay-descriptors/extra-infos/"; + break; + case EXIT_LISTS: + this.directory = "exit-lists/"; + break; + case BRIDGE_STATUSES: + this.directory = "bridge-descriptors/statuses/"; + break; + case BRIDGE_SERVER_DESCRIPTORS: + this.directory = "bridge-descriptors/server-descriptors/"; + break; + case BRIDGE_EXTRA_INFOS: + this.directory = "bridge-descriptors/extra-infos/"; + break; + case BRIDGE_POOL_ASSIGNMENTS: + this.directory = "bridge-pool-assignments/"; + break; + default: + System.err.println("Unknown descriptor type."); + return; + } + } + + private SortedSet<String> localFiles = new TreeSet<String>(); + + public int statLocalFiles() { + File localDirectory = new File(this.inDir, this.directory); + if (localDirectory.exists()) { + for (File file : localDirectory.listFiles()) { + this.localFiles.add(file.getName()); + } + } + return this.localFiles.size(); + } + + private SortedSet<String> remoteFiles = new TreeSet<String>(); + + public int fetchRemoteDirectory() { + String directoryUrl = this.protocolHostNameResourcePrefix + + this.directory; + try { + URL u = new URL(directoryUrl); + HttpURLConnection huc = (HttpURLConnection) u.openConnection(); + huc.setRequestMethod("GET"); + huc.connect(); + if (huc.getResponseCode() != 200) { + System.err.println("Could not fetch " + directoryUrl + + ": " + huc.getResponseCode() + " " + + huc.getResponseMessage() + ". Skipping."); + return 0; + } + BufferedReader br = new BufferedReader(new InputStreamReader( + huc.getInputStream())); + String line; + while ((line = br.readLine()) != null) { + if (!line.trim().startsWith("<tr>") || + !line.contains("<a href=\"")) { + continue; + } + String linePart = line.substring( + line.indexOf("<a href=\"") + "<a href=\"".length()); + if (!linePart.contains("\"")) { + continue; + } + linePart = linePart.substring(0, linePart.indexOf("\"")); + if (linePart.endsWith("/")) { + continue; + } + this.remoteFiles.add(linePart); + } + br.close(); + } catch (IOException e) { + System.err.println("Could not fetch or parse " + directoryUrl + + ". Skipping."); + } + return this.remoteFiles.size(); + } + + public int fetchRemoteFiles() { + int fetchedFiles = 0; + for (String remoteFile : this.remoteFiles) { + if (this.localFiles.contains(remoteFile)) { + continue; + } + String fileUrl = this.protocolHostNameResourcePrefix + + this.directory + remoteFile; + File localTempFile = new File(this.inDir, this.directory + + remoteFile + ".tmp"); + File localFile = new File(this.inDir, this.directory + remoteFile); + try { + localFile.getParentFile().mkdirs(); + URL u = new URL(fileUrl); + HttpURLConnection huc = (HttpURLConnection) u.openConnection(); + huc.setRequestMethod("GET"); + huc.addRequestProperty("Accept-Encoding", "gzip"); + huc.connect(); + if (huc.getResponseCode() != 200) { + System.err.println("Could not fetch " + fileUrl + + ": " + huc.getResponseCode() + " " + + huc.getResponseMessage() + ". Skipping."); + continue; + } + long lastModified = huc.getHeaderFieldDate("Last-Modified", -1L); + InputStream is; + if (huc.getContentEncoding() != null && + huc.getContentEncoding().equalsIgnoreCase("gzip")) { + is = new GZIPInputStream(huc.getInputStream()); + } else { + is = huc.getInputStream(); + } + BufferedInputStream bis = new BufferedInputStream(is); + BufferedOutputStream bos = new BufferedOutputStream( + new FileOutputStream(localTempFile)); + int len; + byte[] data = new byte[1024]; + while ((len = bis.read(data, 0, 1024)) >= 0) { + bos.write(data, 0, len); + } + bis.close(); + bos.close(); + localTempFile.renameTo(localFile); + if (lastModified >= 0) { + localFile.setLastModified(lastModified); + } + fetchedFiles++; + } catch (IOException e) { + System.err.println("Could not fetch or store " + fileUrl + + ". Skipping."); + } + } + return fetchedFiles; + } + + public int deleteOldLocalFiles() { + int deletedFiles = 0; + for (String localFile : this.localFiles) { + if (!this.remoteFiles.contains(localFile)) { + new File(this.inDir, this.directory + localFile).delete(); + deletedFiles++; + } + } + return deletedFiles; + } +} \ No newline at end of file diff --git a/src/org/torproject/onionoo/updater/DescriptorHistory.java b/src/org/torproject/onionoo/updater/DescriptorHistory.java new file mode 100644 index 0000000..0f6f578 --- /dev/null +++ b/src/org/torproject/onionoo/updater/DescriptorHistory.java @@ -0,0 +1,12 @@ +package org.torproject.onionoo.updater; + +enum DescriptorHistory { + RELAY_CONSENSUS_HISTORY, + RELAY_SERVER_HISTORY, + RELAY_EXTRAINFO_HISTORY, + EXIT_LIST_HISTORY, + BRIDGE_STATUS_HISTORY, + BRIDGE_SERVER_HISTORY, + BRIDGE_EXTRAINFO_HISTORY, + BRIDGE_POOLASSIGN_HISTORY, +} \ No newline at end of file diff --git a/src/org/torproject/onionoo/updater/DescriptorQueue.java b/src/org/torproject/onionoo/updater/DescriptorQueue.java new file mode 100644 index 0000000..96362c5 --- /dev/null +++ b/src/org/torproject/onionoo/updater/DescriptorQueue.java @@ -0,0 +1,221 @@ +package org.torproject.onionoo.updater; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; + +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.DescriptorFile; +import org.torproject.descriptor.DescriptorReader; +import org.torproject.descriptor.DescriptorSourceFactory; + +class DescriptorQueue { + + private File inDir; + + private File statusDir; + + private DescriptorReader descriptorReader; + + private File historyFile; + + private Iterator<DescriptorFile> descriptorFiles; + + private List<Descriptor> descriptors; + + private int historySizeBefore; + public int getHistorySizeBefore() { + return this.historySizeBefore; + } + + private int historySizeAfter; + public int getHistorySizeAfter() { + return this.historySizeAfter; + } + + private long returnedDescriptors = 0L; + public long getReturnedDescriptors() { + return this.returnedDescriptors; + } + + private long returnedBytes = 0L; + public long getReturnedBytes() { + return this.returnedBytes; + } + + public DescriptorQueue(File inDir, File statusDir) { + this.inDir = inDir; + this.statusDir = statusDir; + this.descriptorReader = + DescriptorSourceFactory.createDescriptorReader(); + } + + public void addDirectory(DescriptorType descriptorType) { + String directoryName = null; + switch (descriptorType) { + case RELAY_CONSENSUSES: + directoryName = "relay-descriptors/consensuses"; + break; + case RELAY_SERVER_DESCRIPTORS: + directoryName = "relay-descriptors/server-descriptors"; + break; + case RELAY_EXTRA_INFOS: + directoryName = "relay-descriptors/extra-infos"; + break; + case BRIDGE_STATUSES: + directoryName = "bridge-descriptors/statuses"; + break; + case BRIDGE_SERVER_DESCRIPTORS: + directoryName = "bridge-descriptors/server-descriptors"; + break; + case BRIDGE_EXTRA_INFOS: + directoryName = "bridge-descriptors/extra-infos"; + break; + case BRIDGE_POOL_ASSIGNMENTS: + directoryName = "bridge-pool-assignments"; + break; + case EXIT_LISTS: + directoryName = "exit-lists"; + break; + default: + System.err.println("Unknown descriptor type. Not adding directory " + + "to descriptor reader."); + return; + } + File directory = new File(this.inDir, directoryName); + if (directory.exists() && directory.isDirectory()) { + this.descriptorReader.addDirectory(directory); + this.descriptorReader.setMaxDescriptorFilesInQueue(1); + } else { + System.err.println("Directory " + directory.getAbsolutePath() + + " either does not exist or is not a directory. Not adding " + + "to descriptor reader."); + } + } + + public void readHistoryFile(DescriptorHistory descriptorHistory) { + String historyFileName = null; + switch (descriptorHistory) { + case RELAY_EXTRAINFO_HISTORY: + historyFileName = "relay-extrainfo-history"; + break; + case BRIDGE_EXTRAINFO_HISTORY: + historyFileName = "bridge-extrainfo-history"; + break; + case EXIT_LIST_HISTORY: + historyFileName = "exit-list-history"; + break; + case BRIDGE_POOLASSIGN_HISTORY: + historyFileName = "bridge-poolassign-history"; + break; + case RELAY_CONSENSUS_HISTORY: + historyFileName = "relay-consensus-history"; + break; + case BRIDGE_STATUS_HISTORY: + historyFileName = "bridge-status-history"; + break; + case RELAY_SERVER_HISTORY: + historyFileName = "relay-server-history"; + break; + case BRIDGE_SERVER_HISTORY: + historyFileName = "bridge-server-history"; + break; + default: + System.err.println("Unknown descriptor history. Not excluding " + + "files."); + return; + } + this.historyFile = new File(this.statusDir, historyFileName); + if (this.historyFile.exists() && this.historyFile.isFile()) { + SortedMap<String, Long> excludedFiles = new TreeMap<String, Long>(); + try { + BufferedReader br = new BufferedReader(new FileReader( + this.historyFile)); + String line; + while ((line = br.readLine()) != null) { + try { + String[] parts = line.split(" ", 2); + excludedFiles.put(parts[1], Long.parseLong(parts[0])); + } catch (NumberFormatException e) { + System.err.println("Illegal line '" + line + "' in parse " + + "history. Skipping line."); + } + } + br.close(); + } catch (IOException e) { + System.err.println("Could not read history file '" + + this.historyFile.getAbsolutePath() + "'. Not excluding " + + "descriptors in this execution."); + e.printStackTrace(); + return; + } + this.historySizeBefore = excludedFiles.size(); + this.descriptorReader.setExcludedFiles(excludedFiles); + } + } + + public void writeHistoryFile() { + if (this.historyFile == null) { + return; + } + SortedMap<String, Long> excludedAndParsedFiles = + new TreeMap<String, Long>(); + excludedAndParsedFiles.putAll( + this.descriptorReader.getExcludedFiles()); + excludedAndParsedFiles.putAll(this.descriptorReader.getParsedFiles()); + this.historySizeAfter = excludedAndParsedFiles.size(); + try { + this.historyFile.getParentFile().mkdirs(); + BufferedWriter bw = new BufferedWriter(new FileWriter( + this.historyFile)); + for (Map.Entry<String, Long> e : excludedAndParsedFiles.entrySet()) { + String absolutePath = e.getKey(); + long lastModifiedMillis = e.getValue(); + bw.write(String.valueOf(lastModifiedMillis) + " " + absolutePath + + "\n"); + } + bw.close(); + } catch (IOException e) { + System.err.println("Could not write history file '" + + this.historyFile.getAbsolutePath() + "'. Not excluding " + + "descriptors in next execution."); + return; + } + } + + public Descriptor nextDescriptor() { + Descriptor nextDescriptor = null; + if (this.descriptorFiles == null) { + this.descriptorFiles = this.descriptorReader.readDescriptors(); + } + while (this.descriptors == null && this.descriptorFiles.hasNext()) { + DescriptorFile descriptorFile = this.descriptorFiles.next(); + if (descriptorFile.getException() != null) { + System.err.println("Could not parse " + + descriptorFile.getFileName()); + descriptorFile.getException().printStackTrace(); + } + if (descriptorFile.getDescriptors() != null && + !descriptorFile.getDescriptors().isEmpty()) { + this.descriptors = descriptorFile.getDescriptors(); + } + } + if (this.descriptors != null) { + nextDescriptor = this.descriptors.remove(0); + this.returnedDescriptors++; + this.returnedBytes += nextDescriptor.getRawDescriptorBytes().length; + if (this.descriptors.isEmpty()) { + this.descriptors = null; + } + } + return nextDescriptor; + } +} \ No newline at end of file diff --git a/src/org/torproject/onionoo/updater/DescriptorSource.java b/src/org/torproject/onionoo/updater/DescriptorSource.java index 32fbd2a..ea1474f 100644 --- a/src/org/torproject/onionoo/updater/DescriptorSource.java +++ b/src/org/torproject/onionoo/updater/DescriptorSource.java @@ -2,38 +2,19 @@ * See LICENSE for licensing information */ package org.torproject.onionoo.updater; -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.BufferedReader; -import java.io.BufferedWriter; import java.io.File; -import java.io.FileOutputStream; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.net.HttpURLConnection; -import java.net.URL; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.SortedMap; import java.util.SortedSet; -import java.util.TreeMap; import java.util.TreeSet; -import java.util.zip.GZIPInputStream; import org.torproject.descriptor.BridgeNetworkStatus; import org.torproject.descriptor.BridgePoolAssignment; import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.DescriptorFile; -import org.torproject.descriptor.DescriptorReader; -import org.torproject.descriptor.DescriptorSourceFactory; import org.torproject.descriptor.ExitList; import org.torproject.descriptor.ExitListEntry; import org.torproject.descriptor.ExtraInfoDescriptor; @@ -41,383 +22,6 @@ import org.torproject.descriptor.RelayNetworkStatusConsensus; import org.torproject.descriptor.ServerDescriptor; import org.torproject.onionoo.util.Logger; -enum DescriptorHistory { - RELAY_CONSENSUS_HISTORY, - RELAY_SERVER_HISTORY, - RELAY_EXTRAINFO_HISTORY, - EXIT_LIST_HISTORY, - BRIDGE_STATUS_HISTORY, - BRIDGE_SERVER_HISTORY, - BRIDGE_EXTRAINFO_HISTORY, - BRIDGE_POOLASSIGN_HISTORY, -} - -class DescriptorDownloader { - - private final String protocolHostNameResourcePrefix = - "https://collector.torproject.org/recent/"; - - private String directory; - - private final File inDir = new File("in/recent"); - - public DescriptorDownloader(DescriptorType descriptorType) { - switch (descriptorType) { - case RELAY_CONSENSUSES: - this.directory = "relay-descriptors/consensuses/"; - break; - case RELAY_SERVER_DESCRIPTORS: - this.directory = "relay-descriptors/server-descriptors/"; - break; - case RELAY_EXTRA_INFOS: - this.directory = "relay-descriptors/extra-infos/"; - break; - case EXIT_LISTS: - this.directory = "exit-lists/"; - break; - case BRIDGE_STATUSES: - this.directory = "bridge-descriptors/statuses/"; - break; - case BRIDGE_SERVER_DESCRIPTORS: - this.directory = "bridge-descriptors/server-descriptors/"; - break; - case BRIDGE_EXTRA_INFOS: - this.directory = "bridge-descriptors/extra-infos/"; - break; - case BRIDGE_POOL_ASSIGNMENTS: - this.directory = "bridge-pool-assignments/"; - break; - default: - System.err.println("Unknown descriptor type."); - return; - } - } - - private SortedSet<String> localFiles = new TreeSet<String>(); - - public int statLocalFiles() { - File localDirectory = new File(this.inDir, this.directory); - if (localDirectory.exists()) { - for (File file : localDirectory.listFiles()) { - this.localFiles.add(file.getName()); - } - } - return this.localFiles.size(); - } - - private SortedSet<String> remoteFiles = new TreeSet<String>(); - - public int fetchRemoteDirectory() { - String directoryUrl = this.protocolHostNameResourcePrefix - + this.directory; - try { - URL u = new URL(directoryUrl); - HttpURLConnection huc = (HttpURLConnection) u.openConnection(); - huc.setRequestMethod("GET"); - huc.connect(); - if (huc.getResponseCode() != 200) { - System.err.println("Could not fetch " + directoryUrl - + ": " + huc.getResponseCode() + " " - + huc.getResponseMessage() + ". Skipping."); - return 0; - } - BufferedReader br = new BufferedReader(new InputStreamReader( - huc.getInputStream())); - String line; - while ((line = br.readLine()) != null) { - if (!line.trim().startsWith("<tr>") || - !line.contains("<a href=\"")) { - continue; - } - String linePart = line.substring( - line.indexOf("<a href=\"") + "<a href=\"".length()); - if (!linePart.contains("\"")) { - continue; - } - linePart = linePart.substring(0, linePart.indexOf("\"")); - if (linePart.endsWith("/")) { - continue; - } - this.remoteFiles.add(linePart); - } - br.close(); - } catch (IOException e) { - System.err.println("Could not fetch or parse " + directoryUrl - + ". Skipping."); - } - return this.remoteFiles.size(); - } - - public int fetchRemoteFiles() { - int fetchedFiles = 0; - for (String remoteFile : this.remoteFiles) { - if (this.localFiles.contains(remoteFile)) { - continue; - } - String fileUrl = this.protocolHostNameResourcePrefix - + this.directory + remoteFile; - File localTempFile = new File(this.inDir, this.directory - + remoteFile + ".tmp"); - File localFile = new File(this.inDir, this.directory + remoteFile); - try { - localFile.getParentFile().mkdirs(); - URL u = new URL(fileUrl); - HttpURLConnection huc = (HttpURLConnection) u.openConnection(); - huc.setRequestMethod("GET"); - huc.addRequestProperty("Accept-Encoding", "gzip"); - huc.connect(); - if (huc.getResponseCode() != 200) { - System.err.println("Could not fetch " + fileUrl - + ": " + huc.getResponseCode() + " " - + huc.getResponseMessage() + ". Skipping."); - continue; - } - long lastModified = huc.getHeaderFieldDate("Last-Modified", -1L); - InputStream is; - if (huc.getContentEncoding() != null && - huc.getContentEncoding().equalsIgnoreCase("gzip")) { - is = new GZIPInputStream(huc.getInputStream()); - } else { - is = huc.getInputStream(); - } - BufferedInputStream bis = new BufferedInputStream(is); - BufferedOutputStream bos = new BufferedOutputStream( - new FileOutputStream(localTempFile)); - int len; - byte[] data = new byte[1024]; - while ((len = bis.read(data, 0, 1024)) >= 0) { - bos.write(data, 0, len); - } - bis.close(); - bos.close(); - localTempFile.renameTo(localFile); - if (lastModified >= 0) { - localFile.setLastModified(lastModified); - } - fetchedFiles++; - } catch (IOException e) { - System.err.println("Could not fetch or store " + fileUrl - + ". Skipping."); - } - } - return fetchedFiles; - } - - public int deleteOldLocalFiles() { - int deletedFiles = 0; - for (String localFile : this.localFiles) { - if (!this.remoteFiles.contains(localFile)) { - new File(this.inDir, this.directory + localFile).delete(); - deletedFiles++; - } - } - return deletedFiles; - } -} - -class DescriptorQueue { - - private File inDir; - - private File statusDir; - - private DescriptorReader descriptorReader; - - private File historyFile; - - private Iterator<DescriptorFile> descriptorFiles; - - private List<Descriptor> descriptors; - - private int historySizeBefore; - public int getHistorySizeBefore() { - return this.historySizeBefore; - } - - private int historySizeAfter; - public int getHistorySizeAfter() { - return this.historySizeAfter; - } - - private long returnedDescriptors = 0L; - public long getReturnedDescriptors() { - return this.returnedDescriptors; - } - - private long returnedBytes = 0L; - public long getReturnedBytes() { - return this.returnedBytes; - } - - public DescriptorQueue(File inDir, File statusDir) { - this.inDir = inDir; - this.statusDir = statusDir; - this.descriptorReader = - DescriptorSourceFactory.createDescriptorReader(); - } - - public void addDirectory(DescriptorType descriptorType) { - String directoryName = null; - switch (descriptorType) { - case RELAY_CONSENSUSES: - directoryName = "relay-descriptors/consensuses"; - break; - case RELAY_SERVER_DESCRIPTORS: - directoryName = "relay-descriptors/server-descriptors"; - break; - case RELAY_EXTRA_INFOS: - directoryName = "relay-descriptors/extra-infos"; - break; - case BRIDGE_STATUSES: - directoryName = "bridge-descriptors/statuses"; - break; - case BRIDGE_SERVER_DESCRIPTORS: - directoryName = "bridge-descriptors/server-descriptors"; - break; - case BRIDGE_EXTRA_INFOS: - directoryName = "bridge-descriptors/extra-infos"; - break; - case BRIDGE_POOL_ASSIGNMENTS: - directoryName = "bridge-pool-assignments"; - break; - case EXIT_LISTS: - directoryName = "exit-lists"; - break; - default: - System.err.println("Unknown descriptor type. Not adding directory " - + "to descriptor reader."); - return; - } - File directory = new File(this.inDir, directoryName); - if (directory.exists() && directory.isDirectory()) { - this.descriptorReader.addDirectory(directory); - this.descriptorReader.setMaxDescriptorFilesInQueue(1); - } else { - System.err.println("Directory " + directory.getAbsolutePath() - + " either does not exist or is not a directory. Not adding " - + "to descriptor reader."); - } - } - - public void readHistoryFile(DescriptorHistory descriptorHistory) { - String historyFileName = null; - switch (descriptorHistory) { - case RELAY_EXTRAINFO_HISTORY: - historyFileName = "relay-extrainfo-history"; - break; - case BRIDGE_EXTRAINFO_HISTORY: - historyFileName = "bridge-extrainfo-history"; - break; - case EXIT_LIST_HISTORY: - historyFileName = "exit-list-history"; - break; - case BRIDGE_POOLASSIGN_HISTORY: - historyFileName = "bridge-poolassign-history"; - break; - case RELAY_CONSENSUS_HISTORY: - historyFileName = "relay-consensus-history"; - break; - case BRIDGE_STATUS_HISTORY: - historyFileName = "bridge-status-history"; - break; - case RELAY_SERVER_HISTORY: - historyFileName = "relay-server-history"; - break; - case BRIDGE_SERVER_HISTORY: - historyFileName = "bridge-server-history"; - break; - default: - System.err.println("Unknown descriptor history. Not excluding " - + "files."); - return; - } - this.historyFile = new File(this.statusDir, historyFileName); - if (this.historyFile.exists() && this.historyFile.isFile()) { - SortedMap<String, Long> excludedFiles = new TreeMap<String, Long>(); - try { - BufferedReader br = new BufferedReader(new FileReader( - this.historyFile)); - String line; - while ((line = br.readLine()) != null) { - try { - String[] parts = line.split(" ", 2); - excludedFiles.put(parts[1], Long.parseLong(parts[0])); - } catch (NumberFormatException e) { - System.err.println("Illegal line '" + line + "' in parse " - + "history. Skipping line."); - } - } - br.close(); - } catch (IOException e) { - System.err.println("Could not read history file '" - + this.historyFile.getAbsolutePath() + "'. Not excluding " - + "descriptors in this execution."); - e.printStackTrace(); - return; - } - this.historySizeBefore = excludedFiles.size(); - this.descriptorReader.setExcludedFiles(excludedFiles); - } - } - - public void writeHistoryFile() { - if (this.historyFile == null) { - return; - } - SortedMap<String, Long> excludedAndParsedFiles = - new TreeMap<String, Long>(); - excludedAndParsedFiles.putAll( - this.descriptorReader.getExcludedFiles()); - excludedAndParsedFiles.putAll(this.descriptorReader.getParsedFiles()); - this.historySizeAfter = excludedAndParsedFiles.size(); - try { - this.historyFile.getParentFile().mkdirs(); - BufferedWriter bw = new BufferedWriter(new FileWriter( - this.historyFile)); - for (Map.Entry<String, Long> e : excludedAndParsedFiles.entrySet()) { - String absolutePath = e.getKey(); - long lastModifiedMillis = e.getValue(); - bw.write(String.valueOf(lastModifiedMillis) + " " + absolutePath - + "\n"); - } - bw.close(); - } catch (IOException e) { - System.err.println("Could not write history file '" - + this.historyFile.getAbsolutePath() + "'. Not excluding " - + "descriptors in next execution."); - return; - } - } - - public Descriptor nextDescriptor() { - Descriptor nextDescriptor = null; - if (this.descriptorFiles == null) { - this.descriptorFiles = this.descriptorReader.readDescriptors(); - } - while (this.descriptors == null && this.descriptorFiles.hasNext()) { - DescriptorFile descriptorFile = this.descriptorFiles.next(); - if (descriptorFile.getException() != null) { - System.err.println("Could not parse " - + descriptorFile.getFileName()); - descriptorFile.getException().printStackTrace(); - } - if (descriptorFile.getDescriptors() != null && - !descriptorFile.getDescriptors().isEmpty()) { - this.descriptors = descriptorFile.getDescriptors(); - } - } - if (this.descriptors != null) { - nextDescriptor = this.descriptors.remove(0); - this.returnedDescriptors++; - this.returnedBytes += nextDescriptor.getRawDescriptorBytes().length; - if (this.descriptors.isEmpty()) { - this.descriptors = null; - } - } - return nextDescriptor; - } -} - public class DescriptorSource { private final File inDir = new File("in/recent"); diff --git a/src/org/torproject/onionoo/updater/RdnsLookupRequest.java b/src/org/torproject/onionoo/updater/RdnsLookupRequest.java new file mode 100644 index 0000000..4a06d20 --- /dev/null +++ b/src/org/torproject/onionoo/updater/RdnsLookupRequest.java @@ -0,0 +1,43 @@ +/* Copyright 2013 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.updater; + +import java.net.InetAddress; +import java.net.UnknownHostException; + +class RdnsLookupRequest extends Thread { + + private final ReverseDomainNameResolver reverseDomainNameResolver; + private RdnsLookupWorker parent; + private String address, hostName; + private long lookupStartedMillis = -1L, lookupCompletedMillis = -1L; + public RdnsLookupRequest( + ReverseDomainNameResolver reverseDomainNameResolver, + RdnsLookupWorker parent, String address) { + this.reverseDomainNameResolver = reverseDomainNameResolver; + this.parent = parent; + this.address = address; + } + public void run() { + this.lookupStartedMillis = + this.reverseDomainNameResolver.time.currentTimeMillis(); + try { + String result = InetAddress.getByName(this.address).getHostName(); + synchronized (this) { + this.hostName = result; + } + } catch (UnknownHostException e) { + /* We'll try again the next time. */ + } + this.lookupCompletedMillis = + this.reverseDomainNameResolver.time.currentTimeMillis(); + this.parent.interrupt(); + } + public synchronized String getHostName() { + return hostName; + } + public synchronized long getLookupMillis() { + return this.lookupCompletedMillis - this.lookupStartedMillis; + } +} + diff --git a/src/org/torproject/onionoo/updater/RdnsLookupWorker.java b/src/org/torproject/onionoo/updater/RdnsLookupWorker.java new file mode 100644 index 0000000..cf7d580 --- /dev/null +++ b/src/org/torproject/onionoo/updater/RdnsLookupWorker.java @@ -0,0 +1,55 @@ +/* Copyright 2013 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.updater; + +class RdnsLookupWorker extends Thread { + + private final ReverseDomainNameResolver reverseDomainNameResolver; + + RdnsLookupWorker(ReverseDomainNameResolver reverseDomainNameResolver) { + this.reverseDomainNameResolver = reverseDomainNameResolver; + } + + public void run() { + while (this.reverseDomainNameResolver.time.currentTimeMillis() - + ReverseDomainNameResolver.RDNS_LOOKUP_MAX_DURATION_MILLIS + <= this.reverseDomainNameResolver.startedRdnsLookups) { + String rdnsLookupJob = null; + synchronized (this.reverseDomainNameResolver.rdnsLookupJobs) { + for (String job : this.reverseDomainNameResolver.rdnsLookupJobs) { + rdnsLookupJob = job; + this.reverseDomainNameResolver.rdnsLookupJobs.remove(job); + break; + } + } + if (rdnsLookupJob == null) { + break; + } + RdnsLookupRequest request = new RdnsLookupRequest( + this.reverseDomainNameResolver, this, rdnsLookupJob); + request.setDaemon(true); + request.start(); + try { + Thread.sleep( + ReverseDomainNameResolver.RDNS_LOOKUP_MAX_REQUEST_MILLIS); + } catch (InterruptedException e) { + /* Getting interrupted should be the default case. */ + } + String hostName = request.getHostName(); + if (hostName != null) { + synchronized (this.reverseDomainNameResolver.rdnsLookupResults) { + this.reverseDomainNameResolver.rdnsLookupResults.put( + rdnsLookupJob, hostName); + } + } + long lookupMillis = request.getLookupMillis(); + if (lookupMillis >= 0L) { + synchronized (this.reverseDomainNameResolver.rdnsLookupMillis) { + this.reverseDomainNameResolver.rdnsLookupMillis.add( + lookupMillis); + } + } + } + } +} + diff --git a/src/org/torproject/onionoo/updater/ReverseDomainNameResolver.java b/src/org/torproject/onionoo/updater/ReverseDomainNameResolver.java index 8ca7eb4..8694155 100644 --- a/src/org/torproject/onionoo/updater/ReverseDomainNameResolver.java +++ b/src/org/torproject/onionoo/updater/ReverseDomainNameResolver.java @@ -2,8 +2,6 @@ * See LICENSE for licensing information */ package org.torproject.onionoo.updater; -import java.net.InetAddress; -import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -19,84 +17,15 @@ import org.torproject.onionoo.util.Time; public class ReverseDomainNameResolver { - private class RdnsLookupWorker extends Thread { - public void run() { - while (time.currentTimeMillis() - RDNS_LOOKUP_MAX_DURATION_MILLIS - <= startedRdnsLookups) { - String rdnsLookupJob = null; - synchronized (rdnsLookupJobs) { - for (String job : rdnsLookupJobs) { - rdnsLookupJob = job; - rdnsLookupJobs.remove(job); - break; - } - } - if (rdnsLookupJob == null) { - break; - } - RdnsLookupRequest request = new RdnsLookupRequest(this, - rdnsLookupJob); - request.setDaemon(true); - request.start(); - try { - Thread.sleep(RDNS_LOOKUP_MAX_REQUEST_MILLIS); - } catch (InterruptedException e) { - /* Getting interrupted should be the default case. */ - } - String hostName = request.getHostName(); - if (hostName != null) { - synchronized (rdnsLookupResults) { - rdnsLookupResults.put(rdnsLookupJob, hostName); - } - } - long lookupMillis = request.getLookupMillis(); - if (lookupMillis >= 0L) { - synchronized (rdnsLookupMillis) { - rdnsLookupMillis.add(lookupMillis); - } - } - } - } - } - - private class RdnsLookupRequest extends Thread { - private RdnsLookupWorker parent; - private String address, hostName; - private long lookupStartedMillis = -1L, lookupCompletedMillis = -1L; - public RdnsLookupRequest(RdnsLookupWorker parent, String address) { - this.parent = parent; - this.address = address; - } - public void run() { - this.lookupStartedMillis = time.currentTimeMillis(); - try { - String result = InetAddress.getByName(this.address).getHostName(); - synchronized (this) { - this.hostName = result; - } - } catch (UnknownHostException e) { - /* We'll try again the next time. */ - } - this.lookupCompletedMillis = time.currentTimeMillis(); - this.parent.interrupt(); - } - public synchronized String getHostName() { - return hostName; - } - public synchronized long getLookupMillis() { - return this.lookupCompletedMillis - this.lookupStartedMillis; - } - } - - private Time time; + Time time; public ReverseDomainNameResolver() { this.time = ApplicationFactory.getTime(); } - private static final long RDNS_LOOKUP_MAX_REQUEST_MILLIS = + static final long RDNS_LOOKUP_MAX_REQUEST_MILLIS = DateTimeHelper.TEN_SECONDS; - private static final long RDNS_LOOKUP_MAX_DURATION_MILLIS = + static final long RDNS_LOOKUP_MAX_DURATION_MILLIS = DateTimeHelper.FIVE_MINUTES; private static final long RDNS_LOOKUP_MAX_AGE_MILLIS = DateTimeHelper.TWELVE_HOURS; @@ -104,13 +33,13 @@ public class ReverseDomainNameResolver { private Map<String, Long> addressLastLookupTimes; - private Set<String> rdnsLookupJobs; + Set<String> rdnsLookupJobs; - private Map<String, String> rdnsLookupResults; + Map<String, String> rdnsLookupResults; - private List<Long> rdnsLookupMillis; + List<Long> rdnsLookupMillis; - private long startedRdnsLookups; + long startedRdnsLookups; private List<RdnsLookupWorker> rdnsLookupWorkers; @@ -132,7 +61,7 @@ public class ReverseDomainNameResolver { this.rdnsLookupMillis = new ArrayList<Long>(); this.rdnsLookupWorkers = new ArrayList<RdnsLookupWorker>(); for (int i = 0; i < RDNS_LOOKUP_WORKERS_NUM; i++) { - RdnsLookupWorker rdnsLookupWorker = new RdnsLookupWorker(); + RdnsLookupWorker rdnsLookupWorker = new RdnsLookupWorker(this); this.rdnsLookupWorkers.add(rdnsLookupWorker); rdnsLookupWorker.setDaemon(true); rdnsLookupWorker.start(); diff --git a/test/org/torproject/onionoo/ResourceServletTest.java b/test/org/torproject/onionoo/ResourceServletTest.java index 8cd3752..d27f499 100644 --- a/test/org/torproject/onionoo/ResourceServletTest.java +++ b/test/org/torproject/onionoo/ResourceServletTest.java @@ -22,10 +22,10 @@ import java.util.TreeSet; import org.junit.Before; import org.junit.Test; import org.torproject.onionoo.docs.UpdateStatus; +import org.torproject.onionoo.server.HttpServletRequestWrapper; +import org.torproject.onionoo.server.HttpServletResponseWrapper; import org.torproject.onionoo.server.NodeIndexer; import org.torproject.onionoo.server.ResourceServlet; -import org.torproject.onionoo.server.ResourceServlet.HttpServletRequestWrapper; -import org.torproject.onionoo.server.ResourceServlet.HttpServletResponseWrapper; import org.torproject.onionoo.util.ApplicationFactory; import org.torproject.onionoo.util.DateTimeHelper; import org.torproject.onionoo.util.Time;