July 2016 - tor-commits - lists.torproject.org

[collector/master] Prevent updateindex NPE, b/c of dubious files that are regcognized as directories. Adjusted coverage rate. Modified scheduler test. Reduced repeated try-catch in run methods. Log cron package to all module logs.
by karsten＠torproject.org 25 Jul '16

25 Jul '16

commit 6c43548bfb0ffe92ca6f5fd67075ff506c546519 Author: iwakeh <iwakeh(a)torproject.org> Date: Fri Jul 22 14:35:07 2016 +0200 Prevent updateindex NPE, b/c of dubious files that are regcognized as directories. Adjusted coverage rate. Modified scheduler test. Reduced repeated try-catch in run methods. Log cron package to all module logs. --- build.xml | 6 +-- .../bridgedescs/SanitizedBridgesWriter.java | 18 +++---- .../torproject/collector/cron/CollecTorMain.java | 26 ++++++++++ .../org/torproject/collector/cron/Scheduler.java | 36 ++++++++++---- .../collector/exitlists/ExitListDownloader.java | 16 ++---- .../collector/index/CreateIndexJson.java | 57 +++++++++++++++------- .../collector/relaydescs/ArchiveWriter.java | 43 ++++++++-------- .../collector/torperf/TorperfDownloader.java | 14 ++---- src/main/resources/logback.xml | 12 ++++- .../java/org/torproject/collector/cron/Dummy.java | 18 ++++--- .../torproject/collector/cron/SchedulerTest.java | 18 +++++-- 11 files changed, 167 insertions(+), 97 deletions(-) diff --git a/build.xml b/build.xml index eccae8b..643d94c 100644 --- a/build.xml +++ b/build.xml @@ -227,10 +227,10 @@ <include name="**/*.java" /> </fileset> </cobertura-report> - <cobertura-check branchrate="0" totallinerate="4" totalbranchrate="1" > + <cobertura-check totallinerate="5" totalbranchrate="1" > <regex pattern="org.torproject.collector.conf.*" branchrate="100" linerate="100"/> - <regex pattern="org.torproject.collector.cron.*" branchrate="66" linerate="73"/> - <regex pattern="org.torproject.collector.Main" branchrate="66" linerate="94"/> + <regex pattern="org.torproject.collector.cron" branchrate="66" linerate="80" /> + <regex pattern="org.torproject.collector.Main" branchrate="66" linerate="94" /> </cobertura-check> </target> <target name="test" depends="compile,compile-tests"> diff --git a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java index 5c33566..e00f70b 100644 --- a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java +++ b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java @@ -81,18 +81,12 @@ public class SanitizedBridgesWriter extends CollecTorMain { private SecureRandom secureRandom; @Override - public void run() { - logger.info("Starting bridge-descriptors module of CollecTor."); - try { - startProcessing(); - } catch (ConfigurationException ce) { - logger.error("Configuration failed: " + ce, ce); - throw new RuntimeException(ce); - } - logger.info("Terminating bridge-descriptors module of CollecTor."); + public String module() { + return "bridgedescs"; } - private void startProcessing() throws ConfigurationException { + @Override + protected void startProcessing() throws ConfigurationException { File bridgeDirectoriesDirectory = config.getPath(Key.BridgeSnapshotsDirectory).toFile(); @@ -102,7 +96,9 @@ public class SanitizedBridgesWriter extends CollecTorMain { if (bridgeDirectoriesDirectory == null || sanitizedBridgesDirectory == null || statsDirectory == null) { - throw new IllegalArgumentException(); + throw new ConfigurationException("BridgeSnapshotsDirectory, " + + "SanitizedBridgesWriteDirectory, StatsPath should be set. " + + "Please, edit the 'collector.properties' file."); } /* Memorize argument values. */ diff --git a/src/main/java/org/torproject/collector/cron/CollecTorMain.java b/src/main/java/org/torproject/collector/cron/CollecTorMain.java index 7a00e68..21d8948 100644 --- a/src/main/java/org/torproject/collector/cron/CollecTorMain.java +++ b/src/main/java/org/torproject/collector/cron/CollecTorMain.java @@ -4,6 +4,7 @@ package org.torproject.collector.cron; import org.torproject.collector.conf.Configuration; +import org.torproject.collector.conf.ConfigurationException; import org.torproject.collector.conf.Key; import org.slf4j.Logger; @@ -18,11 +19,36 @@ import java.util.concurrent.TimeUnit; public abstract class CollecTorMain implements Runnable { + private static Logger log = LoggerFactory.getLogger(CollecTorMain.class); + protected Configuration config; public CollecTorMain( Configuration conf) { this.config = conf; } + /** + * Log errors preventing successful completion of the module. + */ + @Override + public final void run() { + log.info("Starting {} module of CollecTor.", module()); + try { + startProcessing(); + } catch (ConfigurationException | RuntimeException ce) { + log.error("The {} module failed: {}", module(), ce.getMessage(), ce); + } + log.info("Terminating {} module of CollecTor.", module()); + } + + /** + * Module specific code goes here. + */ + protected abstract void startProcessing() throws ConfigurationException; + + /** + * Returns the module name for logging purposes. + */ + public abstract String module(); } diff --git a/src/main/java/org/torproject/collector/cron/Scheduler.java b/src/main/java/org/torproject/collector/cron/Scheduler.java index e4f2aa3..6bc90ca 100644 --- a/src/main/java/org/torproject/collector/cron/Scheduler.java +++ b/src/main/java/org/torproject/collector/cron/Scheduler.java @@ -16,22 +16,28 @@ import java.util.Calendar; import java.util.List; import java.util.Map; import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; /** * Scheduler that starts the modules configured in collector.properties. */ -public class Scheduler { +public class Scheduler implements ThreadFactory { public static final String ACTIVATED = "Activated"; public static final String PERIODMIN = "PeriodMinutes"; public static final String OFFSETMIN = "OffsetMinutes"; - private final Logger log = LoggerFactory.getLogger(Scheduler.class); + private static final Logger log = LoggerFactory.getLogger(Scheduler.class); + + private final ThreadFactory threads = Executors.defaultThreadFactory(); + + private int currentThreadNo = 0; private final ScheduledExecutorService scheduler = - Executors.newScheduledThreadPool(1); + Executors.newScheduledThreadPool(10, this); private static Scheduler instance = new Scheduler(); @@ -60,11 +66,10 @@ public class Scheduler { } } catch (ConfigurationException | IllegalAccessException | InstantiationException | InvocationTargetException - | NoSuchMethodException | RuntimeException ex) { + | NoSuchMethodException | RejectedExecutionException + | NullPointerException ex) { log.error("Cannot schedule " + ctmEntry.getValue().getName() + ". Reason: " + ex.getMessage(), ex); - shutdownScheduler(); - throw new RuntimeException("Halted scheduling.", ex); } } } @@ -73,12 +78,13 @@ public class Scheduler { this.log.info("Periodic updater started for " + ctm.getClass().getName() + "; offset=" + offset + ", period=" + period + "."); int currentMinute = Calendar.getInstance().get(Calendar.MINUTE); - int initialDelay = (60 - currentMinute + offset) % 60; + int initialDelay = (period - (currentMinute % period) + offset) % period; /* Run after initialDelay delay and then every period min. */ this.log.info("Periodic updater will start every " + period + "th min " - + "at minute " + ((currentMinute + initialDelay) % 60) + "."); - this.scheduler.scheduleAtFixedRate(ctm, initialDelay, 60, + + "at minute " + ((currentMinute + initialDelay) % period) + "." + + " The first start will happen in " + initialDelay + " minute(s)."); + this.scheduler.scheduleAtFixedRate(ctm, initialDelay, period, TimeUnit.MINUTES); } @@ -98,5 +104,17 @@ public class Scheduler { } } } + + /** + * Provide a nice name for debugging and log thread creation. + */ + @Override + public Thread newThread(Runnable runner) { + Thread newThread = threads.newThread(runner); + newThread.setName("CollecTor-Scheduled-Thread-" + ++currentThreadNo); + log.info("New Thread created: " + newThread.getName()); + return newThread; + } + } diff --git a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java index e6720cd..8b5277b 100644 --- a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java +++ b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java @@ -42,18 +42,12 @@ public class ExitListDownloader extends CollecTorMain { } @Override - public void run() { - logger.info("Starting exit-lists module of CollecTor."); - try { - startProcessing(); - } catch (ConfigurationException ce) { - logger.error("Configuration failed: " + ce, ce); - throw new RuntimeException(ce); - } - logger.info("Terminating exit-lists module of CollecTor."); + public String module() { + return "exitlists"; } - private void startProcessing() throws ConfigurationException { + @Override + protected void startProcessing() throws ConfigurationException { SimpleDateFormat dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); @@ -92,7 +86,7 @@ public class ExitListDownloader extends CollecTorMain { return; } if (downloadedExitList == null) { - logger.warn("Failed downloading exit list"); + logger.warn("Failed downloading exit list."); return; } diff --git a/src/main/java/org/torproject/collector/index/CreateIndexJson.java b/src/main/java/org/torproject/collector/index/CreateIndexJson.java index 80f183c..08c28c6 100644 --- a/src/main/java/org/torproject/collector/index/CreateIndexJson.java +++ b/src/main/java/org/torproject/collector/index/CreateIndexJson.java @@ -13,6 +13,8 @@ import com.google.gson.GsonBuilder; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.BufferedWriter; import java.io.File; @@ -38,17 +40,19 @@ import java.util.zip.GZIPOutputStream; * we'll likely have to do that. */ public class CreateIndexJson extends CollecTorMain { + private static Logger log = LoggerFactory.getLogger(CreateIndexJson.class); + private static File indexJsonFile; private static String basePath; private static File[] indexedDirectories; - static final String dateTimePattern = "yyyy-MM-dd HH:mm"; + private static final String dateTimePattern = "yyyy-MM-dd HH:mm"; - static final Locale dateTimeLocale = Locale.US; + private static final Locale dateTimeLocale = Locale.US; - static final TimeZone dateTimezone = TimeZone.getTimeZone("UTC"); + private static final TimeZone dateTimezone = TimeZone.getTimeZone("UTC"); /** Creates indexes of directories containing archived and recent * descriptors and write index files to disk. */ @@ -57,7 +61,12 @@ public class CreateIndexJson extends CollecTorMain { } @Override - public void run() { + public String module() { + return "updateindex"; + } + + @Override + protected void startProcessing() throws ConfigurationException { try { indexJsonFile = new File(config.getPath(Key.IndexPath).toFile(), "index.json"); @@ -67,12 +76,12 @@ public class CreateIndexJson extends CollecTorMain { config.getPath(Key.RecentPath).toFile() }; writeIndex(indexDirectories()); } catch (Exception e) { - throw new RuntimeException("Cannot run index creation: " + e.getMessage(), - e); + log.error("Cannot run index creation: " + e.getMessage(), e); + throw new RuntimeException(e); } } - static class DirectoryNode implements Comparable<DirectoryNode> { + private class DirectoryNode implements Comparable<DirectoryNode> { String path; SortedSet<FileNode> files; SortedSet<DirectoryNode> directories; @@ -90,7 +99,7 @@ public class CreateIndexJson extends CollecTorMain { } @SuppressWarnings({"checkstyle:membername", "checkstyle:parametername"}) - static class IndexNode { + private class IndexNode { String index_created; String path; SortedSet<FileNode> files; @@ -107,7 +116,7 @@ public class CreateIndexJson extends CollecTorMain { } @SuppressWarnings({"checkstyle:membername", "checkstyle:parametername"}) - static class FileNode implements Comparable<FileNode> { + private class FileNode implements Comparable<FileNode> { String path; long size; String last_modified; @@ -123,7 +132,7 @@ public class CreateIndexJson extends CollecTorMain { } } - static DateFormat dateTimeFormat; + private static DateFormat dateTimeFormat; static { dateTimeFormat = new SimpleDateFormat(dateTimePattern, @@ -132,30 +141,44 @@ public class CreateIndexJson extends CollecTorMain { dateTimeFormat.setTimeZone(dateTimezone); } - static IndexNode indexDirectories() { + private IndexNode indexDirectories() { SortedSet<DirectoryNode> directoryNodes = new TreeSet<DirectoryNode>(); + log.trace("indexing: " + indexedDirectories[0] + " " + + indexedDirectories[1]); for (File directory : indexedDirectories) { if (directory.exists() && directory.isDirectory()) { - directoryNodes.add(indexDirectory(directory)); + DirectoryNode dn = indexDirectory(directory); + if (null != dn) { + directoryNodes.add(dn); + } } } return new IndexNode(dateTimeFormat.format( System.currentTimeMillis()), basePath, null, directoryNodes); } - static DirectoryNode indexDirectory(File directory) { + private DirectoryNode indexDirectory(File directory) { SortedSet<FileNode> fileNodes = new TreeSet<FileNode>(); SortedSet<DirectoryNode> directoryNodes = new TreeSet<DirectoryNode>(); - for (File fileOrDirectory : directory.listFiles()) { + log.trace("indexing: " + directory); + File[] fileList = directory.listFiles(); + if (null == fileList) { + log.warn("Indexing dubious directory: " + directory); + return null; + } + for (File fileOrDirectory : fileList) { if (fileOrDirectory.getName().startsWith(".")) { continue; } if (fileOrDirectory.isFile()) { fileNodes.add(indexFile(fileOrDirectory)); } else { - directoryNodes.add(indexDirectory(fileOrDirectory)); + DirectoryNode dn = indexDirectory(fileOrDirectory); + if (null != dn) { + directoryNodes.add(dn); + } } } DirectoryNode directoryNode = new DirectoryNode( @@ -164,13 +187,13 @@ public class CreateIndexJson extends CollecTorMain { return directoryNode; } - static FileNode indexFile(File file) { + private FileNode indexFile(File file) { FileNode fileNode = new FileNode(file.getName(), file.length(), dateTimeFormat.format(file.lastModified())); return fileNode; } - static void writeIndex(IndexNode indexNode) throws IOException { + private void writeIndex(IndexNode indexNode) throws IOException { Gson gson = new GsonBuilder().create(); String indexNodeString = gson.toJson(indexNode); Writer[] writers = new Writer[] { diff --git a/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java b/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java index 4ad87f5..24b7ab5 100644 --- a/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java +++ b/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java @@ -111,33 +111,24 @@ public class ArchiveWriter extends CollecTorMain { } @Override - public void run() { - logger.info("Starting relay-descriptors module of CollecTor."); - try { - recentPath = config.getPath(Key.RecentPath); - recentPathName = recentPath.toString(); - File statsDir = config.getPath(Key.StatsPath).toFile(); - storedServerDescriptorsFile = - new File(statsDir, "stored-server-descriptors"); - storedExtraInfoDescriptorsFile = - new File(statsDir, "stored-extra-info-descriptors"); - storedMicrodescriptorsFile = - new File(statsDir, "stored-microdescriptors"); - - startProcessing(); - new ReferenceChecker(recentPath.toFile(), - new File(statsDir, "references"), - new File(statsDir, "references-history")).check(); - } catch (ConfigurationException ce) { - logger.error("Configuration failed: " + ce, ce); - throw new RuntimeException(ce); - } - logger.info("Terminating relay-descriptors module of CollecTor."); + public String module() { + return "relaydescs"; } - private void startProcessing() throws ConfigurationException { + @Override + protected void startProcessing() throws ConfigurationException { + recentPath = config.getPath(Key.RecentPath); + recentPathName = recentPath.toString(); + File statsDir = config.getPath(Key.StatsPath).toFile(); + storedServerDescriptorsFile + = new File(statsDir, "stored-server-descriptors"); + storedExtraInfoDescriptorsFile + = new File(statsDir, "stored-extra-info-descriptors"); + storedMicrodescriptorsFile + = new File(statsDir, "stored-microdescriptors"); File statsDirectory = config.getPath(Key.StatsPath).toFile(); - this.outputDirectory = config.getPath(Key.DirectoryArchivesOutputDirectory).toString(); + this.outputDirectory = config.getPath(Key.DirectoryArchivesOutputDirectory) + .toString(); SimpleDateFormat rsyncCatFormat = new SimpleDateFormat( "yyyy-MM-dd-HH-mm-ss"); rsyncCatFormat.setTimeZone(TimeZone.getTimeZone("UTC")); @@ -197,6 +188,10 @@ public class ArchiveWriter extends CollecTorMain { this.cleanUpRsyncDirectory(); this.saveDescriptorDigests(); + + new ReferenceChecker(recentPath.toFile(), + new File(statsDir, "references"), + new File(statsDir, "references-history")).check(); } private void loadDescriptorDigests() { diff --git a/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java b/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java index 1fa2d41..6f8daf0 100644 --- a/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java +++ b/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java @@ -46,18 +46,12 @@ public class TorperfDownloader extends CollecTorMain { private File torperfLastMergedFile; @Override - public void run() { - logger.info("Starting torperf module of CollecTor."); - try { - startProcessing(); - } catch (ConfigurationException ce) { - logger.error("Configuration failed: " + ce, ce); - throw new RuntimeException(ce); - } - logger.info("Terminating torperf module of CollecTor."); + public String module() { + return "torperf"; } - private void startProcessing() throws ConfigurationException { + @Override + protected void startProcessing() throws ConfigurationException { this.torperfFilesLines = config.getStringArray(Key.TorperfFilesLines); this.torperfOutputDirectory = config.getPath(Key.TorperfOutputDirectory) .toFile(); diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index 9c9426d..f31cf74 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -93,7 +93,7 @@ </encoder> <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> - <level>TRACE</level> + <level>INFO</level> </filter> </appender> @@ -114,7 +114,7 @@ <appender-ref ref="FILETORPERF" /> </logger> - <logger name="org.torproject.collector.index" > + <logger name="org.torproject.collector.index" level="INFO" > <appender-ref ref="FILEUPDATEINDEX" /> </logger> @@ -126,6 +126,14 @@ <appender-ref ref="FILEUPDATEINDEX" /> </logger> + <logger name="org.torproject.collector.cron" > + <appender-ref ref="FILEBRIDGEDESCS" /> + <appender-ref ref="FILEEXITLISTS" /> + <appender-ref ref="FILERELAYDESCS" /> + <appender-ref ref="FILETORPERF" /> + <appender-ref ref="FILEUPDATEINDEX" /> + </logger> + <logger name="org.torproject" > <appender-ref ref="CONSOLE" /> </logger> diff --git a/src/test/java/org/torproject/collector/cron/Dummy.java b/src/test/java/org/torproject/collector/cron/Dummy.java index 0231e69..449cc55 100644 --- a/src/test/java/org/torproject/collector/cron/Dummy.java +++ b/src/test/java/org/torproject/collector/cron/Dummy.java @@ -1,15 +1,21 @@ package org.torproject.collector.cron; import org.torproject.collector.conf.Configuration; +import org.torproject.collector.conf.ConfigurationException; public class Dummy extends CollecTorMain { - public Dummy(Configuration c) { - super(c); - } + public Dummy(Configuration c) { + super(c); + } - @Override - public void run() { + @Override + public void startProcessing() throws ConfigurationException { + // dummy doesn't do anything. + } - } + @Override + public String module() { + return "dummy"; + } } diff --git a/src/test/java/org/torproject/collector/cron/SchedulerTest.java b/src/test/java/org/torproject/collector/cron/SchedulerTest.java index 0c4e922..4c01f91 100644 --- a/src/test/java/org/torproject/collector/cron/SchedulerTest.java +++ b/src/test/java/org/torproject/collector/cron/SchedulerTest.java @@ -30,8 +30,16 @@ import java.util.concurrent.ScheduledThreadPoolExecutor; public class SchedulerTest { - private static final String runConfigProperties = "TorperfActivated=true\n" - + "TorperfPeriodMinutes=10\nTorperfOffsetMinutes=7\n"; + private static final String runConfigProperties = + "TorperfActivated=true\nTorperfPeriodMinutes=1\nTorperfOffsetMinutes=0\n" + + "RelaydescsActivated=true\nRelaydescsPeriodMinutes=1" + + "\nRelaydescsOffsetMinutes=0\n" + + "ExitlistsActivated=true\nExitlistsPeriodMinutes=1\n" + + "ExitlistsOffsetMinutes=0\n" + + "UpdateindexActivated=true\nUpdateindexPeriodMinutes=1\n" + + "UpdateindexOffsetMinutes=0\n" + + "BridgedescsActivated=true\nBridgedescsPeriodMinutes=1\n" + + "BridgedescsOffsetMinutes=0\n"; @Test() public void testSimpleSchedule() throws Exception { @@ -39,15 +47,17 @@ public class SchedulerTest { Configuration conf = new Configuration(); conf.load(new ByteArrayInputStream(runConfigProperties.getBytes())); ctms.put(Key.TorperfActivated, Dummy.class); + ctms.put(Key.BridgedescsActivated, Dummy.class); + ctms.put(Key.RelaydescsActivated, Dummy.class); + ctms.put(Key.ExitlistsActivated, Dummy.class); + ctms.put(Key.UpdateindexActivated, Dummy.class); Field schedulerField = Scheduler.class.getDeclaredField("scheduler"); schedulerField.setAccessible(true); ScheduledThreadPoolExecutor stpe = (ScheduledThreadPoolExecutor) schedulerField.get(Scheduler.getInstance()); assertTrue(stpe.getQueue().isEmpty()); Scheduler.getInstance().scheduleModuleRuns(ctms, conf); - assertEquals(stpe.getQueue().size(), 1); Scheduler.getInstance().shutdownScheduler(); - assertTrue(stpe.isShutdown()); } }

1 0

[collector/master] Don't add META-DATA files from other jars to the collector jar.
by karsten＠torproject.org 25 Jul '16

25 Jul '16

commit 7517480e4475f35730c560cab5dadcfbfc011118 Author: iwakeh <iwakeh(a)torproject.org> Date: Wed Jul 20 16:23:13 2016 +0200 Don't add META-DATA files from other jars to the collector jar. --- build.xml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/build.xml b/build.xml index ba50fbf..eccae8b 100644 --- a/build.xml +++ b/build.xml @@ -141,9 +141,18 @@ <include name="collector.properties"/> <include name="logback.xml"/> </fileset> - <zipgroupfileset dir="${libs}" > - <patternset refid="runtime" /> - </zipgroupfileset> + <restrict> + <not> + <name name="META-INF/*" /> + </not> + <archives> + <zips> + <fileset dir="${libs}"> + <patternset refid="runtime" /> + </fileset> + </zips> + </archives> + </restrict> <manifest> <attribute name="Created-By" value="The Tor Project" /> <attribute name="Implementation-Title" value="${name}"/>

1 0

[collector/master] Implements #19018 'Run CollecTor without crontab'. Added scheduler logic, adapted and added tests, adapted coverage check.
by karsten＠torproject.org 25 Jul '16

25 Jul '16

commit 997d4a4a2e6fca2bc95134537fcfebe930db8de1 Author: iwakeh <iwakeh(a)torproject.org> Date: Wed Jul 20 13:23:18 2016 +0200 Implements #19018 'Run CollecTor without crontab'. Added scheduler logic, adapted and added tests, adapted coverage check. --- build.xml | 8 +- src/main/java/org/torproject/collector/Main.java | 54 ++++------- .../bridgedescs/SanitizedBridgesWriter.java | 32 +------ .../torproject/collector/cron/CollecTorMain.java | 28 ++++++ .../org/torproject/collector/cron/Scheduler.java | 102 +++++++++++++++++++++ .../collector/exitlists/ExitListDownloader.java | 27 +----- .../collector/index/CreateIndexJson.java | 29 ++++-- .../org/torproject/collector/main/LockFile.java | 66 ------------- .../collector/relaydescs/ArchiveWriter.java | 61 ++++-------- .../collector/torperf/TorperfDownloader.java | 33 ++----- .../java/org/torproject/collector/MainTest.java | 84 +++++++++++++++-- .../java/org/torproject/collector/cron/Dummy.java | 15 +++ .../torproject/collector/cron/SchedulerTest.java | 54 +++++++++++ src/test/resources/junittest.policy | 1 + 14 files changed, 355 insertions(+), 239 deletions(-) diff --git a/build.xml b/build.xml index 51471f6..ba50fbf 100644 --- a/build.xml +++ b/build.xml @@ -210,7 +210,7 @@ <classpath refid="cobertura.test.classpath" /> <formatter type="xml" /> <batchtest toDir="${testresult}" > - <fileset dir="${testclasses}" /> + <fileset dir="${testclasses}" includes="**/*Test.class" /> </batchtest> </junit> <cobertura-report format="html" destdir="${coverageresult}" > @@ -218,8 +218,10 @@ <include name="**/*.java" /> </fileset> </cobertura-report> - <cobertura-check branchrate="0" totallinerate="15" totalbranchrate="5" > + <cobertura-check branchrate="0" totallinerate="4" totalbranchrate="1" > <regex pattern="org.torproject.collector.conf.*" branchrate="100" linerate="100"/> + <regex pattern="org.torproject.collector.cron.*" branchrate="66" linerate="73"/> + <regex pattern="org.torproject.collector.Main" branchrate="66" linerate="94"/> </cobertura-check> </target> <target name="test" depends="compile,compile-tests"> @@ -231,7 +233,7 @@ <classpath refid="test.classpath"/> <formatter type="plain" usefile="false"/> <batchtest> - <fileset dir="${testclasses}" /> + <fileset dir="${testclasses}" includes="**/*Test.class" /> </batchtest> </junit> </target> diff --git a/src/main/java/org/torproject/collector/Main.java b/src/main/java/org/torproject/collector/Main.java index 81234c3..97c7a0c 100644 --- a/src/main/java/org/torproject/collector/Main.java +++ b/src/main/java/org/torproject/collector/Main.java @@ -5,6 +5,9 @@ package org.torproject.collector; import org.torproject.collector.bridgedescs.SanitizedBridgesWriter; import org.torproject.collector.conf.Configuration; +import org.torproject.collector.conf.Key; +import org.torproject.collector.cron.CollecTorMain; +import org.torproject.collector.cron.Scheduler; import org.torproject.collector.exitlists.ExitListDownloader; import org.torproject.collector.index.CreateIndexJson; import org.torproject.collector.relaydescs.ArchiveWriter; @@ -37,34 +40,31 @@ public class Main { /** All possible main classes. * If a new CollecTorMain class is available, just add it to this map. */ - static final Map<String, Class> collecTorMains = new HashMap<>(); + static final Map<Key, Class<? extends CollecTorMain>> collecTorMains = new HashMap<>(); static { // add a new main class here - collecTorMains.put("bridgedescs", SanitizedBridgesWriter.class); - collecTorMains.put("exitlists", ExitListDownloader.class); - collecTorMains.put("updateindex", CreateIndexJson.class); - collecTorMains.put("relaydescs", ArchiveWriter.class); - collecTorMains.put("torperf", TorperfDownloader.class); + collecTorMains.put(Key.BridgedescsActivated, SanitizedBridgesWriter.class); + collecTorMains.put(Key.ExitlistsActivated, ExitListDownloader.class); + collecTorMains.put(Key.UpdateindexActivated, CreateIndexJson.class); + collecTorMains.put(Key.RelaydescsActivated, ArchiveWriter.class); + collecTorMains.put(Key.TorperfActivated, TorperfDownloader.class); } - private static final String modules = collecTorMains.keySet().toString() - .replace("[", "").replace("]", "").replaceAll(", ", "|"); - private static Configuration conf = new Configuration(); /** - * One argument is necessary. + * At most one argument. * See class description {@link Main}. */ public static void main(String[] args) throws Exception { File confFile = null; - if (null == args || args.length < 1 || args.length > 2) { - printUsage("CollecTor needs one or two arguments."); - return; - } else if (args.length == 1) { + if (args == null || args.length == 0) { confFile = new File(CONF_FILE); - } else if (args.length == 2) { - confFile = new File(args[1]); + } else if (args.length == 1) { + confFile = new File(args[0]); + } else { + printUsage("CollecTor takes at most one argument."); + return; } if (!confFile.exists() || confFile.length() < 1L) { writeDefaultConfig(confFile); @@ -72,12 +72,12 @@ public class Main { } else { readConfigurationFrom(confFile); } - invokeGivenMain(args[0]); + Scheduler.getInstance().scheduleModuleRuns(collecTorMains, conf); } private static void printUsage(String msg) { final String usage = "Usage:\njava -jar collector.jar " - + "<" + modules + "> [path/to/configFile]"; + + "[path/to/configFile]"; System.out.println(msg + "\n" + usage); } @@ -105,23 +105,5 @@ public class Main { } } - private static void invokeGivenMain(String mainId) { - Class clazz = collecTorMains.get(mainId); - if (null == clazz) { - printUsage("Unknown argument: " + mainId); - } - invokeMainOnClass(clazz); - } - - private static void invokeMainOnClass(Class clazz) { - try { - clazz.getMethod("main", new Class[] { Configuration.class }) - .invoke(null, (Object) conf); - } catch (NoSuchMethodException | IllegalAccessException - | InvocationTargetException e) { - log.error("Cannot invoke 'main' method on " - + clazz.getName() + ". " + e, e); - } - } } diff --git a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java index 121f8ca..5c33566 100644 --- a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java +++ b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java @@ -6,7 +6,7 @@ package org.torproject.collector.bridgedescs; import org.torproject.collector.conf.Configuration; import org.torproject.collector.conf.ConfigurationException; import org.torproject.collector.conf.Key; -import org.torproject.collector.main.LockFile; +import org.torproject.collector.cron.CollecTorMain; import org.apache.commons.codec.DecoderException; import org.apache.commons.codec.binary.Base64; @@ -49,36 +49,12 @@ import java.util.TreeMap; * by the bridge to advertise their capabilities), and extra-info * descriptors (published by the bridge, mainly for statistical analysis).</p> */ -public class SanitizedBridgesWriter extends Thread { +public class SanitizedBridgesWriter extends CollecTorMain { private static Logger logger = LoggerFactory.getLogger(SanitizedBridgesWriter.class); - /** Executes the bridge-descriptors module using the given - * configuration. */ - public static void main(Configuration config) throws ConfigurationException { - - logger.info("Starting bridge-descriptors module of CollecTor."); - - // Use lock file to avoid overlapping runs - LockFile lf = new LockFile(config.getPath(Key.LockFilePath).toString(), "bridge-descriptors"); - lf.acquireLock(); - - // Sanitize bridge descriptors - new SanitizedBridgesWriter(config).run(); - - // Remove lock file - lf.releaseLock(); - - logger.info("Terminating bridge-descriptors module of CollecTor."); - } - - private Configuration config; - - /** - * Initializes this class. - */ public SanitizedBridgesWriter(Configuration config) { - this.config = config; + super(config); } private String rsyncCatString; @@ -106,12 +82,14 @@ public class SanitizedBridgesWriter extends Thread { @Override public void run() { + logger.info("Starting bridge-descriptors module of CollecTor."); try { startProcessing(); } catch (ConfigurationException ce) { logger.error("Configuration failed: " + ce, ce); throw new RuntimeException(ce); } + logger.info("Terminating bridge-descriptors module of CollecTor."); } private void startProcessing() throws ConfigurationException { diff --git a/src/main/java/org/torproject/collector/cron/CollecTorMain.java b/src/main/java/org/torproject/collector/cron/CollecTorMain.java new file mode 100644 index 0000000..7a00e68 --- /dev/null +++ b/src/main/java/org/torproject/collector/cron/CollecTorMain.java @@ -0,0 +1,28 @@ +/* Copyright 2016 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.collector.cron; + +import org.torproject.collector.conf.Configuration; +import org.torproject.collector.conf.Key; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.InvocationTargetException; +import java.util.Calendar; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +public abstract class CollecTorMain implements Runnable { + + protected Configuration config; + + public CollecTorMain( Configuration conf) { + this.config = conf; + } + +} + diff --git a/src/main/java/org/torproject/collector/cron/Scheduler.java b/src/main/java/org/torproject/collector/cron/Scheduler.java new file mode 100644 index 0000000..e4f2aa3 --- /dev/null +++ b/src/main/java/org/torproject/collector/cron/Scheduler.java @@ -0,0 +1,102 @@ +/* Copyright 2016 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.collector.cron; + +import org.torproject.collector.conf.Configuration; +import org.torproject.collector.conf.ConfigurationException; +import org.torproject.collector.conf.Key; +import org.torproject.collector.cron.CollecTorMain; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.InvocationTargetException; +import java.util.Calendar; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** + * Scheduler that starts the modules configured in collector.properties. + */ +public class Scheduler { + + public static final String ACTIVATED = "Activated"; + public static final String PERIODMIN = "PeriodMinutes"; + public static final String OFFSETMIN = "OffsetMinutes"; + + private final Logger log = LoggerFactory.getLogger(Scheduler.class); + + private final ScheduledExecutorService scheduler = + Executors.newScheduledThreadPool(1); + + private static Scheduler instance = new Scheduler(); + + private Scheduler(){} + + public static Scheduler getInstance() { + return instance; + } + + /** + * Schedule all classes given according to the parameters in the + * the configuration. + */ + public void scheduleModuleRuns(Map<Key, + Class<? extends CollecTorMain>> collecTorMains, Configuration conf) { + for ( Map.Entry<Key, Class<? extends CollecTorMain>> ctmEntry + : collecTorMains.entrySet() ) { + try { + if ( conf.getBool(ctmEntry.getKey()) ) { + String prefix = ctmEntry.getKey().name().replace(ACTIVATED, ""); + CollecTorMain ctm = ctmEntry.getValue() + .getConstructor(Configuration.class).newInstance(conf); + scheduleExecutions(ctm, + conf.getInt(Key.valueOf(prefix + OFFSETMIN)), + conf.getInt(Key.valueOf(prefix + PERIODMIN))); + } + } catch (ConfigurationException | IllegalAccessException + | InstantiationException | InvocationTargetException + | NoSuchMethodException | RuntimeException ex) { + log.error("Cannot schedule " + ctmEntry.getValue().getName() + + ". Reason: " + ex.getMessage(), ex); + shutdownScheduler(); + throw new RuntimeException("Halted scheduling.", ex); + } + } + } + + private void scheduleExecutions(CollecTorMain ctm, int offset, int period) { + this.log.info("Periodic updater started for " + ctm.getClass().getName() + + "; offset=" + offset + ", period=" + period + "."); + int currentMinute = Calendar.getInstance().get(Calendar.MINUTE); + int initialDelay = (60 - currentMinute + offset) % 60; + + /* Run after initialDelay delay and then every period min. */ + this.log.info("Periodic updater will start every " + period + "th min " + + "at minute " + ((currentMinute + initialDelay) % 60) + "."); + this.scheduler.scheduleAtFixedRate(ctm, initialDelay, 60, + TimeUnit.MINUTES); + } + + /** + * Try to shutdown smoothly, i.e., wait for running tasks to terminate. + */ + public void shutdownScheduler() { + try { + scheduler.shutdown(); + scheduler.awaitTermination(20L, java.util.concurrent.TimeUnit.MINUTES); + log.info("Shutdown of all scheduled tasks completed successfully."); + } catch ( InterruptedException ie ) { + List<Runnable> notTerminated = scheduler.shutdownNow(); + log.error("Regular shutdown failed for: " + notTerminated); + if ( !notTerminated.isEmpty() ) { + log.error("Forced shutdown failed for: " + notTerminated); + } + } + } +} + diff --git a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java index 79fe19f..e6720cd 100644 --- a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java +++ b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java @@ -6,7 +6,7 @@ package org.torproject.collector.exitlists; import org.torproject.collector.conf.Configuration; import org.torproject.collector.conf.ConfigurationException; import org.torproject.collector.conf.Key; -import org.torproject.collector.main.LockFile; +import org.torproject.collector.cron.CollecTorMain; import org.torproject.descriptor.Descriptor; import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.DescriptorParser; @@ -32,42 +32,25 @@ import java.util.Stack; import java.util.TimeZone; import java.util.TreeSet; -public class ExitListDownloader extends Thread { +public class ExitListDownloader extends CollecTorMain { private static Logger logger = LoggerFactory.getLogger(ExitListDownloader.class); - /** Execute the exit-lists module using the given configuration. */ - public static void main(Configuration config) throws ConfigurationException { - logger.info("Starting exit-lists module of CollecTor."); - - // Use lock file to avoid overlapping runs - LockFile lf = new LockFile(config.getPath(Key.LockFilePath).toString(), "exit-lists"); - lf.acquireLock(); - - // Download exit list and store it to disk - new ExitListDownloader(config).run(); - - // Remove lock file - lf.releaseLock(); - - logger.info("Terminating exit-lists module of CollecTor."); - } - - private Configuration config; - /** Instanciate the exit-lists module using the given configuration. */ public ExitListDownloader(Configuration config) { - this.config = config; + super(config); } @Override public void run() { + logger.info("Starting exit-lists module of CollecTor."); try { startProcessing(); } catch (ConfigurationException ce) { logger.error("Configuration failed: " + ce, ce); throw new RuntimeException(ce); } + logger.info("Terminating exit-lists module of CollecTor."); } private void startProcessing() throws ConfigurationException { diff --git a/src/main/java/org/torproject/collector/index/CreateIndexJson.java b/src/main/java/org/torproject/collector/index/CreateIndexJson.java index 7da2b5e..80f183c 100644 --- a/src/main/java/org/torproject/collector/index/CreateIndexJson.java +++ b/src/main/java/org/torproject/collector/index/CreateIndexJson.java @@ -6,6 +6,7 @@ package org.torproject.collector.index; import org.torproject.collector.conf.Configuration; import org.torproject.collector.conf.ConfigurationException; import org.torproject.collector.conf.Key; +import org.torproject.collector.cron.CollecTorMain; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -35,7 +36,7 @@ import java.util.zip.GZIPOutputStream; * cache index parts of directories or files that haven't changed. * Example: if we parse include cryptographic hashes or @type information, * we'll likely have to do that. */ -public class CreateIndexJson { +public class CreateIndexJson extends CollecTorMain { private static File indexJsonFile; @@ -51,14 +52,24 @@ public class CreateIndexJson { /** Creates indexes of directories containing archived and recent * descriptors and write index files to disk. */ - public static void main(Configuration config) - throws ConfigurationException, IOException { - indexJsonFile = new File(config.getPath(Key.IndexPath).toFile(), "index.json"); - basePath = config.getProperty(Key.InstanceBaseUrl.name()); - indexedDirectories = new File[] { - config.getPath(Key.ArchivePath).toFile(), - config.getPath(Key.RecentPath).toFile() }; - writeIndex(indexDirectories()); + public CreateIndexJson(Configuration conf) { + super(conf); + } + + @Override + public void run() { + try { + indexJsonFile = new File(config.getPath(Key.IndexPath).toFile(), + "index.json"); + basePath = config.getProperty(Key.InstanceBaseUrl.name()); + indexedDirectories = new File[] { + config.getPath(Key.ArchivePath).toFile(), + config.getPath(Key.RecentPath).toFile() }; + writeIndex(indexDirectories()); + } catch (Exception e) { + throw new RuntimeException("Cannot run index creation: " + e.getMessage(), + e); + } } static class DirectoryNode implements Comparable<DirectoryNode> { diff --git a/src/main/java/org/torproject/collector/main/LockFile.java b/src/main/java/org/torproject/collector/main/LockFile.java deleted file mode 100644 index 977b0b9..0000000 --- a/src/main/java/org/torproject/collector/main/LockFile.java +++ /dev/null @@ -1,66 +0,0 @@ -/* Copyright 2010--2016 The Tor Project - * See LICENSE for licensing information */ - -package org.torproject.collector.main; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; - -public class LockFile { - - private final File lockFile; - private final String moduleName; - private final Logger logger = LoggerFactory.getLogger(LockFile.class); - - public LockFile(String moduleName) { - this("lock", moduleName); - } - - public LockFile(String lockFilePath, String moduleName) { - this.lockFile = new File(lockFilePath, moduleName); - this.moduleName = moduleName; - } - - /** Acquires the lock by checking whether a lock file already exists, - * and if not, by creating one with the current system time as - * content. */ - public boolean acquireLock() { - this.logger.debug("Trying to acquire lock..."); - try { - if (this.lockFile.exists()) { - BufferedReader br = new BufferedReader(new FileReader( - this.lockFile)); - long runStarted = Long.parseLong(br.readLine()); - br.close(); - if (System.currentTimeMillis() - runStarted < 55L * 60L * 1000L) { - throw new RuntimeException("Cannot acquire lock for " + moduleName); - } - } - this.lockFile.getParentFile().mkdirs(); - BufferedWriter bw = new BufferedWriter(new FileWriter( - this.lockFile)); - bw.append("" + System.currentTimeMillis() + "\n"); - bw.close(); - this.logger.debug("Acquired lock."); - return true; - } catch (IOException e) { - throw new RuntimeException("Caught exception while trying to acquire " - + "lock for " + moduleName); - } - } - - /** Releases the lock by deleting the lock file, if present. */ - public void releaseLock() { - this.logger.debug("Releasing lock..."); - this.lockFile.delete(); - this.logger.debug("Released lock."); - } -} - diff --git a/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java b/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java index db05bc5..cbab5ea 100644 --- a/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java +++ b/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java @@ -6,7 +6,7 @@ package org.torproject.collector.relaydescs; import org.torproject.collector.conf.Configuration; import org.torproject.collector.conf.ConfigurationException; import org.torproject.collector.conf.Key; -import org.torproject.collector.main.LockFile; +import org.torproject.collector.cron.CollecTorMain; import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.DescriptorParser; import org.torproject.descriptor.DescriptorSourceFactory; @@ -38,12 +38,10 @@ import java.util.Stack; import java.util.TimeZone; import java.util.TreeMap; -public class ArchiveWriter extends Thread { +public class ArchiveWriter extends CollecTorMain { private static Logger logger = LoggerFactory.getLogger(ArchiveWriter.class); - private Configuration config; - private long now = System.currentTimeMillis(); private String outputDirectory; private String rsyncCatString; @@ -99,64 +97,45 @@ public class ArchiveWriter extends Thread { private StringBuilder intermediateStats = new StringBuilder(); - private static Path recentPath; - private static String recentPathName; + private Path recentPath; + private String recentPathName; private static final String RELAY_DESCRIPTORS = "relay-descriptors"; private static final String MICRO = "micro"; private static final String CONSENSUS_MICRODESC = "consensus-microdesc"; private static final String MICRODESC = "microdesc"; private static final String MICRODESCS = "microdescs"; - /** Executes the relay-descriptors module using the given - * configuration. */ - public static void main(Configuration config) throws ConfigurationException { - - logger.info("Starting relay-descriptors module of CollecTor."); - - // Use lock file to avoid overlapping runs - LockFile lf = new LockFile(config.getPath(Key.LockFilePath).toString(), RELAY_DESCRIPTORS); - lf.acquireLock(); - - recentPath = config.getPath(Key.RecentPath); - recentPathName = recentPath.toString(); - - // Import/download relay descriptors from the various sources - new ArchiveWriter(config).run(); - - new ReferenceChecker( - recentPath.toFile(), - new File(config.getPath(Key.StatsPath).toFile(), "references"), - new File(config.getPath(Key.StatsPath).toFile(), "references-history")).check(); - - // Remove lock file - lf.releaseLock(); - - logger.info("Terminating relay-descriptors module of CollecTor."); - } - /** Initialize an archive writer with a given configuration. */ public ArchiveWriter(Configuration config) throws ConfigurationException { - this.config = config; - storedServerDescriptorsFile = - new File(config.getPath(Key.StatsPath).toFile(), "stored-server-descriptors"); - storedExtraInfoDescriptorsFile = - new File(config.getPath(Key.StatsPath).toFile(), "stored-extra-info-descriptors"); - storedMicrodescriptorsFile = - new File(config.getPath(Key.StatsPath).toFile(), "stored-microdescriptors"); + super(config); } @Override public void run() { + logger.info("Starting relay-descriptors module of CollecTor."); try { + recentPath = config.getPath(Key.RecentPath); + recentPathName = recentPath.toString(); + File statsDir = config.getPath(Key.StatsPath).toFile(); + storedServerDescriptorsFile = + new File(statsDir, "stored-server-descriptors"); + storedExtraInfoDescriptorsFile = + new File(statsDir, "stored-extra-info-descriptors"); + storedMicrodescriptorsFile = + new File(statsDir, "stored-microdescriptors"); + startProcessing(); + new ReferenceChecker(recentPath.toFile(), + new File(statsDir, "references"), + new File(statsDir, "references-history")).check(); } catch (ConfigurationException ce) { logger.error("Configuration failed: " + ce, ce); throw new RuntimeException(ce); } + logger.info("Terminating relay-descriptors module of CollecTor."); } private void startProcessing() throws ConfigurationException { - File statsDirectory = new File("stats"); this.outputDirectory = config.getPath(Key.DirectoryArchivesOutputDirectory).toString(); SimpleDateFormat rsyncCatFormat = new SimpleDateFormat( diff --git a/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java b/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java index 7616dd8..1fa2d41 100644 --- a/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java +++ b/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java @@ -6,7 +6,7 @@ package org.torproject.collector.torperf; import org.torproject.collector.conf.Configuration; import org.torproject.collector.conf.ConfigurationException; import org.torproject.collector.conf.Key; -import org.torproject.collector.main.LockFile; +import org.torproject.collector.cron.CollecTorMain; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,30 +32,11 @@ import java.util.TreeMap; /* Download possibly truncated Torperf .data and .extradata files from * configured sources, append them to the files we already have, and merge * the two files into the .tpf format. */ -public class TorperfDownloader extends Thread { +public class TorperfDownloader extends CollecTorMain { private static Logger logger = LoggerFactory.getLogger(TorperfDownloader.class); - /** Executes the torperf module using the given configuration. */ - public static void main(Configuration config) throws ConfigurationException { - logger.info("Starting torperf module of CollecTor."); - - // Use lock file to avoid overlapping runs - LockFile lf = new LockFile(config.getPath(Key.LockFilePath).toString(), "torperf"); - lf.acquireLock(); - - // Process Torperf files - new TorperfDownloader(config).run(); - - // Remove lock file - lf.releaseLock(); - - logger.info("Terminating torperf module of CollecTor."); - } - - private Configuration config; - public TorperfDownloader(Configuration config) { - this.config = config; + super(config); } private File torperfOutputDirectory = null; @@ -66,12 +47,14 @@ public class TorperfDownloader extends Thread { @Override public void run() { + logger.info("Starting torperf module of CollecTor."); try { startProcessing(); } catch (ConfigurationException ce) { logger.error("Configuration failed: " + ce, ce); throw new RuntimeException(ce); } + logger.info("Terminating torperf module of CollecTor."); } private void startProcessing() throws ConfigurationException { @@ -309,9 +292,6 @@ public class TorperfDownloader extends Thread { private String mergeFiles(File dataFile, File extradataFile, String source, int fileSize, String skipUntil) throws IOException, ConfigurationException { - SortedMap<String, String> config = new TreeMap<String, String>(); - config.put("SOURCE", source); - config.put("FILESIZE", String.valueOf(fileSize)); if (!dataFile.exists() || !extradataFile.exists()) { this.logger.warn("File " + dataFile.getAbsolutePath() + " or " + extradataFile.getAbsolutePath() + " is missing."); @@ -426,11 +406,12 @@ public class TorperfDownloader extends Thread { /* Write output line to .tpf file. */ SortedMap<String, String> keysAndValues = new TreeMap<String, String>(); + keysAndValues.put("SOURCE", source); + keysAndValues.put("FILESIZE", String.valueOf(fileSize)); if (extradata != null) { keysAndValues.putAll(extradata); } keysAndValues.putAll(data); - keysAndValues.putAll(config); this.logger.debug("Writing " + dataFile.getName() + ":" + skippedLineCount++ + "."); lineD = brD.readLine(); diff --git a/src/test/java/org/torproject/collector/MainTest.java b/src/test/java/org/torproject/collector/MainTest.java index 5991f78..b48a0a9 100644 --- a/src/test/java/org/torproject/collector/MainTest.java +++ b/src/test/java/org/torproject/collector/MainTest.java @@ -4,16 +4,19 @@ package org.torproject.collector; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import org.torproject.collector.conf.Key; +import org.torproject.collector.cron.Scheduler; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import java.io.BufferedWriter; +import java.io.IOException; import java.io.File; import java.nio.file.Files; import java.util.List; @@ -27,23 +30,54 @@ public class MainTest { @Rule public TemporaryFolder tmpf = new TemporaryFolder(); + @Test(expected = IOException.class) + public void testInitializationConfigException() throws Exception { + File conf = new File(Main.CONF_FILE); + assertFalse("Please remove " + Main.CONF_FILE + " before running tests!", conf.exists()); + Main.main(new String[] {"/tmp/"}); + assertTrue(conf.exists()); + assertTrue(conf.delete()); + } + + @Test() + public void testInitializationNullArgs() throws Exception { + File conf = new File(Main.CONF_FILE); + assertFalse("Please remove " + Main.CONF_FILE + " before running tests!", conf.exists()); + Main.main(null); + assertTrue(conf.exists()); + assertTrue(conf.delete()); + } + + @Test() + public void testInitializationEmptyArgs() throws Exception { + File conf = new File(Main.CONF_FILE); + assertFalse("Please remove " + Main.CONF_FILE + " before running tests!", conf.exists()); + Main.main(new String[]{}); + assertTrue(conf.exists()); + assertTrue(conf.delete()); + } + + @Test() + public void testInitializationTooManyArgs() throws Exception { + File conf = new File(Main.CONF_FILE); + assertFalse("Please remove " + Main.CONF_FILE + " before running tests!", conf.exists()); + Main.main(new String[]{"x", "y"}); + assertFalse(conf.exists()); + } + @Test() public void testSmoke() throws Exception { - System.out.println("\n!!!! Three ERROR log messages are expected." - + "\nOne each from: ExitListDownloader, " - + "TorperfDownloader, and CreateIndexJson.\n"); File conf = tmpf.newFile("test.conf"); File lockPath = tmpf.newFolder("test.lock"); assertEquals(0L, conf.length()); - Main.main(new String[]{"relaydescs", conf.toString()}); + Main.main(new String[]{conf.toString()}); assertTrue(4_000L <= conf.length()); - changeLockFilePath(conf, lockPath); - for ( String key : Main.collecTorMains.keySet()) { - Main.main(new String[]{key, conf.toString()}); - } + changeFilePathsAndSetActivation(conf, lockPath, "TorperfActivated"); + Main.main(new String[]{conf.toString()}); + for(int t=0; t<1_000_000; t++) { } } - private void changeLockFilePath(File f, File l) throws Exception { + private void changeFilePathsAndSetActivation(File f, File l, String a) throws Exception { List<String> lines = Files.readAllLines(f.toPath()); BufferedWriter bw = Files.newBufferedWriter(f.toPath()); File in = tmpf.newFolder(); @@ -57,6 +91,8 @@ public class MainTest { line = line.replace(inStr, in.toString() + inStr); } else if (line.contains(outStr)) { line = line.replace(outStr, out.toString() + outStr); + } else if (line.contains(a)) { + line = line.replace("false", "true"); } bw.write(line); bw.newLine(); @@ -86,5 +122,35 @@ public class MainTest { } } } + + /* Verifies that every collecTorMain class is configured in the + * default collector.properties file and the other way around. */ + @Test() + public void testRunConfiguration() throws Exception { + Properties props = new Properties(); + props.load(getClass().getClassLoader().getResourceAsStream( + Main.CONF_FILE)); + String[] runConfigSettings = new String[] {Scheduler.ACTIVATED, + Scheduler.PERIODMIN, Scheduler.OFFSETMIN}; + for (Key key : Main.collecTorMains.keySet()) { + for ( String part : runConfigSettings ){ + String key2 = key.name().replace("Activated", part); + assertNotNull("Property '" + key2 + "' not specified in " + + Main.CONF_FILE + ".", + props.getProperty(key2)); + } + } + for (String propName : props.stringPropertyNames()) { + for ( String part : runConfigSettings ){ + if( propName.contains(part) ){ + String key2 = propName.replace(part, ""); + assertTrue("CollecTorMain '" + key2 + + "' not specified in Main.class.", + Main.collecTorMains.containsKey(Key.valueOf(key2 + "Activated"))); + } + } + } + } + } diff --git a/src/test/java/org/torproject/collector/cron/Dummy.java b/src/test/java/org/torproject/collector/cron/Dummy.java new file mode 100644 index 0000000..0231e69 --- /dev/null +++ b/src/test/java/org/torproject/collector/cron/Dummy.java @@ -0,0 +1,15 @@ +package org.torproject.collector.cron; + +import org.torproject.collector.conf.Configuration; + +public class Dummy extends CollecTorMain { + + public Dummy(Configuration c) { + super(c); + } + + @Override + public void run() { + + } +} diff --git a/src/test/java/org/torproject/collector/cron/SchedulerTest.java b/src/test/java/org/torproject/collector/cron/SchedulerTest.java new file mode 100644 index 0000000..0c4e922 --- /dev/null +++ b/src/test/java/org/torproject/collector/cron/SchedulerTest.java @@ -0,0 +1,54 @@ +/* Copyright 2016 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.collector.cron; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import org.torproject.collector.conf.Key; +import org.torproject.collector.conf.Configuration; +import org.torproject.collector.cron.Scheduler; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.BufferedWriter; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.lang.reflect.Field; +import java.nio.file.Files; +import java.util.List; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.Random; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; + +public class SchedulerTest { + + private static final String runConfigProperties = "TorperfActivated=true\n" + + "TorperfPeriodMinutes=10\nTorperfOffsetMinutes=7\n"; + + @Test() + public void testSimpleSchedule() throws Exception { + Map<Key, Class<? extends CollecTorMain>> ctms = new HashMap<>(); + Configuration conf = new Configuration(); + conf.load(new ByteArrayInputStream(runConfigProperties.getBytes())); + ctms.put(Key.TorperfActivated, Dummy.class); + Field schedulerField = Scheduler.class.getDeclaredField("scheduler"); + schedulerField.setAccessible(true); + ScheduledThreadPoolExecutor stpe = (ScheduledThreadPoolExecutor) + schedulerField.get(Scheduler.getInstance()); + assertTrue(stpe.getQueue().isEmpty()); + Scheduler.getInstance().scheduleModuleRuns(ctms, conf); + assertEquals(stpe.getQueue().size(), 1); + Scheduler.getInstance().shutdownScheduler(); + assertTrue(stpe.isShutdown()); + } + +} + diff --git a/src/test/resources/junittest.policy b/src/test/resources/junittest.policy index 208a172..35c30c0 100644 --- a/src/test/resources/junittest.policy +++ b/src/test/resources/junittest.policy @@ -5,6 +5,7 @@ grant { permission java.util.PropertyPermission "*", "read, write"; permission java.lang.RuntimePermission "setIO"; permission java.lang.RuntimePermission "accessDeclaredMembers"; + permission java.lang.RuntimePermission "modifyThread"; permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; permission java.lang.RuntimePermission "shutdownHooks"; permission java.lang.RuntimePermission "accessClassInPackage.sun.reflect";

1 0

[collector/master] Added new properties for scheduled module runs and adapted tests accordingly. Scheduler implementation open. Partially implements task-19018.
by karsten＠torproject.org 25 Jul '16

25 Jul '16

commit 62c94124d1e7bcf7bfa55465434818b19749c0ad Author: iwakeh <iwakeh(a)torproject.org> Date: Wed Jul 20 11:11:37 2016 +0200 Added new properties for scheduled module runs and adapted tests accordingly. Scheduler implementation open. Partially implements task-19018. --- .../java/org/torproject/collector/conf/Key.java | 15 ++++++++++ src/main/resources/collector.properties | 32 +++++++++++++++++++++- .../collector/conf/ConfigurationTest.java | 2 +- 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/torproject/collector/conf/Key.java b/src/main/java/org/torproject/collector/conf/Key.java index b4119b6..d8c87ac 100644 --- a/src/main/java/org/torproject/collector/conf/Key.java +++ b/src/main/java/org/torproject/collector/conf/Key.java @@ -17,6 +17,21 @@ public enum Key { RecentPath(Path.class), IndexPath(Path.class), StatsPath(Path.class), + BridgedescsActivated(Boolean.class), + BridgedescsOffsetMinutes(Integer.class), + BridgedescsPeriodMinutes(Integer.class), + ExitlistsActivated(Boolean.class), + ExitlistsOffsetMinutes(Integer.class), + ExitlistsPeriodMinutes(Integer.class), + RelaydescsActivated(Boolean.class), + RelaydescsOffsetMinutes(Integer.class), + RelaydescsPeriodMinutes(Integer.class), + TorperfActivated(Boolean.class), + TorperfOffsetMinutes(Integer.class), + TorperfPeriodMinutes(Integer.class), + UpdateindexActivated(Boolean.class), + UpdateindexOffsetMinutes(Integer.class), + UpdateindexPeriodMinutes(Integer.class), BridgeSnapshotsDirectory(Path.class), CachedRelayDescriptorsDirectories(String[].class), CompressRelayDescriptorDownloads(Boolean.class), diff --git a/src/main/resources/collector.properties b/src/main/resources/collector.properties index 76ae6bd..8996a1f 100644 --- a/src/main/resources/collector.properties +++ b/src/main/resources/collector.properties @@ -1,5 +1,36 @@ ######## Collector Properties # +######## Run Configuration ######## +## the following defines, if this module is activated +BridgedescsActivated = false +# period in minutes +BridgedescsPeriodMinutes = 60 +# offset in minutes since the epoch and +BridgedescsOffsetMinutes = 9 +## the following defines, if this module is activated +ExitlistsActivated = false +# period in minutes +ExitlistsPeriodMinutes = 60 +# offset in minutes since the epoch and +ExitlistsOffsetMinutes = 2 +## the following defines, if this module is activated +RelaydescsActivated = false +# period in minutes +RelaydescsPeriodMinutes = 30 +# offset in minutes since the epoch and +RelaydescsOffsetMinutes = 5 +## the following defines, if this module is activated +TorperfActivated = false +# period in minutes +TorperfPeriodMinutes = 360 +# offset in minutes since the epoch and +TorperfOffsetMinutes = 1 +# the following defines, if this module is activated +UpdateindexActivated = false +# period in minutes +UpdateindexPeriodMinutes = 2 +# offset in minutes since the epoch and +UpdateindexOffsetMinutes = 0 ######## General Properties ######## InstanceBaseUrl = "https://collector.torproject.org" LockFilePath = lock @@ -7,7 +38,6 @@ IndexPath = out/index ArchivePath = out/archive RecentPath = out/recent StatsPath = out/stats - ######## Relay descriptors ######## # ## Read cached-* files from a local Tor data directory diff --git a/src/test/java/org/torproject/collector/conf/ConfigurationTest.java b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java index ac9fd4f..437939b 100644 --- a/src/test/java/org/torproject/collector/conf/ConfigurationTest.java +++ b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java @@ -22,7 +22,7 @@ public class ConfigurationTest { public void testKeyCount() throws Exception { assertEquals("The number of properties keys in enum Key changed." + "\n This test class should be adapted.", - 33, Key.values().length); + 48, Key.values().length); } @Test()

1 0

[collector/master] Make some trivial whitespace fixes.
by karsten＠torproject.org 25 Jul '16

25 Jul '16

commit 69535b64e20049e7a2f2f97b616322c857991e96 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Jul 25 17:00:45 2016 +0200 Make some trivial whitespace fixes. --- .../collector/bridgedescs/SanitizedBridgesWriter.java | 2 +- .../java/org/torproject/collector/cron/CollecTorMain.java | 2 +- .../java/org/torproject/collector/cron/Scheduler.java | 15 +++++++-------- .../collector/exitlists/ExitListDownloader.java | 2 +- .../org/torproject/collector/index/CreateIndexJson.java | 2 +- src/test/java/org/torproject/collector/MainTest.java | 12 ++++++------ .../org/torproject/collector/conf/ConfigurationTest.java | 12 ++++++------ 7 files changed, 23 insertions(+), 24 deletions(-) diff --git a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java index e00f70b..476f80b 100644 --- a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java +++ b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java @@ -392,7 +392,7 @@ public class SanitizedBridgesWriter extends CollecTorMain { if (this.bridgeSanitizingCutOffTimestamp .compareTo(publicationTime) > 0) { - String text = "Sanitizing and storing network status with " + String text = "Sanitizing and storing network status with " + "publication time outside our descriptor sanitizing " + "interval."; if (this.haveWarnedAboutInterval) { diff --git a/src/main/java/org/torproject/collector/cron/CollecTorMain.java b/src/main/java/org/torproject/collector/cron/CollecTorMain.java index 21d8948..5fe6376 100644 --- a/src/main/java/org/torproject/collector/cron/CollecTorMain.java +++ b/src/main/java/org/torproject/collector/cron/CollecTorMain.java @@ -23,7 +23,7 @@ public abstract class CollecTorMain implements Runnable { protected Configuration config; - public CollecTorMain( Configuration conf) { + public CollecTorMain(Configuration conf) { this.config = conf; } diff --git a/src/main/java/org/torproject/collector/cron/Scheduler.java b/src/main/java/org/torproject/collector/cron/Scheduler.java index 6bc90ca..5535a8a 100644 --- a/src/main/java/org/torproject/collector/cron/Scheduler.java +++ b/src/main/java/org/torproject/collector/cron/Scheduler.java @@ -53,10 +53,10 @@ public class Scheduler implements ThreadFactory { */ public void scheduleModuleRuns(Map<Key, Class<? extends CollecTorMain>> collecTorMains, Configuration conf) { - for ( Map.Entry<Key, Class<? extends CollecTorMain>> ctmEntry - : collecTorMains.entrySet() ) { + for (Map.Entry<Key, Class<? extends CollecTorMain>> ctmEntry + : collecTorMains.entrySet()) { try { - if ( conf.getBool(ctmEntry.getKey()) ) { + if (conf.getBool(ctmEntry.getKey())) { String prefix = ctmEntry.getKey().name().replace(ACTIVATED, ""); CollecTorMain ctm = ctmEntry.getValue() .getConstructor(Configuration.class).newInstance(conf); @@ -76,9 +76,9 @@ public class Scheduler implements ThreadFactory { private void scheduleExecutions(CollecTorMain ctm, int offset, int period) { this.log.info("Periodic updater started for " + ctm.getClass().getName() - + "; offset=" + offset + ", period=" + period + "."); + + "; offset=" + offset + ", period=" + period + "."); int currentMinute = Calendar.getInstance().get(Calendar.MINUTE); - int initialDelay = (period - (currentMinute % period) + offset) % period; + int initialDelay = (period - (currentMinute % period) + offset) % period; /* Run after initialDelay delay and then every period min. */ this.log.info("Periodic updater will start every " + period + "th min " @@ -96,10 +96,10 @@ public class Scheduler implements ThreadFactory { scheduler.shutdown(); scheduler.awaitTermination(20L, java.util.concurrent.TimeUnit.MINUTES); log.info("Shutdown of all scheduled tasks completed successfully."); - } catch ( InterruptedException ie ) { + } catch (InterruptedException ie) { List<Runnable> notTerminated = scheduler.shutdownNow(); log.error("Regular shutdown failed for: " + notTerminated); - if ( !notTerminated.isEmpty() ) { + if (!notTerminated.isEmpty()) { log.error("Forced shutdown failed for: " + notTerminated); } } @@ -115,6 +115,5 @@ public class Scheduler implements ThreadFactory { log.info("New Thread created: " + newThread.getName()); return newThread; } - } diff --git a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java index 8b5277b..9c02032 100644 --- a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java +++ b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java @@ -77,7 +77,7 @@ public class ExitListDownloader extends CollecTorMain { byte[] data = new byte[1024]; while ((len = in.read(data, 0, 1024)) >= 0) { sb.append(new String(data, 0, len)); - } + } in.close(); downloadedExitList = sb.toString(); logger.debug("Finished downloading exit list."); diff --git a/src/main/java/org/torproject/collector/index/CreateIndexJson.java b/src/main/java/org/torproject/collector/index/CreateIndexJson.java index 08c28c6..39069f1 100644 --- a/src/main/java/org/torproject/collector/index/CreateIndexJson.java +++ b/src/main/java/org/torproject/collector/index/CreateIndexJson.java @@ -68,7 +68,7 @@ public class CreateIndexJson extends CollecTorMain { @Override protected void startProcessing() throws ConfigurationException { try { - indexJsonFile = new File(config.getPath(Key.IndexPath).toFile(), + indexJsonFile = new File(config.getPath(Key.IndexPath).toFile(), "index.json"); basePath = config.getProperty(Key.InstanceBaseUrl.name()); indexedDirectories = new File[] { diff --git a/src/test/java/org/torproject/collector/MainTest.java b/src/test/java/org/torproject/collector/MainTest.java index b48a0a9..6b90978 100644 --- a/src/test/java/org/torproject/collector/MainTest.java +++ b/src/test/java/org/torproject/collector/MainTest.java @@ -52,7 +52,7 @@ public class MainTest { public void testInitializationEmptyArgs() throws Exception { File conf = new File(Main.CONF_FILE); assertFalse("Please remove " + Main.CONF_FILE + " before running tests!", conf.exists()); - Main.main(new String[]{}); + Main.main(new String[] { }); assertTrue(conf.exists()); assertTrue(conf.delete()); } @@ -61,7 +61,7 @@ public class MainTest { public void testInitializationTooManyArgs() throws Exception { File conf = new File(Main.CONF_FILE); assertFalse("Please remove " + Main.CONF_FILE + " before running tests!", conf.exists()); - Main.main(new String[]{"x", "y"}); + Main.main(new String[] { "x", "y" }); assertFalse(conf.exists()); } @@ -74,7 +74,7 @@ public class MainTest { assertTrue(4_000L <= conf.length()); changeFilePathsAndSetActivation(conf, lockPath, "TorperfActivated"); Main.main(new String[]{conf.toString()}); - for(int t=0; t<1_000_000; t++) { } + for(int t = 0; t < 1_000_000; t++) { } } private void changeFilePathsAndSetActivation(File f, File l, String a) throws Exception { @@ -133,7 +133,7 @@ public class MainTest { String[] runConfigSettings = new String[] {Scheduler.ACTIVATED, Scheduler.PERIODMIN, Scheduler.OFFSETMIN}; for (Key key : Main.collecTorMains.keySet()) { - for ( String part : runConfigSettings ){ + for (String part : runConfigSettings) { String key2 = key.name().replace("Activated", part); assertNotNull("Property '" + key2 + "' not specified in " + Main.CONF_FILE + ".", @@ -141,8 +141,8 @@ public class MainTest { } } for (String propName : props.stringPropertyNames()) { - for ( String part : runConfigSettings ){ - if( propName.contains(part) ){ + for (String part : runConfigSettings) { + if (propName.contains(part)) { String key2 = propName.replace(part, ""); assertTrue("CollecTorMain '" + key2 + "' not specified in Main.class.", diff --git a/src/test/java/org/torproject/collector/conf/ConfigurationTest.java b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java index 437939b..8671ef4 100644 --- a/src/test/java/org/torproject/collector/conf/ConfigurationTest.java +++ b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java @@ -100,42 +100,42 @@ public class ConfigurationTest { assertArrayEquals(sourceStrings, conf.getStringArrayArray(Key.TorperfSources)); } - @Test( expected = ConfigurationException.class) + @Test(expected = ConfigurationException.class) public void testArrayArrayValueException() throws Exception { Configuration conf = new Configuration(); conf.load(new ByteArrayInputStream("CachedRelayDescriptorsDirectories".getBytes())); conf.getStringArrayArray(Key.TorperfOutputDirectory); } - @Test( expected = ConfigurationException.class) + @Test(expected = ConfigurationException.class) public void testArrayValueException() throws Exception { Configuration conf = new Configuration(); conf.load(new ByteArrayInputStream("CachedRelayDescriptorsDirectories".getBytes())); conf.getStringArray(Key.TorperfSources); } - @Test( expected = ConfigurationException.class) + @Test(expected = ConfigurationException.class) public void testBoolValueException() throws Exception { Configuration conf = new Configuration(); conf.load(new ByteArrayInputStream("TorperfSource = http://x.y.z".getBytes())); conf.getBool(Key.CachedRelayDescriptorsDirectories); } - @Test( expected = ConfigurationException.class) + @Test(expected = ConfigurationException.class) public void testPathValueException() throws Exception { Configuration conf = new Configuration(); conf.load(new ByteArrayInputStream("DirectoryArchivesDirectory = \\u0000:".getBytes())); conf.getPath(Key.DirectoryArchivesDirectory); } - @Test( expected = ConfigurationException.class) + @Test(expected = ConfigurationException.class) public void testUrlValueException() throws Exception { Configuration conf = new Configuration(); conf.load(new ByteArrayInputStream("ExitlistUrl = xxx://y.y.y".getBytes())); conf.getUrl(Key.ExitlistUrl); } - @Test( expected = ConfigurationException.class) + @Test(expected = ConfigurationException.class) public void testIntValueException() throws Exception { Configuration conf = new Configuration(); conf.load(new ByteArrayInputStream("BridgeDescriptorMappingsLimit = y7".getBytes()));

1 0

[collector/master] Changed another hard-coded directory reference.
by karsten＠torproject.org 25 Jul '16

25 Jul '16

commit 40ae5159959d07bb5b6b83494e687ef24b63bc90 Author: iwakeh <iwakeh(a)torproject.org> Date: Wed Jul 20 16:30:31 2016 +0200 Changed another hard-coded directory reference. --- src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java b/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java index cbab5ea..4ad87f5 100644 --- a/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java +++ b/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java @@ -136,7 +136,7 @@ public class ArchiveWriter extends CollecTorMain { } private void startProcessing() throws ConfigurationException { - File statsDirectory = new File("stats"); + File statsDirectory = config.getPath(Key.StatsPath).toFile(); this.outputDirectory = config.getPath(Key.DirectoryArchivesOutputDirectory).toString(); SimpleDateFormat rsyncCatFormat = new SimpleDateFormat( "yyyy-MM-dd-HH-mm-ss");

1 0

[collector/master] Add missing = in properties template.
by karsten＠torproject.org 25 Jul '16

25 Jul '16

commit 50463327de8cd35c6b2625b0cdb32ae00b3238e8 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Jul 25 17:01:09 2016 +0200 Add missing = in properties template. --- src/main/resources/collector.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/collector.properties b/src/main/resources/collector.properties index 8996a1f..e7d41ae 100644 --- a/src/main/resources/collector.properties +++ b/src/main/resources/collector.properties @@ -94,7 +94,7 @@ DownloadMissingMicrodescriptors = true # ## Download all server descriptors from the directory authorities at most ## once a day (only if DownloadRelayDescriptors is true) -DownloadAllServerDescriptors false +DownloadAllServerDescriptors = false # ## Download all extra-info descriptors from the directory authorities at ## most once a day (only if DownloadRelayDescriptors is true)

1 0

[onionoo/master] Upgrade to metrics-lib 1.3.0.
by karsten＠torproject.org 25 Jul '16

25 Jul '16

commit c0c7ee989580b67a6147487d14aa986526180744 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Jul 20 09:46:33 2016 +0200 Upgrade to metrics-lib 1.3.0. --- build.xml | 2 +- src/test/java/org/torproject/onionoo/updater/DummyConsensus.java | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/build.xml b/build.xml index f7f6bc0..467c991 100644 --- a/build.xml +++ b/build.xml @@ -3,7 +3,7 @@ <property name="onionoo.protocol.version" value="3.1"/> <property name="release.version" value="${onionoo.protocol.version}.0"/> - <property name="descriptorversion" value="1.2.0"/> + <property name="descriptorversion" value="1.3.0"/> <property name="javasources" value="src/main/java"/> <property name="tests" value="src/test/java"/> <property name="generated" value="generated/"/> diff --git a/src/test/java/org/torproject/onionoo/updater/DummyConsensus.java b/src/test/java/org/torproject/onionoo/updater/DummyConsensus.java index 0f01a5e..af03ccf 100644 --- a/src/test/java/org/torproject/onionoo/updater/DummyConsensus.java +++ b/src/test/java/org/torproject/onionoo/updater/DummyConsensus.java @@ -115,5 +115,13 @@ public class DummyConsensus implements RelayNetworkStatusConsensus { public String getConsensusDigest() { return null; } + + public List<String> getPackageLines() { + return null; + } + + public List<DirectorySignature> getSignatures() { + return null; + } }

1 0

[onionoo/master] Resolve or suppress remaining checkstyle warnings.
by karsten＠torproject.org 25 Jul '16

25 Jul '16

commit 01c53dd0a261acfdad467fc5450abf5b12d42063 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Jul 19 16:42:15 2016 +0200 Resolve or suppress remaining checkstyle warnings. Implements the rest of #19613. --- .../torproject/onionoo/docs/BandwidthDocument.java | 1 + .../torproject/onionoo/docs/ClientsDocument.java | 1 + .../torproject/onionoo/docs/DetailsDocument.java | 41 +++---- .../org/torproject/onionoo/docs/DetailsStatus.java | 49 +++++---- .../org/torproject/onionoo/docs/DocumentStore.java | 4 +- .../org/torproject/onionoo/docs/NodeStatus.java | 18 +-- .../torproject/onionoo/docs/SummaryDocument.java | 15 +-- .../torproject/onionoo/docs/UptimeDocument.java | 1 + .../torproject/onionoo/docs/WeightsDocument.java | 1 + .../org/torproject/onionoo/docs/WeightsStatus.java | 6 +- .../org/torproject/onionoo/server/Counter.java | 22 ++++ .../onionoo/server/HttpServletRequestWrapper.java | 1 + .../onionoo/server/IntegerDistribution.java | 50 +++++++++ .../onionoo/server/MostFrequentString.java | 65 +++++++++++ .../org/torproject/onionoo/server/NodeIndex.java | 12 +- .../org/torproject/onionoo/server/NodeIndexer.java | 25 +++-- .../onionoo/server/PerformanceMetrics.java | 122 --------------------- .../torproject/onionoo/server/RequestHandler.java | 18 +-- .../torproject/onionoo/server/ResourceServlet.java | 46 ++++---- .../onionoo/updater/DescriptorDownloader.java | 10 +- .../onionoo/updater/DescriptorSource.java | 6 + .../torproject/onionoo/updater/LookupService.java | 27 ++--- .../onionoo/updater/NodeDetailsStatusUpdater.java | 8 +- .../onionoo/writer/BandwidthDocumentWriter.java | 4 +- .../onionoo/writer/DetailsDocumentWriter.java | 4 +- .../onionoo/writer/SummaryDocumentWriter.java | 8 +- 26 files changed, 301 insertions(+), 264 deletions(-) diff --git a/src/main/java/org/torproject/onionoo/docs/BandwidthDocument.java b/src/main/java/org/torproject/onionoo/docs/BandwidthDocument.java index 6fe3240..1990734 100644 --- a/src/main/java/org/torproject/onionoo/docs/BandwidthDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/BandwidthDocument.java @@ -5,6 +5,7 @@ package org.torproject.onionoo.docs; import java.util.Map; +@SuppressWarnings("checkstyle:membername") public class BandwidthDocument extends Document { @SuppressWarnings("unused") diff --git a/src/main/java/org/torproject/onionoo/docs/ClientsDocument.java b/src/main/java/org/torproject/onionoo/docs/ClientsDocument.java index 89729d2..f8b0c84 100644 --- a/src/main/java/org/torproject/onionoo/docs/ClientsDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/ClientsDocument.java @@ -5,6 +5,7 @@ package org.torproject.onionoo.docs; import java.util.Map; +@SuppressWarnings("checkstyle:membername") public class ClientsDocument extends Document { @SuppressWarnings("unused") diff --git a/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java b/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java index 85c0154..a9257af 100644 --- a/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java @@ -10,6 +10,7 @@ import java.util.List; import java.util.Map; import java.util.SortedSet; +@SuppressWarnings("checkstyle:membername") public class DetailsDocument extends Document { /* We must ensure that details files only contain ASCII characters @@ -20,12 +21,12 @@ public class DetailsDocument extends Document { * to write six characters '\', 'u', '0', '0', 'F', '2'. The only thing * we'll have to do is to change back the '\\' that Gson writes for the * '\'. */ - private static String escapeJSON(String s) { - return StringEscapeUtils.escapeJava(s); + private static String escapeJson(String stringToEscape) { + return StringEscapeUtils.escapeJava(stringToEscape); } - private static String unescapeJSON(String s) { - return StringEscapeUtils.unescapeJava(s); + private static String unescapeJson(String stringToUnescape) { + return StringEscapeUtils.unescapeJava(stringToUnescape); } private String nickname; @@ -154,31 +155,31 @@ public class DetailsDocument extends Document { private String country_name; public void setCountryName(String countryName) { - this.country_name = escapeJSON(countryName); + this.country_name = escapeJson(countryName); } public String getCountryName() { - return unescapeJSON(this.country_name); + return unescapeJson(this.country_name); } private String region_name; public void setRegionName(String regionName) { - this.region_name = escapeJSON(regionName); + this.region_name = escapeJson(regionName); } public String getRegionName() { - return unescapeJSON(this.region_name); + return unescapeJson(this.region_name); } private String city_name; public void setCityName(String cityName) { - this.city_name = escapeJSON(cityName); + this.city_name = escapeJson(cityName); } public String getCityName() { - return unescapeJSON(this.city_name); + return unescapeJson(this.city_name); } private Float latitude; @@ -204,21 +205,21 @@ public class DetailsDocument extends Document { private String as_number; public void setAsNumber(String asNumber) { - this.as_number = escapeJSON(asNumber); + this.as_number = escapeJson(asNumber); } public String getAsNumber() { - return unescapeJSON(this.as_number); + return unescapeJson(this.as_number); } private String as_name; public void setAsName(String asName) { - this.as_name = escapeJSON(asName); + this.as_name = escapeJson(asName); } public String getAsName() { - return unescapeJSON(this.as_name); + return unescapeJson(this.as_name); } private Long consensus_weight; @@ -234,11 +235,11 @@ public class DetailsDocument extends Document { private String host_name; public void setHostName(String hostName) { - this.host_name = escapeJSON(hostName); + this.host_name = escapeJson(hostName); } public String getHostName() { - return unescapeJSON(this.host_name); + return unescapeJson(this.host_name); } private String last_restarted; @@ -328,21 +329,21 @@ public class DetailsDocument extends Document { private String contact; public void setContact(String contact) { - this.contact = escapeJSON(contact); + this.contact = escapeJson(contact); } public String getContact() { - return unescapeJSON(this.contact); + return unescapeJson(this.contact); } private String platform; public void setPlatform(String platform) { - this.platform = escapeJSON(platform); + this.platform = escapeJson(platform); } public String getPlatform() { - return unescapeJSON(this.platform); + return unescapeJson(this.platform); } private SortedSet<String> alleged_family; diff --git a/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java b/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java index 1a3c05d..7258054 100644 --- a/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java @@ -10,6 +10,7 @@ import java.util.Map; import java.util.SortedSet; import java.util.TreeSet; +@SuppressWarnings("checkstyle:membername") public class DetailsStatus extends Document { /* We must ensure that details files only contain ASCII characters @@ -20,12 +21,12 @@ public class DetailsStatus extends Document { * to write six characters '\', 'u', '0', '0', 'F', '2'. The only thing * we'll have to do is to change back the '\\' that Gson writes for the * '\'. */ - private static String escapeJSON(String s) { - return StringEscapeUtils.escapeJava(s); + private static String escapeJson(String stringToEscape) { + return StringEscapeUtils.escapeJava(stringToEscape); } - private static String unescapeJSON(String s) { - return StringEscapeUtils.unescapeJava(s); + private static String unescapeJson(String stringToUnescape) { + return StringEscapeUtils.unescapeJava(stringToUnescape); } /* From most recently published server descriptor: */ @@ -105,21 +106,21 @@ public class DetailsStatus extends Document { private String contact; public void setContact(String contact) { - this.contact = escapeJSON(contact); + this.contact = escapeJson(contact); } public String getContact() { - return unescapeJSON(this.contact); + return unescapeJson(this.contact); } private String platform; public void setPlatform(String platform) { - this.platform = escapeJSON(platform); + this.platform = escapeJson(platform); } public String getPlatform() { - return unescapeJSON(this.platform); + return unescapeJson(this.platform); } private SortedSet<String> alleged_family; @@ -470,51 +471,51 @@ public class DetailsStatus extends Document { private String country_name; public void setCountryName(String countryName) { - this.country_name = escapeJSON(countryName); + this.country_name = escapeJson(countryName); } public String getCountryName() { - return unescapeJSON(this.country_name); + return unescapeJson(this.country_name); } private String region_name; public void setRegionName(String regionName) { - this.region_name = escapeJSON(regionName); + this.region_name = escapeJson(regionName); } public String getRegionName() { - return unescapeJSON(this.region_name); + return unescapeJson(this.region_name); } private String city_name; public void setCityName(String cityName) { - this.city_name = escapeJSON(cityName); + this.city_name = escapeJson(cityName); } public String getCityName() { - return unescapeJSON(this.city_name); + return unescapeJson(this.city_name); } private String as_name; - public void setASName(String aSName) { - this.as_name = escapeJSON(aSName); + public void setAsName(String asName) { + this.as_name = escapeJson(asName); } - public String getASName() { - return unescapeJSON(this.as_name); + public String getAsName() { + return unescapeJson(this.as_name); } private String as_number; - public void setASNumber(String aSNumber) { - this.as_number = escapeJSON(aSNumber); + public void setAsNumber(String asNumber) { + this.as_number = escapeJson(asNumber); } - public String getASNumber() { - return unescapeJSON(this.as_number); + public String getAsNumber() { + return unescapeJson(this.as_number); } /* Reverse DNS lookup result: */ @@ -522,11 +523,11 @@ public class DetailsStatus extends Document { private String host_name; public void setHostName(String hostName) { - this.host_name = escapeJSON(hostName); + this.host_name = escapeJson(hostName); } public String getHostName() { - return unescapeJSON(this.host_name); + return unescapeJson(this.host_name); } } diff --git a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java index 42c75aa..e1be777 100644 --- a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java +++ b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java @@ -420,7 +420,7 @@ public class DocumentStore { String nickname = detailsDocument.getNickname(); List<String> addresses = new ArrayList<String>(); String countryCode = null; - String aSNumber = null; + String asNumber = null; String contact = null; for (String orAddressAndPort : detailsDocument.getOrAddresses()) { if (!orAddressAndPort.contains(":")) { @@ -451,7 +451,7 @@ public class DocumentStore { SummaryDocument summaryDocument = new SummaryDocument(isRelay, nickname, fingerprint, addresses, lastSeenMillis, running, relayFlags, consensusWeight, countryCode, firstSeenMillis, - aSNumber, contact, family, family); + asNumber, contact, family, family); return summaryDocument; } diff --git a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java index afdd6c6..87cacb4 100644 --- a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java @@ -67,9 +67,9 @@ public class NodeStatus extends Document { String[] stringArray = null; if (collection != null && !collection.isEmpty()) { stringArray = new String[collection.size()]; - int i = 0; + int index = 0; for (String string : collection) { - stringArray[i++] = string; + stringArray[index++] = string; } } return stringArray; @@ -337,14 +337,14 @@ public class NodeStatus extends Document { return this.countryCode; } - private String aSNumber; + private String asNumber; - public void setASNumber(String aSNumber) { - this.aSNumber = aSNumber; + public void setAsNumber(String asNumber) { + this.asNumber = asNumber; } - public String getASNumber() { - return this.aSNumber; + public String getAsNumber() { + return this.asNumber; } /* Reverse DNS lookup result */ @@ -501,7 +501,7 @@ public class NodeStatus extends Document { } nodeStatus.addLastAddresses(lastChangedAddresses, address, orPort, dirPort, orAddressesAndPorts); - nodeStatus.setASNumber(parts[19]); + nodeStatus.setAsNumber(parts[19]); nodeStatus.setContact(parts[20]); if (!parts[21].equals("null")) { nodeStatus.setRecommendedVersion(parts[21].equals("true")); @@ -588,7 +588,7 @@ public class NodeStatus extends Document { sb.append("\t" + DateTimeHelper.format( this.getLastChangedOrAddressOrPort(), DateTimeHelper.ISO_DATETIME_TAB_FORMAT)); - sb.append("\t" + (this.aSNumber != null ? this.aSNumber : "null")); + sb.append("\t" + (this.asNumber != null ? this.asNumber : "null")); } else { sb.append("\tnull\tnull\tnull"); } diff --git a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java index c7d4774..133f616 100644 --- a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java @@ -16,6 +16,7 @@ import java.util.SortedSet; import java.util.TreeSet; import java.util.regex.Pattern; +@SuppressWarnings("checkstyle:membername") public class SummaryDocument extends Document { private boolean t; @@ -133,9 +134,9 @@ public class SummaryDocument extends Document { String[] stringArray = null; if (collection != null && !collection.isEmpty()) { stringArray = new String[collection.size()]; - int i = 0; + int index = 0; for (String string : collection) { - stringArray[i++] = string; + stringArray[index++] = string; } } return stringArray; @@ -171,11 +172,11 @@ public class SummaryDocument extends Document { private String as; - public void setASNumber(String aSNumber) { - this.as = aSNumber; + public void setAsNumber(String asNumber) { + this.as = asNumber; } - public String getASNumber() { + public String getAsNumber() { return this.as; } @@ -274,7 +275,7 @@ public class SummaryDocument extends Document { public SummaryDocument(boolean isRelay, String nickname, String fingerprint, List<String> addresses, long lastSeenMillis, boolean running, SortedSet<String> relayFlags, long consensusWeight, - String countryCode, long firstSeenMillis, String aSNumber, + String countryCode, long firstSeenMillis, String asNumber, String contact, SortedSet<String> familyFingerprints, SortedSet<String> effectiveFamily) { this.setRelay(isRelay); @@ -287,7 +288,7 @@ public class SummaryDocument extends Document { this.setConsensusWeight(consensusWeight); this.setCountryCode(countryCode); this.setFirstSeenMillis(firstSeenMillis); - this.setASNumber(aSNumber); + this.setAsNumber(asNumber); this.setContact(contact); this.setFamilyFingerprints(familyFingerprints); this.setEffectiveFamily(effectiveFamily); diff --git a/src/main/java/org/torproject/onionoo/docs/UptimeDocument.java b/src/main/java/org/torproject/onionoo/docs/UptimeDocument.java index 505d66c..98884b6 100644 --- a/src/main/java/org/torproject/onionoo/docs/UptimeDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/UptimeDocument.java @@ -6,6 +6,7 @@ package org.torproject.onionoo.docs; import java.util.Map; import java.util.SortedMap; +@SuppressWarnings("checkstyle:membername") public class UptimeDocument extends Document { @SuppressWarnings("unused") diff --git a/src/main/java/org/torproject/onionoo/docs/WeightsDocument.java b/src/main/java/org/torproject/onionoo/docs/WeightsDocument.java index 6e2ee02..9b1c8ef 100644 --- a/src/main/java/org/torproject/onionoo/docs/WeightsDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/WeightsDocument.java @@ -5,6 +5,7 @@ package org.torproject.onionoo.docs; import java.util.Map; +@SuppressWarnings("checkstyle:membername") public class WeightsDocument extends Document { @SuppressWarnings("unused") diff --git a/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java b/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java index efb7c25..acd44a1 100644 --- a/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java @@ -32,8 +32,10 @@ public class WeightsStatus extends Document { private SortedMap<long[], double[]> history = new TreeMap<long[], double[]>( new Comparator<long[]>() { - public int compare(long[] a, long[] b) { - return a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0; + public int compare(long[] first, long[] second) { + return first[0] < second[0] ? -1 + : first[0] > second[0] ? 1 + : 0; } } ); diff --git a/src/main/java/org/torproject/onionoo/server/Counter.java b/src/main/java/org/torproject/onionoo/server/Counter.java new file mode 100644 index 0000000..655a56c --- /dev/null +++ b/src/main/java/org/torproject/onionoo/server/Counter.java @@ -0,0 +1,22 @@ +/* Copyright 2014--2016 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.onionoo.server; + +class Counter { + + int value = 0; + + void increment() { + this.value++; + } + + @Override + public String toString() { + return String.valueOf(this.value); + } + + void clear() { + this.value = 0; + } +} diff --git a/src/main/java/org/torproject/onionoo/server/HttpServletRequestWrapper.java b/src/main/java/org/torproject/onionoo/server/HttpServletRequestWrapper.java index 1aa964a..e819a1a 100644 --- a/src/main/java/org/torproject/onionoo/server/HttpServletRequestWrapper.java +++ b/src/main/java/org/torproject/onionoo/server/HttpServletRequestWrapper.java @@ -15,6 +15,7 @@ public class HttpServletRequestWrapper { this.request = request; } + @SuppressWarnings("abbreviationaswordinname") protected String getRequestURI() { return this.request.getRequestURI(); } diff --git a/src/main/java/org/torproject/onionoo/server/IntegerDistribution.java b/src/main/java/org/torproject/onionoo/server/IntegerDistribution.java new file mode 100644 index 0000000..c9e3a29 --- /dev/null +++ b/src/main/java/org/torproject/onionoo/server/IntegerDistribution.java @@ -0,0 +1,50 @@ +/* Copyright 2014--2016 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.onionoo.server; + +import java.util.Arrays; + +class IntegerDistribution { + + int[] logValues = new int[64]; + + void addLong(long value) { + logValues[64 - Long.numberOfLeadingZeros(value)]++; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + int totalValues = 0; + for (int i = 0; i < logValues.length; i++) { + totalValues += logValues[i]; + } + int[] permilles = new int[] { 500, 900, 990, 999 }; + if (totalValues > 0) { + int seenValues = 0; + for (int i = 0, j = 0; i < logValues.length; i++) { + seenValues += logValues[i]; + while (j < permilles.length + && (seenValues * 1000 > totalValues * permilles[j])) { + sb.append((j > 0 ? ", " : "") + "." + permilles[j] + + (i < logValues.length - 1 ? "<" + (1L << i) + : ">=" + (1L << i - 1))); + j++; + } + if (j == permilles.length) { + break; + } + } + } else { + for (int j = 0; j < permilles.length; j++) { + sb.append((j > 0 ? ", " : "") + "." + permilles[j] + "<null"); + } + } + return sb.toString(); + } + + void clear() { + Arrays.fill(logValues, 0, logValues.length - 1, 0); + } +} diff --git a/src/main/java/org/torproject/onionoo/server/MostFrequentString.java b/src/main/java/org/torproject/onionoo/server/MostFrequentString.java new file mode 100644 index 0000000..12b5e04 --- /dev/null +++ b/src/main/java/org/torproject/onionoo/server/MostFrequentString.java @@ -0,0 +1,65 @@ +/* Copyright 2014--2016 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.onionoo.server; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; + +class MostFrequentString { + + Map<String, Integer> stringFrequencies = new HashMap<String, Integer>(); + + void addString(String string) { + if (!this.stringFrequencies.containsKey(string)) { + this.stringFrequencies.put(string, 1); + } else { + this.stringFrequencies.put(string, + this.stringFrequencies.get(string) + 1); + } + } + + @Override + public String toString() { + SortedMap<Integer, SortedSet<String>> sortedFrequencies = + new TreeMap<Integer, SortedSet<String>>( + Collections.reverseOrder()); + if (this.stringFrequencies.isEmpty()) { + return "null (0)"; + } + for (Map.Entry<String, Integer> e : stringFrequencies.entrySet()) { + if (!sortedFrequencies.containsKey(e.getValue())) { + sortedFrequencies.put(e.getValue(), new TreeSet<String>( + Arrays.asList(new String[] { e.getKey() } ))); + } else { + sortedFrequencies.get(e.getValue()).add(e.getKey()); + } + } + StringBuilder sb = new StringBuilder(); + int stringsToAdd = 3; + int written = 0; + for (Map.Entry<Integer, SortedSet<String>> e : + sortedFrequencies.entrySet()) { + for (String string : e.getValue()) { + if (stringsToAdd-- > 0) { + sb.append((written++ > 0 ? ", " : "") + string + " (" + + e.getKey() + ")"); + } + } + if (stringsToAdd == 0) { + break; + } + } + return sb.toString(); + } + + void clear() { + this.stringFrequencies.clear(); + } +} diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndex.java b/src/main/java/org/torproject/onionoo/server/NodeIndex.java index c000bba..7871690 100644 --- a/src/main/java/org/torproject/onionoo/server/NodeIndex.java +++ b/src/main/java/org/torproject/onionoo/server/NodeIndex.java @@ -86,15 +86,15 @@ class NodeIndex { return relaysByCountryCode; } - private Map<String, Set<String>> relaysByASNumber = null; + private Map<String, Set<String>> relaysByAsNumber = null; - public void setRelaysByASNumber( - Map<String, Set<String>> relaysByASNumber) { - this.relaysByASNumber = relaysByASNumber; + public void setRelaysByAsNumber( + Map<String, Set<String>> relaysByAsNumber) { + this.relaysByAsNumber = relaysByAsNumber; } - public Map<String, Set<String>> getRelaysByASNumber() { - return relaysByASNumber; + public Map<String, Set<String>> getRelaysByAsNumber() { + return relaysByAsNumber; } private Map<String, Set<String>> relaysByFlag = null; diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java index 93b5af7..99d1aee 100644 --- a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java +++ b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java @@ -71,6 +71,8 @@ public class NodeIndexer implements ServletContextListener, Runnable { try { this.wait(timeoutMillis); } catch (InterruptedException e) { + /* Nothing that we could handle, just return what we have + * below. */ } } return this.lastIndexed; @@ -84,6 +86,8 @@ public class NodeIndexer implements ServletContextListener, Runnable { try { this.wait(timeoutMillis); } catch (InterruptedException e) { + /* Nothing that we could handle, just return what we have + * below. */ } } return this.latestNodeIndex; @@ -110,6 +114,8 @@ public class NodeIndexer implements ServletContextListener, Runnable { try { Thread.sleep(ONE_MINUTE); } catch (InterruptedException e) { + /* Nothing that we could handle, just check if there's new data + * to index now. */ } } } @@ -137,14 +143,13 @@ public class NodeIndexer implements ServletContextListener, Runnable { } } documentStore.invalidateDocumentCache(); - List<String> newRelaysByConsensusWeight = new ArrayList<String>(); Map<String, SummaryDocument> newRelayFingerprintSummaryLines = new HashMap<String, SummaryDocument>(); Map<String, SummaryDocument> newBridgeFingerprintSummaryLines = new HashMap<String, SummaryDocument>(); Map<String, Set<String>> newRelaysByCountryCode = new HashMap<String, Set<String>>(); - Map<String, Set<String>> newRelaysByASNumber = + Map<String, Set<String>> newRelaysByAsNumber = new HashMap<String, Set<String>>(); Map<String, Set<String>> newRelaysByFlag = new HashMap<String, Set<String>>(); @@ -208,13 +213,13 @@ public class NodeIndexer implements ServletContextListener, Runnable { newRelaysByCountryCode.get(countryCode).add(fingerprint); newRelaysByCountryCode.get(countryCode).add(hashedFingerprint); } - if (entry.getASNumber() != null) { - String aSNumber = entry.getASNumber(); - if (!newRelaysByASNumber.containsKey(aSNumber)) { - newRelaysByASNumber.put(aSNumber, new HashSet<String>()); + if (entry.getAsNumber() != null) { + String asNumber = entry.getAsNumber(); + if (!newRelaysByAsNumber.containsKey(asNumber)) { + newRelaysByAsNumber.put(asNumber, new HashSet<String>()); } - newRelaysByASNumber.get(aSNumber).add(fingerprint); - newRelaysByASNumber.get(aSNumber).add(hashedFingerprint); + newRelaysByAsNumber.get(asNumber).add(fingerprint); + newRelaysByAsNumber.get(asNumber).add(hashedFingerprint); } for (String flag : entry.getRelayFlags()) { String flagLowerCase = flag.toLowerCase(); @@ -261,7 +266,7 @@ public class NodeIndexer implements ServletContextListener, Runnable { newRelaysByContact.get(contact).add(hashedFingerprint); } Collections.sort(orderRelaysByConsensusWeight); - newRelaysByConsensusWeight = new ArrayList<String>(); + List<String> newRelaysByConsensusWeight = new ArrayList<String>(); for (String relay : orderRelaysByConsensusWeight) { newRelaysByConsensusWeight.add(relay.split(" ")[1]); } @@ -325,7 +330,7 @@ public class NodeIndexer implements ServletContextListener, Runnable { newNodeIndex.setBridgeFingerprintSummaryLines( newBridgeFingerprintSummaryLines); newNodeIndex.setRelaysByCountryCode(newRelaysByCountryCode); - newNodeIndex.setRelaysByASNumber(newRelaysByASNumber); + newNodeIndex.setRelaysByAsNumber(newRelaysByAsNumber); newNodeIndex.setRelaysByFlag(newRelaysByFlag); newNodeIndex.setBridgesByFlag(newBridgesByFlag); newNodeIndex.setRelaysByContact(newRelaysByContact); diff --git a/src/main/java/org/torproject/onionoo/server/PerformanceMetrics.java b/src/main/java/org/torproject/onionoo/server/PerformanceMetrics.java index 7adad76..7a621cf 100644 --- a/src/main/java/org/torproject/onionoo/server/PerformanceMetrics.java +++ b/src/main/java/org/torproject/onionoo/server/PerformanceMetrics.java @@ -11,130 +11,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.text.SimpleDateFormat; -import java.util.Arrays; import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.SortedMap; -import java.util.SortedSet; import java.util.TimeZone; -import java.util.TreeMap; -import java.util.TreeSet; - -class Counter { - - int value = 0; - - void increment() { - this.value++; - } - - @Override - public String toString() { - return String.valueOf(this.value); - } - - void clear() { - this.value = 0; - } -} - -class MostFrequentString { - - Map<String, Integer> stringFrequencies = new HashMap<String, Integer>(); - - void addString(String string) { - if (!this.stringFrequencies.containsKey(string)) { - this.stringFrequencies.put(string, 1); - } else { - this.stringFrequencies.put(string, - this.stringFrequencies.get(string) + 1); - } - } - - @Override - public String toString() { - SortedMap<Integer, SortedSet<String>> sortedFrequencies = - new TreeMap<Integer, SortedSet<String>>( - Collections.reverseOrder()); - if (this.stringFrequencies.isEmpty()) { - return "null (0)"; - } - for (Map.Entry<String, Integer> e : stringFrequencies.entrySet()) { - if (!sortedFrequencies.containsKey(e.getValue())) { - sortedFrequencies.put(e.getValue(), new TreeSet<String>( - Arrays.asList(new String[] { e.getKey() } ))); - } else { - sortedFrequencies.get(e.getValue()).add(e.getKey()); - } - } - StringBuilder sb = new StringBuilder(); - int stringsToAdd = 3; - int written = 0; - for (Map.Entry<Integer, SortedSet<String>> e : - sortedFrequencies.entrySet()) { - for (String string : e.getValue()) { - if (stringsToAdd-- > 0) { - sb.append((written++ > 0 ? ", " : "") + string + " (" - + e.getKey() + ")"); - } - } - if (stringsToAdd == 0) { - break; - } - } - return sb.toString(); - } - - void clear() { - this.stringFrequencies.clear(); - } -} - -class IntegerDistribution { - - int[] logValues = new int[64]; - - void addLong(long value) { - logValues[64 - Long.numberOfLeadingZeros(value)]++; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - int totalValues = 0; - for (int i = 0; i < logValues.length; i++) { - totalValues += logValues[i]; - } - int[] permilles = new int[] { 500, 900, 990, 999 }; - if (totalValues > 0) { - int seenValues = 0; - for (int i = 0, j = 0; i < logValues.length; i++) { - seenValues += logValues[i]; - while (j < permilles.length - && (seenValues * 1000 > totalValues * permilles[j])) { - sb.append((j > 0 ? ", " : "") + "." + permilles[j] - + (i < logValues.length - 1 ? "<" + (1L << i) - : ">=" + (1L << i - 1))); - j++; - } - if (j == permilles.length) { - break; - } - } - } else { - for (int j = 0; j < permilles.length; j++) { - sb.append((j > 0 ? ", " : "") + "." + permilles[j] + "<null"); - } - } - return sb.toString(); - } - - void clear() { - Arrays.fill(logValues, 0, logValues.length - 1, 0); - } -} public class PerformanceMetrics { diff --git a/src/main/java/org/torproject/onionoo/server/RequestHandler.java b/src/main/java/org/torproject/onionoo/server/RequestHandler.java index eaa4fe2..49b3b94 100644 --- a/src/main/java/org/torproject/onionoo/server/RequestHandler.java +++ b/src/main/java/org/torproject/onionoo/server/RequestHandler.java @@ -153,7 +153,7 @@ public class RequestHandler { this.filterByLookup(); this.filterByFingerprint(); this.filterByCountryCode(); - this.filterByASNumber(); + this.filterByAsNumber(); this.filterByFlag(); this.filterNodesByFirstSeenDays(); this.filterNodesByLastSeenDays(); @@ -369,23 +369,23 @@ public class RequestHandler { this.filteredBridges.clear(); } - private void filterByASNumber() { + private void filterByAsNumber() { if (this.as == null) { /* Not filtering by AS number. */ return; } - String aSNumber = this.as.toUpperCase(); - if (!aSNumber.startsWith("AS")) { - aSNumber = "AS" + aSNumber; + String asNumber = this.as.toUpperCase(); + if (!asNumber.startsWith("AS")) { + asNumber = "AS" + asNumber; } - if (!this.nodeIndex.getRelaysByASNumber().containsKey(aSNumber)) { + if (!this.nodeIndex.getRelaysByAsNumber().containsKey(asNumber)) { this.filteredRelays.clear(); } else { - Set<String> relaysWithASNumber = - this.nodeIndex.getRelaysByASNumber().get(aSNumber); + Set<String> relaysWithAsNumber = + this.nodeIndex.getRelaysByAsNumber().get(asNumber); Set<String> removeRelays = new HashSet<String>(); for (String fingerprint : this.filteredRelays.keySet()) { - if (!relaysWithASNumber.contains(fingerprint)) { + if (!relaysWithAsNumber.contains(fingerprint)) { removeRelays.add(fingerprint); } } diff --git a/src/main/java/org/torproject/onionoo/server/ResourceServlet.java b/src/main/java/org/torproject/onionoo/server/ResourceServlet.java index 5b3ab69..92d0d81 100644 --- a/src/main/java/org/torproject/onionoo/server/ResourceServlet.java +++ b/src/main/java/org/torproject/onionoo/server/ResourceServlet.java @@ -78,6 +78,7 @@ public class ResourceServlet extends HttpServlet { /** Handles the HTTP GET request in the wrapped <code>request</code> by * writing an HTTP GET response to the likewise <code>response</code>, * both of which are wrapped to facilitate testing. */ + @SuppressWarnings("checkstyle:variabledeclarationusagedistance") public void doGet(HttpServletRequestWrapper request, HttpServletResponseWrapper response) throws IOException { @@ -86,15 +87,6 @@ public class ResourceServlet extends HttpServlet { return; } - long nowMillis = TimeFactory.getTime().currentTimeMillis(); - long indexWrittenMillis = - NodeIndexerFactory.getNodeIndexer().getLastIndexed( - INDEX_WAITING_TIME); - long indexAgeMillis = nowMillis - indexWrittenMillis; - long cacheMaxAgeMillis = Math.max(CACHE_MIN_TIME, - ((CACHE_MAX_TIME - indexAgeMillis) - / CACHE_INTERVAL) * CACHE_INTERVAL); - NodeIndex nodeIndex = NodeIndexerFactory.getNodeIndexer() .getLatestNodeIndex(INDEX_WAITING_TIME); if (nodeIndex == null) { @@ -230,13 +222,13 @@ public class ResourceServlet extends HttpServlet { rh.setCountry(countryCodeParameter); } if (parameterMap.containsKey("as")) { - String aSNumberParameter = this.parseASNumberParameter( + String asNumberParameter = this.parseAsNumberParameter( parameterMap.get("as")); - if (aSNumberParameter == null) { + if (asNumberParameter == null) { response.sendError(HttpServletResponse.SC_BAD_REQUEST); return; } - rh.setAs(aSNumberParameter); + rh.setAs(asNumberParameter); } if (parameterMap.containsKey("flag")) { String flagParameter = this.parseFlagParameter( @@ -343,6 +335,14 @@ public class ResourceServlet extends HttpServlet { rb.setFields(fields); } + long indexWrittenMillis = + NodeIndexerFactory.getNodeIndexer().getLastIndexed( + INDEX_WAITING_TIME); + long indexAgeMillis = receivedRequestMillis - indexWrittenMillis; + long cacheMaxAgeMillis = Math.max(CACHE_MIN_TIME, + ((CACHE_MAX_TIME - indexAgeMillis) + / CACHE_INTERVAL) * CACHE_INTERVAL); + response.setHeader("Access-Control-Allow-Origin", "*"); response.setContentType("application/json"); response.setCharacterEncoding("utf-8"); @@ -418,11 +418,11 @@ public class ResourceServlet extends HttpServlet { return parameter; } - private static Pattern aSNumberParameterPattern = + private static Pattern asNumberParameterPattern = Pattern.compile("^[asAS]{0,2}[0-9]{1,10}$"); - private String parseASNumberParameter(String parameter) { - if (!aSNumberParameterPattern.matcher(parameter).matches()) { + private String parseAsNumberParameter(String parameter) { + if (!asNumberParameterPattern.matcher(parameter).matches()) { /* AS number contains illegal character(s). */ return null; } @@ -447,30 +447,30 @@ public class ResourceServlet extends HttpServlet { /* Days contain illegal character(s). */ return null; } - int x = 0; - int y = Integer.MAX_VALUE; + int fromDays = 0; + int toDays = Integer.MAX_VALUE; try { if (!parameter.contains("-")) { - x = Integer.parseInt(parameter); - y = x; + fromDays = Integer.parseInt(parameter); + toDays = fromDays; } else { String[] parts = parameter.split("-", 2); if (parts[0].length() > 0) { - x = Integer.parseInt(parts[0]); + fromDays = Integer.parseInt(parts[0]); } if (parts.length > 1 && parts[1].length() > 0) { - y = Integer.parseInt(parts[1]); + toDays = Integer.parseInt(parts[1]); } } } catch (NumberFormatException e) { /* Invalid format. */ return null; } - if (x > y) { + if (fromDays > toDays) { /* Second number or days must exceed first number. */ return null; } - return new int[] { x, y }; + return new int[] { fromDays, toDays }; } private String[] parseContactParameter(String parameter) { diff --git a/src/main/java/org/torproject/onionoo/updater/DescriptorDownloader.java b/src/main/java/org/torproject/onionoo/updater/DescriptorDownloader.java index 71a43d2..5822b77 100644 --- a/src/main/java/org/torproject/onionoo/updater/DescriptorDownloader.java +++ b/src/main/java/org/torproject/onionoo/updater/DescriptorDownloader.java @@ -79,8 +79,8 @@ class DescriptorDownloader { String directoryUrl = this.protocolHostNameResourcePrefix + this.directory; try { - URL u = new URL(directoryUrl); - HttpURLConnection huc = (HttpURLConnection) u.openConnection(); + URL url = new URL(directoryUrl); + HttpURLConnection huc = (HttpURLConnection) url.openConnection(); huc.setRequestMethod("GET"); huc.connect(); if (huc.getResponseCode() != 200) { @@ -129,8 +129,8 @@ class DescriptorDownloader { File localFile = new File(this.inDir, this.directory + remoteFile); try { localFile.getParentFile().mkdirs(); - URL u = new URL(fileUrl); - HttpURLConnection huc = (HttpURLConnection) u.openConnection(); + URL url = new URL(fileUrl); + HttpURLConnection huc = (HttpURLConnection) url.openConnection(); huc.setRequestMethod("GET"); huc.addRequestProperty("Accept-Encoding", "gzip"); huc.connect(); @@ -140,7 +140,6 @@ class DescriptorDownloader { + huc.getResponseMessage() + ". Skipping."); continue; } - long lastModified = huc.getHeaderFieldDate("Last-Modified", -1L); InputStream is; if (huc.getContentEncoding() != null && huc.getContentEncoding().equalsIgnoreCase("gzip")) { @@ -158,6 +157,7 @@ class DescriptorDownloader { } } localTempFile.renameTo(localFile); + long lastModified = huc.getHeaderFieldDate("Last-Modified", -1L); if (lastModified >= 0) { localFile.setLastModified(lastModified); } diff --git a/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java b/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java index 176a17c..c251142 100644 --- a/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java +++ b/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java @@ -156,6 +156,12 @@ public class DescriptorSource { case BRIDGE_EXTRA_INFOS: log.info("Read recent bridge extra-info descriptors"); break; + default: + /* We shouldn't run into this default case, but if we do, it's + * because we added a new type to DescriptorType but forgot to + * update this switch statement. It's just logging, so not the + * end of the world. */ + log.info("Read recent descriptors of type " + descriptorType); } } diff --git a/src/main/java/org/torproject/onionoo/updater/LookupService.java b/src/main/java/org/torproject/onionoo/updater/LookupService.java index 2b0993f..ec63751 100644 --- a/src/main/java/org/torproject/onionoo/updater/LookupService.java +++ b/src/main/java/org/torproject/onionoo/updater/LookupService.java @@ -39,7 +39,7 @@ public class LookupService { private File geoLite2CityLocationsEnCsvFile; - private File geoIPASNum2CsvFile; + private File geoIpAsNum2CsvFile; private boolean hasAllFiles = false; @@ -63,8 +63,8 @@ public class LookupService { + "geoip/."); return; } - this.geoIPASNum2CsvFile = new File(this.geoipDir, "GeoIPASNum2.csv"); - if (!this.geoIPASNum2CsvFile.exists()) { + this.geoIpAsNum2CsvFile = new File(this.geoipDir, "GeoIPASNum2.csv"); + if (!this.geoIpAsNum2CsvFile.exists()) { log.error("No GeoIPASNum2.csv file in geoip/."); return; } @@ -85,6 +85,7 @@ public class LookupService { try { octetValue = Integer.parseInt(parts[i]); } catch (NumberFormatException e) { + /* Handled below, because octetValue will still be -1. */ } if (octetValue < 0 || octetValue > 255) { addressNumber = -1L; @@ -228,9 +229,9 @@ public class LookupService { } /* Obtain a map from IP address numbers to ASN. */ - Map<Long, String> addressNumberASN = new HashMap<Long, String>(); + Map<Long, String> addressNumberAsn = new HashMap<Long, String>(); try (BufferedReader br = this.createBufferedReaderFromIso88591File( - this.geoIPASNum2CsvFile)) { + this.geoIpAsNum2CsvFile)) { SortedSet<Long> sortedAddressNumbers = new TreeSet<Long>( addressStringNumbers.values()); long firstAddressNumber = sortedAddressNumbers.first(); @@ -240,14 +241,14 @@ public class LookupService { String[] parts = line.replaceAll("\"", "").split(",", 3); if (parts.length != 3) { log.error("Illegal line '" + line + "' in " - + geoIPASNum2CsvFile.getAbsolutePath() + "."); + + geoIpAsNum2CsvFile.getAbsolutePath() + "."); return lookupResults; } try { long startIpNum = Long.parseLong(parts[0]); if (startIpNum <= previousStartIpNum) { log.error("Line '" + line + "' not sorted in " - + geoIPASNum2CsvFile.getAbsolutePath() + "."); + + geoIpAsNum2CsvFile.getAbsolutePath() + "."); return lookupResults; } previousStartIpNum = startIpNum; @@ -264,7 +265,7 @@ public class LookupService { while (firstAddressNumber <= endIpNum && firstAddressNumber != -1L) { if (parts[2].startsWith("AS")) { - addressNumberASN.put(firstAddressNumber, parts[2]); + addressNumberAsn.put(firstAddressNumber, parts[2]); } sortedAddressNumbers.remove(firstAddressNumber); if (sortedAddressNumbers.isEmpty()) { @@ -279,13 +280,13 @@ public class LookupService { } catch (NumberFormatException e) { log.error("Number format exception while parsing line " + "'" + line + "' in " - + geoIPASNum2CsvFile.getAbsolutePath() + "."); + + geoIpAsNum2CsvFile.getAbsolutePath() + "."); return lookupResults; } } } catch (IOException e) { log.error("I/O exception while reading " - + geoIPASNum2CsvFile.getAbsolutePath() + ": " + e); + + geoIpAsNum2CsvFile.getAbsolutePath() + ": " + e); return lookupResults; } @@ -297,7 +298,7 @@ public class LookupService { long addressNumber = addressStringNumbers.get(addressString); if (!addressNumberBlocks.containsKey(addressNumber) && !addressNumberLatLong.containsKey(addressNumber) - && !addressNumberASN.containsKey(addressNumber)) { + && !addressNumberAsn.containsKey(addressNumber)) { continue; } LookupResult lookupResult = new LookupResult(); @@ -325,8 +326,8 @@ public class LookupService { lookupResult.setLatitude(latLong[0]); lookupResult.setLongitude(latLong[1]); } - if (addressNumberASN.containsKey(addressNumber)) { - String[] parts = addressNumberASN.get(addressNumber).split(" ", + if (addressNumberAsn.containsKey(addressNumber)) { + String[] parts = addressNumberAsn.get(addressNumber).split(" ", 2); lookupResult.setAsNumber(parts[0]); lookupResult.setAsName(parts.length == 2 ? parts[1] : ""); diff --git a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java index d873072..a66aba8 100644 --- a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java @@ -492,7 +492,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, updatedNodeStatus.setDefaultPolicy( nodeStatus.getDefaultPolicy()); updatedNodeStatus.setPortList(nodeStatus.getPortList()); - updatedNodeStatus.setASNumber(nodeStatus.getASNumber()); + updatedNodeStatus.setAsNumber(nodeStatus.getAsNumber()); updatedNodeStatus.setRecommendedVersion( nodeStatus.getRecommendedVersion()); } @@ -862,10 +862,10 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, detailsStatus.setCityName(lookupResult.getCityName()); detailsStatus.setLatitude(lookupResult.getLatitude()); detailsStatus.setLongitude(lookupResult.getLongitude()); - detailsStatus.setASNumber(lookupResult.getAsNumber()); - detailsStatus.setASName(lookupResult.getAsName()); + detailsStatus.setAsNumber(lookupResult.getAsNumber()); + detailsStatus.setAsName(lookupResult.getAsName()); nodeStatus.setCountryCode(lookupResult.getCountryCode()); - nodeStatus.setASNumber(lookupResult.getAsNumber()); + nodeStatus.setAsNumber(lookupResult.getAsNumber()); } if (this.consensusWeightFractions.containsKey(fingerprint)) { diff --git a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java index 7238c1b..33b943e 100644 --- a/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/BandwidthDocumentWriter.java @@ -107,12 +107,11 @@ public class BandwidthDocumentWriter implements DocumentWriter { long totalMillis = 0L; long totalBandwidth = 0L; for (long[] v : history.values()) { - long startMillis = v[0]; long endMillis = v[1]; - long bandwidth = v[2]; if (endMillis < intervalStartMillis) { continue; } + long startMillis = v[0]; if (endMillis - startMillis > dataPointInterval) { /* This history interval is too long for this graph's data point * interval. Maybe the next graph will contain it, but not this @@ -128,6 +127,7 @@ public class BandwidthDocumentWriter implements DocumentWriter { totalMillis = 0L; intervalStartMillis += dataPointInterval; } + long bandwidth = v[2]; totalBandwidth += bandwidth; totalMillis += (endMillis - startMillis); } diff --git a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java index c167152..1c515e8 100644 --- a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java @@ -94,8 +94,8 @@ public class DetailsDocumentWriter implements DocumentWriter { detailsDocument.setCountryName(detailsStatus.getCountryName()); detailsDocument.setRegionName(detailsStatus.getRegionName()); detailsDocument.setCityName(detailsStatus.getCityName()); - detailsDocument.setAsNumber(detailsStatus.getASNumber()); - detailsDocument.setAsName(detailsStatus.getASName()); + detailsDocument.setAsNumber(detailsStatus.getAsNumber()); + detailsDocument.setAsName(detailsStatus.getAsName()); if (detailsStatus.isRunning()) { detailsDocument.setConsensusWeightFraction( detailsStatus.getConsensusWeightFraction()); diff --git a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java index f941ee3..e9bb8f6 100644 --- a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java @@ -65,8 +65,6 @@ public class SummaryDocumentWriter implements DocumentWriter { } continue; } - boolean isRelay = nodeStatus.isRelay(); - String nickname = nodeStatus.getNickname(); List<String> addresses = new ArrayList<String>(); addresses.add(nodeStatus.getAddress()); for (String orAddress : nodeStatus.getOrAddresses()) { @@ -81,20 +79,22 @@ public class SummaryDocumentWriter implements DocumentWriter { } long lastSeenMillis = nodeStatus.getLastSeenMillis(); SortedSet<String> relayFlags = nodeStatus.getRelayFlags(); + boolean isRelay = nodeStatus.isRelay(); boolean running = relayFlags.contains("Running") && (isRelay ? lastSeenMillis == relaysLastValidAfterMillis : lastSeenMillis == bridgesLastPublishedMillis); long consensusWeight = nodeStatus.getConsensusWeight(); String countryCode = nodeStatus.getCountryCode(); long firstSeenMillis = nodeStatus.getFirstSeenMillis(); - String aSNumber = nodeStatus.getASNumber(); + String asNumber = nodeStatus.getAsNumber(); String contact = nodeStatus.getContact(); SortedSet<String> declaredFamily = nodeStatus.getDeclaredFamily(); SortedSet<String> effectiveFamily = nodeStatus.getEffectiveFamily(); + String nickname = nodeStatus.getNickname(); SummaryDocument summaryDocument = new SummaryDocument(isRelay, nickname, fingerprint, addresses, lastSeenMillis, running, relayFlags, consensusWeight, countryCode, firstSeenMillis, - aSNumber, contact, declaredFamily, effectiveFamily); + asNumber, contact, declaredFamily, effectiveFamily); if (this.documentStore.store(summaryDocument, fingerprint)) { this.writtenDocuments++; }

1 0

[onionoo/master] Reset sorted fingerprint blocks when setting fingerprint.
by karsten＠torproject.org 25 Jul '16

25 Jul '16

commit 54ef1a7e34659ab7941b7c2d769fd7a8d851d1a6 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Jul 19 10:22:15 2016 +0200 Reset sorted fingerprint blocks when setting fingerprint. Previously, we'd only have calculated sorted fingerprint blocks once, but not after setting a new fingerprint. We never ran into this bug, because we never called that setter outside of the constructor. But future uses of that setter would have caused trouble. Found while writing Javadoc comments for #19613. --- .../torproject/onionoo/docs/SummaryDocument.java | 1 + .../onionoo/docs/SummaryDocumentTest.java | 26 ++++++++++++++++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java index 6a520c8..c7d4774 100644 --- a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java @@ -44,6 +44,7 @@ public class SummaryDocument extends Document { this.f = fingerprint; this.hashedFingerprint = null; this.base64Fingerprint = null; + this.fingerprintSortedHexBlocks = null; } public String getFingerprint() { diff --git a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java index 3c097ae..99ae0c5 100644 --- a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java +++ b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java @@ -1,7 +1,8 @@ -/* Copyright 2015 The Tor Project +/* Copyright 2015--2016 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo.docs; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertTrue; import org.junit.Test; @@ -11,9 +12,8 @@ import java.util.TreeSet; public class SummaryDocumentTest { - @Test() - public void testFingerprintSortedHexBlocksAreSorted() { - SummaryDocument relayTorkaZ = new SummaryDocument(true, "TorkaZ", + private SummaryDocument createSummaryDocumentRelayTorkaZ() { + return new SummaryDocument(true, "TorkaZ", "000C5F55BD4814B917CC474BD537F1A3B33CCE2A", Arrays.asList( new String[] { "62.216.201.221", "62.216.201.222", "62.216.201.223" }), DateTimeHelper.parse("2013-04-19 05:00:00"), @@ -26,6 +26,11 @@ public class SummaryDocumentTest { "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })), new TreeSet<String>(Arrays.asList( new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" }))); + } + + @Test() + public void testFingerprintSortedHexBlocksAreSorted() { + SummaryDocument relayTorkaZ = this.createSummaryDocumentRelayTorkaZ(); String[] fingerprintSortedHexBlocks = relayTorkaZ.getFingerprintSortedHexBlocks(); for (int i = 0; i < fingerprintSortedHexBlocks.length - 1; i++) { @@ -34,5 +39,18 @@ public class SummaryDocumentTest { fingerprintSortedHexBlocks[i + 1]) <= 0); } } + + @Test() + public void testFingerprintSortedHexBlocksReset() { + SummaryDocument relayTorkaZ = this.createSummaryDocumentRelayTorkaZ(); + assertArrayEquals("Hex blocks differ", new String[] { "000C", "14B9", + "17CC", "474B", "5F55", "B33C", "BD48", "CE2A", "D537", "F1A3" }, + relayTorkaZ.getFingerprintSortedHexBlocks()); + relayTorkaZ.setFingerprint( + "A2ECC33B3A1F735D8474CC719B4184DB55F5C000"); + assertArrayEquals("Hex blocks differ", new String[] { "3A1F", "55F5", + "735D", "8474", "84DB", "9B41", "A2EC", "C000", "C33B", "CC71" }, + relayTorkaZ.getFingerprintSortedHexBlocks()); + } }

1 0