commit 8e5a7a7d9ebc47bfbde6b1463e0e146a7eaa247e Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Wed Nov 7 12:04:02 2012 -0500 Make five modules executable on their own. This will prevent one failing module from breaking the other four. It will also make it easier to change the timing of execution. --- .gitignore | 2 +- bin/run-bridgedescs | 5 + bin/run-bridgepools | 5 + bin/run-exitlists | 5 + bin/run-relaydescs | 5 + bin/run-torperf | 5 + build.xml | 44 ++++++++-- run.sh | 5 - .../db/bridgedescs/SanitizedBridgesWriter.java | 39 ++++++++-- .../BridgePoolAssignmentsProcessor.java | 38 ++++++++-- .../ernie/db/exitlists/ExitListDownloader.java | 37 +++++++-- src/org/torproject/ernie/db/main/LockFile.java | 11 ++- .../ernie/db/main/LoggingConfiguration.java | 7 +- src/org/torproject/ernie/db/main/Main.java | 85 -------------------- .../ernie/db/relaydescs/ArchiveWriter.java | 31 +++++++ .../ernie/db/torperf/TorperfDownloader.java | 37 +++++++-- 16 files changed, 232 insertions(+), 129 deletions(-) diff --git a/.gitignore b/.gitignore index 22955e3..dd63dac 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,7 @@ out/ doc/manual.aux doc/manual.log lib/ -log.* +log/ rsync/ stats/ tarballs/ diff --git a/bin/run-bridgedescs b/bin/run-bridgedescs new file mode 100755 index 0000000..4bd6d28 --- /dev/null +++ b/bin/run-bridgedescs @@ -0,0 +1,5 @@ +#!/bin/sh + +# TODO is there a better way to suppress Ant's output? +ant bridgedescs -q | grep -Ev "^$|^BUILD SUCCESSFUL|^Total time: " + diff --git a/bin/run-bridgepools b/bin/run-bridgepools new file mode 100755 index 0000000..dba4e1f --- /dev/null +++ b/bin/run-bridgepools @@ -0,0 +1,5 @@ +#!/bin/sh + +# TODO is there a better way to suppress Ant's output? +ant bridgepools -q | grep -Ev "^$|^BUILD SUCCESSFUL|^Total time: " + diff --git a/bin/run-exitlists b/bin/run-exitlists new file mode 100755 index 0000000..768afa1 --- /dev/null +++ b/bin/run-exitlists @@ -0,0 +1,5 @@ +#!/bin/sh + +# TODO is there a better way to suppress Ant's output? +ant exitlists -q | grep -Ev "^$|^BUILD SUCCESSFUL|^Total time: " + diff --git a/bin/run-relaydescs b/bin/run-relaydescs new file mode 100755 index 0000000..f91f804 --- /dev/null +++ b/bin/run-relaydescs @@ -0,0 +1,5 @@ +#!/bin/sh + +# TODO is there a better way to suppress Ant's output? +ant relaydescs -q | grep -Ev "^$|^BUILD SUCCESSFUL|^Total time: " + diff --git a/bin/run-torperf b/bin/run-torperf new file mode 100755 index 0000000..b192e87 --- /dev/null +++ b/bin/run-torperf @@ -0,0 +1,5 @@ +#!/bin/sh + +# TODO is there a better way to suppress Ant's output? +ant torperf -q | grep -Ev "^$|^BUILD SUCCESSFUL|^Total time: " + diff --git a/build.xml b/build.xml index 7cba58c..058db5d 100644 --- a/build.xml +++ b/build.xml @@ -1,4 +1,4 @@ -<project default="run" name="ERNIE" basedir="."> +<project default="compile" name="ERNIE" basedir="."> <property name="sources" value="src/"/> <property name="classes" value="classes/"/> <property name="docs" value="javadoc/"/> @@ -23,13 +23,6 @@ <classpath refid="classpath"/> </javac> </target> - <target name="run" depends="compile"> - <java fork="true" - maxmemory="2048m" - classname="org.torproject.ernie.db.main.Main"> - <classpath refid="classpath"/> - </java> - </target> <target name="docs"> <mkdir dir="${docs}"/> <javadoc destdir="${docs}"> @@ -37,5 +30,40 @@ <fileset dir="${sources}/" includes="**/*.java" /> </javadoc> </target> + <target name="bridgedescs" depends="compile"> + <java fork="true" + maxmemory="2048m" + classname="org.torproject.ernie.db.bridgedescs.SanitizedBridgesWriter"> + <classpath refid="classpath"/> + </java> + </target> + <target name="bridgepools" depends="compile"> + <java fork="true" + maxmemory="2048m" + classname="org.torproject.ernie.db.bridgepools.BridgePoolAssignmentProcessor"> + <classpath refid="classpath"/> + </java> + </target> + <target name="exitlists" depends="compile"> + <java fork="true" + maxmemory="2048m" + classname="org.torproject.ernie.db.exitlists.ExitListDownloader"> + <classpath refid="classpath"/> + </java> + </target> + <target name="relaydescs" depends="compile"> + <java fork="true" + maxmemory="2048m" + classname="org.torproject.ernie.db.relaydescs.ArchiveWriter"> + <classpath refid="classpath"/> + </java> + </target> + <target name="torperf" depends="compile"> + <java fork="true" + maxmemory="2048m" + classname="org.torproject.ernie.db.torperf.TorperfDownloader"> + <classpath refid="classpath"/> + </java> + </target> </project> diff --git a/run.sh b/run.sh deleted file mode 100755 index e804876..0000000 --- a/run.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -# TODO is there a better way to suppress Ant's output? -ant -q | grep -Ev "^$|^BUILD SUCCESSFUL|^Total time: " - diff --git a/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java b/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java index 7de9961..0c4e76a 100644 --- a/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java +++ b/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java @@ -30,6 +30,8 @@ import org.apache.commons.codec.binary.Base64; import org.apache.commons.codec.binary.Hex; import org.apache.commons.codec.digest.DigestUtils; import org.torproject.ernie.db.main.Configuration; +import org.torproject.ernie.db.main.LockFile; +import org.torproject.ernie.db.main.LoggingConfiguration; /** * Sanitizes bridge descriptors, i.e., removes all possibly sensitive @@ -46,6 +48,37 @@ import org.torproject.ernie.db.main.Configuration; */ public class SanitizedBridgesWriter extends Thread { + public static void main(String[] args) { + + /* Initialize logging configuration. */ + new LoggingConfiguration("bridge-descriptors"); + Logger logger = Logger.getLogger( + SanitizedBridgesWriter.class.getName()); + logger.info("Starting bridge-descriptors module of ERNIE."); + + // Initialize configuration + Configuration config = new Configuration(); + + // Use lock file to avoid overlapping runs + LockFile lf = new LockFile("bridge-descriptors"); + if (!lf.acquireLock()) { + logger.severe("Warning: ERNIE is already running or has not exited " + + "cleanly! Exiting!"); + System.exit(1); + } + + // Sanitize bridge descriptors + if (config.getImportBridgeSnapshots() && + config.getWriteSanitizedBridges()) { + new SanitizedBridgesWriter(config).run(); + } + + // Remove lock file + lf.releaseLock(); + + logger.info("Terminating bridge-descriptors module of ERNIE."); + } + private Configuration config; /** @@ -83,12 +116,6 @@ public class SanitizedBridgesWriter extends Thread { public void run() { - if (((System.currentTimeMillis() / 60000L) % 60L) > 30L) { - /* Don't start in second half of an hour, when we only want to - * process other data. */ - return; - } - File bridgeDirectoriesDirectory = new File(config.getBridgeSnapshotsDirectory()); File sanitizedBridgesDirectory = diff --git a/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java b/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java index 43d3427..ac45787 100644 --- a/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java +++ b/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java @@ -27,9 +27,41 @@ import org.apache.commons.codec.binary.Hex; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; import org.torproject.ernie.db.main.Configuration; +import org.torproject.ernie.db.main.LockFile; +import org.torproject.ernie.db.main.LoggingConfiguration; public class BridgePoolAssignmentsProcessor extends Thread { + public static void main(String[] args) { + + /* Initialize logging configuration. */ + new LoggingConfiguration("bridge-pool-assignments"); + Logger logger = Logger.getLogger( + BridgePoolAssignmentsProcessor.class.getName()); + logger.info("Starting bridge-pool-assignments module of ERNIE."); + + // Initialize configuration + Configuration config = new Configuration(); + + // Use lock file to avoid overlapping runs + LockFile lf = new LockFile("bridge-pool-assignments"); + if (!lf.acquireLock()) { + logger.severe("Warning: ERNIE is already running or has not exited " + + "cleanly! Exiting!"); + System.exit(1); + } + + // Process bridge pool assignments + if (config.getProcessBridgePoolAssignments()) { + new BridgePoolAssignmentsProcessor(config).run(); + } + + // Remove lock file + lf.releaseLock(); + + logger.info("Terminating bridge-pool-assignments module of ERNIE."); + } + private Configuration config; public BridgePoolAssignmentsProcessor(Configuration config) { @@ -38,12 +70,6 @@ public class BridgePoolAssignmentsProcessor extends Thread { public void run() { - if (((System.currentTimeMillis() / 60000L) % 60L) > 30L) { - /* Don't start in second half of an hour, when we only want to - * process other data. */ - return; - } - File assignmentsDirectory = new File(config.getAssignmentsDirectory()); File sanitizedAssignmentsDirectory = diff --git a/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java b/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java index fef3777..825de65 100644 --- a/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java +++ b/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java @@ -27,20 +27,45 @@ import org.torproject.descriptor.ExitList; import org.torproject.descriptor.ExitListEntry; import org.torproject.descriptor.impl.DescriptorParseException; import org.torproject.ernie.db.main.Configuration; +import org.torproject.ernie.db.main.LockFile; +import org.torproject.ernie.db.main.LoggingConfiguration; public class ExitListDownloader extends Thread { + public static void main(String[] args) { + + /* Initialize logging configuration. */ + new LoggingConfiguration("exit-lists"); + Logger logger = Logger.getLogger(ExitListDownloader.class.getName()); + logger.info("Starting exit-lists module of ERNIE."); + + // Initialize configuration + Configuration config = new Configuration(); + + // Use lock file to avoid overlapping runs + LockFile lf = new LockFile("exit-lists"); + if (!lf.acquireLock()) { + logger.severe("Warning: ERNIE is already running or has not exited " + + "cleanly! Exiting!"); + System.exit(1); + } + + // Download exit list and store it to disk + if (config.getDownloadExitList()) { + new ExitListDownloader(config).run(); + } + + // Remove lock file + lf.releaseLock(); + + logger.info("Terminating exit-lists module of ERNIE."); + } + public ExitListDownloader(Configuration config) { } public void run() { - if (((System.currentTimeMillis() / 60000L) % 60L) > 30L) { - /* Don't start in second half of an hour, when we only want to - * process other data. */ - return; - } - Logger logger = Logger.getLogger(ExitListDownloader.class.getName()); SimpleDateFormat dateTimeFormat = diff --git a/src/org/torproject/ernie/db/main/LockFile.java b/src/org/torproject/ernie/db/main/LockFile.java index 68375ec..d356d90 100644 --- a/src/org/torproject/ernie/db/main/LockFile.java +++ b/src/org/torproject/ernie/db/main/LockFile.java @@ -15,8 +15,8 @@ public class LockFile { private File lockFile; private Logger logger; - public LockFile() { - this.lockFile = new File("lock"); + public LockFile(String moduleName) { + this.lockFile = new File("lock/" + moduleName); this.logger = Logger.getLogger(LockFile.class.getName()); } @@ -24,14 +24,17 @@ public class LockFile { this.logger.fine("Trying to acquire lock..."); try { if (this.lockFile.exists()) { - BufferedReader br = new BufferedReader(new FileReader("lock")); + BufferedReader br = new BufferedReader(new FileReader( + this.lockFile)); long runStarted = Long.parseLong(br.readLine()); br.close(); if (System.currentTimeMillis() - runStarted < 55L * 60L * 1000L) { return false; } } - BufferedWriter bw = new BufferedWriter(new FileWriter("lock")); + this.lockFile.getParentFile().mkdirs(); + BufferedWriter bw = new BufferedWriter(new FileWriter( + this.lockFile)); bw.append("" + System.currentTimeMillis() + "\n"); bw.close(); this.logger.fine("Acquired lock."); diff --git a/src/org/torproject/ernie/db/main/LoggingConfiguration.java b/src/org/torproject/ernie/db/main/LoggingConfiguration.java index 78a60d1..c60eb34 100644 --- a/src/org/torproject/ernie/db/main/LoggingConfiguration.java +++ b/src/org/torproject/ernie/db/main/LoggingConfiguration.java @@ -2,6 +2,7 @@ * See LICENSE for licensing information */ package org.torproject.ernie.db.main; +import java.io.File; import java.io.IOException; import java.text.SimpleDateFormat; import java.util.Date; @@ -32,7 +33,7 @@ import java.util.logging.Logger; * - FINEST: Most detailed debug messages. Not used. */ public class LoggingConfiguration { - public LoggingConfiguration() { + public LoggingConfiguration(String moduleName) { /* Remove default console handler. */ for (Handler h : Logger.getLogger("").getHandlers()) { @@ -77,7 +78,9 @@ public class LoggingConfiguration { } }; try { - FileHandler fh = new FileHandler("log", 5000000, 5, true); + new File("log").mkdirs(); + FileHandler fh = new FileHandler("log/" + moduleName, 5000000, 5, + true); fh.setFormatter(ff); fh.setLevel(Level.FINE); Logger.getLogger("").addHandler(fh); diff --git a/src/org/torproject/ernie/db/main/Main.java b/src/org/torproject/ernie/db/main/Main.java deleted file mode 100644 index d1361cf..0000000 --- a/src/org/torproject/ernie/db/main/Main.java +++ /dev/null @@ -1,85 +0,0 @@ -/* Copyright 2010--2012 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.ernie.db.main; - -import java.util.ArrayList; -import java.util.List; -import java.util.logging.Logger; - -import org.torproject.ernie.db.bridgedescs.SanitizedBridgesWriter; -import org.torproject.ernie.db.bridgepools.BridgePoolAssignmentsProcessor; -import org.torproject.ernie.db.exitlists.ExitListDownloader; -import org.torproject.ernie.db.relaydescs.ArchiveWriter; -import org.torproject.ernie.db.torperf.TorperfDownloader; - -/** - * Coordinate downloading and parsing of descriptors and extraction of - * statistically relevant data for later processing with R. - */ -public class Main { - public static void main(String[] args) { - - /* Initialize logging configuration. */ - new LoggingConfiguration(); - - Logger logger = Logger.getLogger(Main.class.getName()); - logger.info("Starting ERNIE."); - - // Initialize configuration - Configuration config = new Configuration(); - - // Use lock file to avoid overlapping runs - LockFile lf = new LockFile(); - if (!lf.acquireLock()) { - logger.severe("Warning: ERNIE is already running or has not exited " - + "cleanly! Exiting!"); - System.exit(1); - } - - // Keep a list of all threads, so that we can join them all before - // exiting. - List<Thread> threads = new ArrayList<Thread>(); - - // Import/download relay descriptors from the various sources - if (config.getWriteDirectoryArchives()) { - threads.add(new ArchiveWriter(config)); - } - - // Sanitize bridge descriptors - if (config.getImportBridgeSnapshots() && - config.getWriteSanitizedBridges()) { - threads.add(new SanitizedBridgesWriter(config)); - } - - // Download exit list and store it to disk - if (config.getDownloadExitList()) { - threads.add(new ExitListDownloader(config)); - } - - // Process bridge pool assignments - if (config.getProcessBridgePoolAssignments()) { - threads.add(new BridgePoolAssignmentsProcessor(config)); - } - - // Process Torperf files - if (config.getProcessTorperfFiles()) { - threads.add(new TorperfDownloader(config)); - } - - // Run threads - for (Thread thread : threads) { - thread.start(); - } - for (Thread thread : threads) { - try { - thread.join(); - } catch (InterruptedException e) { - } - } - - // Remove lock file - lf.releaseLock(); - - logger.info("Terminating ERNIE."); - } -} diff --git a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java index 2f1df0f..1ffb0cb 100644 --- a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java +++ b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java @@ -31,9 +31,40 @@ import org.torproject.descriptor.DescriptorParser; import org.torproject.descriptor.DescriptorSourceFactory; import org.torproject.descriptor.impl.DescriptorParseException; import org.torproject.ernie.db.main.Configuration; +import org.torproject.ernie.db.main.LockFile; +import org.torproject.ernie.db.main.LoggingConfiguration; public class ArchiveWriter extends Thread { + public static void main(String[] args) { + + /* Initialize logging configuration. */ + new LoggingConfiguration("relay-descriptors"); + Logger logger = Logger.getLogger(ArchiveWriter.class.getName()); + logger.info("Starting relay-descriptors module of ERNIE."); + + // Initialize configuration + Configuration config = new Configuration(); + + // Use lock file to avoid overlapping runs + LockFile lf = new LockFile("relay-descriptors"); + if (!lf.acquireLock()) { + logger.severe("Warning: ERNIE is already running or has not exited " + + "cleanly! Exiting!"); + System.exit(1); + } + + // Import/download relay descriptors from the various sources + if (config.getWriteDirectoryArchives()) { + new ArchiveWriter(config).run(); + } + + // Remove lock file + lf.releaseLock(); + + logger.info("Terminating relay-descriptors module of ERNIE."); + } + private Configuration config; public ArchiveWriter(Configuration config) { diff --git a/src/org/torproject/ernie/db/torperf/TorperfDownloader.java b/src/org/torproject/ernie/db/torperf/TorperfDownloader.java index 4201858..13d5846 100644 --- a/src/org/torproject/ernie/db/torperf/TorperfDownloader.java +++ b/src/org/torproject/ernie/db/torperf/TorperfDownloader.java @@ -23,12 +23,43 @@ import java.util.logging.Level; import java.util.logging.Logger; import org.torproject.ernie.db.main.Configuration; +import org.torproject.ernie.db.main.LockFile; +import org.torproject.ernie.db.main.LoggingConfiguration; /* Download possibly truncated Torperf .data and .extradata files from * configured sources, append them to the files we already have, and merge * the two files into the .tpf format. */ public class TorperfDownloader extends Thread { + public static void main(String[] args) { + + /* Initialize logging configuration. */ + new LoggingConfiguration("torperf"); + Logger logger = Logger.getLogger(TorperfDownloader.class.getName()); + logger.info("Starting torperf module of ERNIE."); + + // Initialize configuration + Configuration config = new Configuration(); + + // Use lock file to avoid overlapping runs + LockFile lf = new LockFile("torperf"); + if (!lf.acquireLock()) { + logger.severe("Warning: ERNIE is already running or has not exited " + + "cleanly! Exiting!"); + System.exit(1); + } + + // Process Torperf files + if (config.getProcessTorperfFiles()) { + new TorperfDownloader(config).run(); + } + + // Remove lock file + lf.releaseLock(); + + logger.info("Terminating torperf module of ERNIE."); + } + private Configuration config; public TorperfDownloader(Configuration config) { @@ -43,12 +74,6 @@ public class TorperfDownloader extends Thread { public void run() { - if (((System.currentTimeMillis() / 60000L) % 60L) > 30L) { - /* Don't start in second half of an hour, when we only want to - * process other data. */ - return; - } - File torperfOutputDirectory = new File(config.getTorperfOutputDirectory()); SortedMap<String, String> torperfSources = config.getTorperfSources();