commit fa377c3535e9a0fd4ed6bcd051b89f01ebf30739 Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Thu May 28 10:36:49 2015 +0200 Switch to using metrics-lib's DescriptorCollector. --- .gitmodules | 3 +++ build.xml | 9 ++++++++- deps/metrics-lib | 1 + run-exonerator.sh | 3 --- .../torproject/exonerator/ExoneraTorDatabaseImporter.java | 13 +++++++++++++ 5 files changed, 25 insertions(+), 4 deletions(-) diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..c6b6388 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "deps/metrics-lib"] + path = deps/metrics-lib + url = https://git.torproject.org/metrics-lib diff --git a/build.xml b/build.xml index b6a1e53..3064bf8 100644 --- a/build.xml +++ b/build.xml @@ -11,6 +11,9 @@ <property name="warfile" value="exonerator.war"/> <path id="classpath"> <pathelement path="${classes}"/> + <fileset dir="deps/metrics-lib"> + <include name="descriptor.jar"/> + </fileset> <fileset dir="/usr/share/java"> <include name="commons-codec.jar"/> <include name="postgresql-jdbc3.jar"/> @@ -25,8 +28,12 @@ <mkdir dir="${classes}"/> </target> + <target name="metrics-lib"> + <ant dir="deps/metrics-lib"/> + </target> + <!-- Compile all servlets and plain Java classes. --> - <target name="compile" depends="init"> + <target name="compile" depends="metrics-lib, init"> <javac destdir="${classes}" srcdir="${sources}" source="1.5" diff --git a/deps/metrics-lib b/deps/metrics-lib new file mode 160000 index 0000000..bd18dd0 --- /dev/null +++ b/deps/metrics-lib @@ -0,0 +1 @@ +Subproject commit bd18dd0d2f788d1d4ee7acbac5032e97264bb855 diff --git a/run-exonerator.sh b/run-exonerator.sh index 6cd34bf..d090ec9 100755 --- a/run-exonerator.sh +++ b/run-exonerator.sh @@ -1,6 +1,3 @@ #!/bin/bash -rsync -arz --delete metrics.torproject.org::metrics-recent/relay-descriptors/consensuses exonerator-import/ -rsync -arz --delete metrics.torproject.org::metrics-recent/relay-descriptors/server-descriptors exonerator-import/ -rsync -arz --delete metrics.torproject.org::metrics-recent/exit-lists exonerator-import/ ant run | grep "\[java\]" diff --git a/src/org/torproject/exonerator/ExoneraTorDatabaseImporter.java b/src/org/torproject/exonerator/ExoneraTorDatabaseImporter.java index 348c04c..6111888 100644 --- a/src/org/torproject/exonerator/ExoneraTorDatabaseImporter.java +++ b/src/org/torproject/exonerator/ExoneraTorDatabaseImporter.java @@ -32,6 +32,8 @@ import java.util.TimeZone; import org.apache.commons.codec.binary.Base64; import org.apache.commons.codec.binary.Hex; import org.apache.commons.codec.digest.DigestUtils; +import org.torproject.descriptor.DescriptorCollector; +import org.torproject.descriptor.DescriptorSourceFactory; /* Import Tor descriptors into the ExoneraTor database. */ public class ExoneraTorDatabaseImporter { @@ -42,6 +44,7 @@ public class ExoneraTorDatabaseImporter { openDatabaseConnection(); prepareDatabaseStatements(); createLockFile(); + fetchDescriptors(); readImportHistoryToMemory(); parseDescriptors(); writeImportHistoryToDisk(); @@ -150,6 +153,16 @@ public class ExoneraTorDatabaseImporter { } } + /* Fetch recent descriptors from CollecTor. */ + private static void fetchDescriptors() { + DescriptorCollector collector = + DescriptorSourceFactory.createDescriptorCollector(); + collector.collectDescriptors("https://collector.torproject.org", + new String[] { "/recent/relay-descriptors/consensuses/", + "/recent/relay-descriptors/server-descriptors/", + "/recent/exit-lists/" }, 0L, new File(importDirString), true); + } + /* Last and next parse histories containing paths of parsed files and * last modified times. */ private static Map<String, Long>