commit 6a2425042dcc2e61152ea6bd6a93216842a4aed7 Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Sun Aug 9 15:45:17 2015 +0200 Use metrics-lib's DescriptorCollector rather than rsync. --- deps/metrics-lib | 2 +- .../src/org/torproject/metrics/advbwdist/Main.java | 5 ++- modules/collectdescs/build.xml | 42 ++++++++++++++++++++ .../org/torproject/metrics/collectdescs/Main.java | 27 +++++++++++++ .../src/org/torproject/metrics/hidserv/Main.java | 4 +- modules/legacy/config.template | 6 +-- shared/bin/01-collectdescs.sh | 5 +++ shared/bin/01-rsync-descriptors.sh | 3 -- 8 files changed, 83 insertions(+), 11 deletions(-) diff --git a/deps/metrics-lib b/deps/metrics-lib index bd18dd0..1960762 160000 --- a/deps/metrics-lib +++ b/deps/metrics-lib @@ -1 +1 @@ -Subproject commit bd18dd0d2f788d1d4ee7acbac5032e97264bb855 +Subproject commit 1960762ca3ad515a22b90fdd3a1f4c68a3fa9588 diff --git a/modules/advbwdist/src/org/torproject/metrics/advbwdist/Main.java b/modules/advbwdist/src/org/torproject/metrics/advbwdist/Main.java index 09eaab0..9ac2bbb 100644 --- a/modules/advbwdist/src/org/torproject/metrics/advbwdist/Main.java +++ b/modules/advbwdist/src/org/torproject/metrics/advbwdist/Main.java @@ -28,7 +28,8 @@ public class Main { DescriptorReader descriptorReader = DescriptorSourceFactory.createDescriptorReader(); descriptorReader.addDirectory( - new File("../../shared/in/relay-descriptors/server-descriptors")); + new File("../../shared/in/recent/relay-descriptors/" + + "server-descriptors")); Iterator<DescriptorFile> descriptorFiles = descriptorReader.readDescriptors(); Map<String, Long> serverDescriptors = @@ -52,7 +53,7 @@ public class Main { /* Parse consensuses, keeping a parse history. */ descriptorReader = DescriptorSourceFactory.createDescriptorReader(); descriptorReader.addDirectory( - new File("../../shared/in/relay-descriptors/consensuses")); + new File("../../shared/in/recent/relay-descriptors/consensuses")); descriptorReader.setExcludeFiles( new File("status/parsed-consensuses")); descriptorFiles = descriptorReader.readDescriptors(); diff --git a/modules/collectdescs/build.xml b/modules/collectdescs/build.xml new file mode 100644 index 0000000..bcbea76 --- /dev/null +++ b/modules/collectdescs/build.xml @@ -0,0 +1,42 @@ +<project default="run" name="collectdescs" basedir="."> + + <property name="sources" value="src"/> + <property name="classes" value="classes"/> + <path id="classpath"> + <pathelement path="${classes}"/> + <fileset dir="/usr/share/java"> + <include name="commons-compress-1.4.1.jar"/> + <include name="commons-lang-2.6.jar"/> + </fileset> + <fileset dir="../../deps/metrics-lib"> + <include name="descriptor.jar"/> + </fileset> + </path> + + <target name="metrics-lib"> + <ant dir="../../deps/metrics-lib"/> + </target> + + <target name="compile" depends="metrics-lib"> + <mkdir dir="${classes}"/> + <javac destdir="${classes}" + srcdir="${sources}" + source="1.6" + target="1.6" + debug="true" + deprecation="true" + optimize="false" + failonerror="true" + includeantruntime="false"> + <classpath refid="classpath"/> + </javac> + </target> + + <target name="run" depends="compile"> + <java fork="true" + classname="org.torproject.metrics.collectdescs.Main"> + <classpath refid="classpath"/> + </java> + </target> +</project> + diff --git a/modules/collectdescs/src/org/torproject/metrics/collectdescs/Main.java b/modules/collectdescs/src/org/torproject/metrics/collectdescs/Main.java new file mode 100644 index 0000000..c95df18 --- /dev/null +++ b/modules/collectdescs/src/org/torproject/metrics/collectdescs/Main.java @@ -0,0 +1,27 @@ +/* Copyright 2015 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.metrics.collectdescs; + +import java.io.File; + +import org.torproject.descriptor.DescriptorCollector; +import org.torproject.descriptor.DescriptorSourceFactory; + +public class Main { + public static void main(String[] args) { + /* Fetch recent descriptors from CollecTor. */ + DescriptorCollector collector = + DescriptorSourceFactory.createDescriptorCollector(); + collector.collectDescriptors( + "https://collector.torproject.org", new String[] { + "/recent/bridge-descriptors/extra-infos/", + "/recent/bridge-descriptors/server-descriptors/", + "/recent/bridge-descriptors/statuses/", + "/recent/exit-lists/", + "/recent/relay-descriptors/consensuses/", + "/recent/relay-descriptors/extra-infos/", + "/recent/relay-descriptors/server-descriptors/", + "/recent/torperf/" }, 0L, new File("../../shared/in"), true); + } +} + diff --git a/modules/hidserv/src/org/torproject/metrics/hidserv/Main.java b/modules/hidserv/src/org/torproject/metrics/hidserv/Main.java index 1e53bd0..7405b78 100644 --- a/modules/hidserv/src/org/torproject/metrics/hidserv/Main.java +++ b/modules/hidserv/src/org/torproject/metrics/hidserv/Main.java @@ -18,9 +18,9 @@ public class Main { /* Initialize directories and file paths. */ Set<File> inDirectories = new HashSet<File>(); inDirectories.add( - new File("../../shared/in/relay-descriptors/consensuses")); + new File("../../shared/in/recent/relay-descriptors/consensuses")); inDirectories.add( - new File("../../shared/in/relay-descriptors/extra-infos")); + new File("../../shared/in/recent/relay-descriptors/extra-infos")); File statusDirectory = new File("status"); File hidservStatsExtrapolatedCsvFile = new File("stats/hidserv.csv"); diff --git a/modules/legacy/config.template b/modules/legacy/config.template index e7b5611..8a7bc96 100644 --- a/modules/legacy/config.template +++ b/modules/legacy/config.template @@ -2,7 +2,7 @@ #ImportDirectoryArchives 0 # ## Relative path to directory to import directory archives from -#DirectoryArchivesDirectory /srv/metrics.torproject.org/web/shared/in/relay-descriptors/ +#DirectoryArchivesDirectory /srv/metrics.torproject.org/web/shared/in/recent/relay-descriptors/ # ## Keep a history of imported directory archive files to know which files ## have been imported before. This history can be useful when importing @@ -14,7 +14,7 @@ #ImportSanitizedBridges 0 # ## Relative path to directory to import sanitized bridges from -#SanitizedBridgesDirectory /srv/metrics.torproject.org/web/shared/in/bridge-descriptors/ +#SanitizedBridgesDirectory /srv/metrics.torproject.org/web/shared/in/recent/bridge-descriptors/ # ## Keep a history of imported sanitized bridge descriptors. This history ## can be useful when importing from a changing data source to avoid @@ -43,5 +43,5 @@ #ImportWriteTorperfStats 0 # ## Relative path to directory to import torperf results from -#TorperfDirectory /srv/metrics.torproject.org/web/shared/in/torperf/ +#TorperfDirectory /srv/metrics.torproject.org/web/shared/in/recent/torperf/ # diff --git a/shared/bin/01-collectdescs.sh b/shared/bin/01-collectdescs.sh new file mode 100755 index 0000000..18b900f --- /dev/null +++ b/shared/bin/01-collectdescs.sh @@ -0,0 +1,5 @@ +#!/bin/sh +cd modules/collectdescs/ +ant | grep "\[java\]" +cd ../../ + diff --git a/shared/bin/01-rsync-descriptors.sh b/shared/bin/01-rsync-descriptors.sh deleted file mode 100755 index fa62c5e..0000000 --- a/shared/bin/01-rsync-descriptors.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -rsync -arz --delete --exclude 'relay-descriptors/votes' --exclude 'relay-descriptors/microdescs' metrics.torproject.org::metrics-recent shared/in -