[tor-commits] [metrics-web/master] Use metrics-lib's DescriptorCollector rather than rsync.
karsten at torproject.org
karsten at torproject.org
Sun Aug 16 10:55:26 UTC 2015
commit 6a2425042dcc2e61152ea6bd6a93216842a4aed7
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Sun Aug 9 15:45:17 2015 +0200
Use metrics-lib's DescriptorCollector rather than rsync.
---
deps/metrics-lib | 2 +-
.../src/org/torproject/metrics/advbwdist/Main.java | 5 ++-
modules/collectdescs/build.xml | 42 ++++++++++++++++++++
.../org/torproject/metrics/collectdescs/Main.java | 27 +++++++++++++
.../src/org/torproject/metrics/hidserv/Main.java | 4 +-
modules/legacy/config.template | 6 +--
shared/bin/01-collectdescs.sh | 5 +++
shared/bin/01-rsync-descriptors.sh | 3 --
8 files changed, 83 insertions(+), 11 deletions(-)
diff --git a/deps/metrics-lib b/deps/metrics-lib
index bd18dd0..1960762 160000
--- a/deps/metrics-lib
+++ b/deps/metrics-lib
@@ -1 +1 @@
-Subproject commit bd18dd0d2f788d1d4ee7acbac5032e97264bb855
+Subproject commit 1960762ca3ad515a22b90fdd3a1f4c68a3fa9588
diff --git a/modules/advbwdist/src/org/torproject/metrics/advbwdist/Main.java b/modules/advbwdist/src/org/torproject/metrics/advbwdist/Main.java
index 09eaab0..9ac2bbb 100644
--- a/modules/advbwdist/src/org/torproject/metrics/advbwdist/Main.java
+++ b/modules/advbwdist/src/org/torproject/metrics/advbwdist/Main.java
@@ -28,7 +28,8 @@ public class Main {
DescriptorReader descriptorReader =
DescriptorSourceFactory.createDescriptorReader();
descriptorReader.addDirectory(
- new File("../../shared/in/relay-descriptors/server-descriptors"));
+ new File("../../shared/in/recent/relay-descriptors/"
+ + "server-descriptors"));
Iterator<DescriptorFile> descriptorFiles =
descriptorReader.readDescriptors();
Map<String, Long> serverDescriptors =
@@ -52,7 +53,7 @@ public class Main {
/* Parse consensuses, keeping a parse history. */
descriptorReader = DescriptorSourceFactory.createDescriptorReader();
descriptorReader.addDirectory(
- new File("../../shared/in/relay-descriptors/consensuses"));
+ new File("../../shared/in/recent/relay-descriptors/consensuses"));
descriptorReader.setExcludeFiles(
new File("status/parsed-consensuses"));
descriptorFiles = descriptorReader.readDescriptors();
diff --git a/modules/collectdescs/build.xml b/modules/collectdescs/build.xml
new file mode 100644
index 0000000..bcbea76
--- /dev/null
+++ b/modules/collectdescs/build.xml
@@ -0,0 +1,42 @@
+<project default="run" name="collectdescs" basedir=".">
+
+ <property name="sources" value="src"/>
+ <property name="classes" value="classes"/>
+ <path id="classpath">
+ <pathelement path="${classes}"/>
+ <fileset dir="/usr/share/java">
+ <include name="commons-compress-1.4.1.jar"/>
+ <include name="commons-lang-2.6.jar"/>
+ </fileset>
+ <fileset dir="../../deps/metrics-lib">
+ <include name="descriptor.jar"/>
+ </fileset>
+ </path>
+
+ <target name="metrics-lib">
+ <ant dir="../../deps/metrics-lib"/>
+ </target>
+
+ <target name="compile" depends="metrics-lib">
+ <mkdir dir="${classes}"/>
+ <javac destdir="${classes}"
+ srcdir="${sources}"
+ source="1.6"
+ target="1.6"
+ debug="true"
+ deprecation="true"
+ optimize="false"
+ failonerror="true"
+ includeantruntime="false">
+ <classpath refid="classpath"/>
+ </javac>
+ </target>
+
+ <target name="run" depends="compile">
+ <java fork="true"
+ classname="org.torproject.metrics.collectdescs.Main">
+ <classpath refid="classpath"/>
+ </java>
+ </target>
+</project>
+
diff --git a/modules/collectdescs/src/org/torproject/metrics/collectdescs/Main.java b/modules/collectdescs/src/org/torproject/metrics/collectdescs/Main.java
new file mode 100644
index 0000000..c95df18
--- /dev/null
+++ b/modules/collectdescs/src/org/torproject/metrics/collectdescs/Main.java
@@ -0,0 +1,27 @@
+/* Copyright 2015 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.metrics.collectdescs;
+
+import java.io.File;
+
+import org.torproject.descriptor.DescriptorCollector;
+import org.torproject.descriptor.DescriptorSourceFactory;
+
+public class Main {
+ public static void main(String[] args) {
+ /* Fetch recent descriptors from CollecTor. */
+ DescriptorCollector collector =
+ DescriptorSourceFactory.createDescriptorCollector();
+ collector.collectDescriptors(
+ "https://collector.torproject.org", new String[] {
+ "/recent/bridge-descriptors/extra-infos/",
+ "/recent/bridge-descriptors/server-descriptors/",
+ "/recent/bridge-descriptors/statuses/",
+ "/recent/exit-lists/",
+ "/recent/relay-descriptors/consensuses/",
+ "/recent/relay-descriptors/extra-infos/",
+ "/recent/relay-descriptors/server-descriptors/",
+ "/recent/torperf/" }, 0L, new File("../../shared/in"), true);
+ }
+}
+
diff --git a/modules/hidserv/src/org/torproject/metrics/hidserv/Main.java b/modules/hidserv/src/org/torproject/metrics/hidserv/Main.java
index 1e53bd0..7405b78 100644
--- a/modules/hidserv/src/org/torproject/metrics/hidserv/Main.java
+++ b/modules/hidserv/src/org/torproject/metrics/hidserv/Main.java
@@ -18,9 +18,9 @@ public class Main {
/* Initialize directories and file paths. */
Set<File> inDirectories = new HashSet<File>();
inDirectories.add(
- new File("../../shared/in/relay-descriptors/consensuses"));
+ new File("../../shared/in/recent/relay-descriptors/consensuses"));
inDirectories.add(
- new File("../../shared/in/relay-descriptors/extra-infos"));
+ new File("../../shared/in/recent/relay-descriptors/extra-infos"));
File statusDirectory = new File("status");
File hidservStatsExtrapolatedCsvFile = new File("stats/hidserv.csv");
diff --git a/modules/legacy/config.template b/modules/legacy/config.template
index e7b5611..8a7bc96 100644
--- a/modules/legacy/config.template
+++ b/modules/legacy/config.template
@@ -2,7 +2,7 @@
#ImportDirectoryArchives 0
#
## Relative path to directory to import directory archives from
-#DirectoryArchivesDirectory /srv/metrics.torproject.org/web/shared/in/relay-descriptors/
+#DirectoryArchivesDirectory /srv/metrics.torproject.org/web/shared/in/recent/relay-descriptors/
#
## Keep a history of imported directory archive files to know which files
## have been imported before. This history can be useful when importing
@@ -14,7 +14,7 @@
#ImportSanitizedBridges 0
#
## Relative path to directory to import sanitized bridges from
-#SanitizedBridgesDirectory /srv/metrics.torproject.org/web/shared/in/bridge-descriptors/
+#SanitizedBridgesDirectory /srv/metrics.torproject.org/web/shared/in/recent/bridge-descriptors/
#
## Keep a history of imported sanitized bridge descriptors. This history
## can be useful when importing from a changing data source to avoid
@@ -43,5 +43,5 @@
#ImportWriteTorperfStats 0
#
## Relative path to directory to import torperf results from
-#TorperfDirectory /srv/metrics.torproject.org/web/shared/in/torperf/
+#TorperfDirectory /srv/metrics.torproject.org/web/shared/in/recent/torperf/
#
diff --git a/shared/bin/01-collectdescs.sh b/shared/bin/01-collectdescs.sh
new file mode 100755
index 0000000..18b900f
--- /dev/null
+++ b/shared/bin/01-collectdescs.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+cd modules/collectdescs/
+ant | grep "\[java\]"
+cd ../../
+
diff --git a/shared/bin/01-rsync-descriptors.sh b/shared/bin/01-rsync-descriptors.sh
deleted file mode 100755
index fa62c5e..0000000
--- a/shared/bin/01-rsync-descriptors.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh
-rsync -arz --delete --exclude 'relay-descriptors/votes' --exclude 'relay-descriptors/microdescs' metrics.torproject.org::metrics-recent shared/in
-
More information about the tor-commits
mailing list