commit 829863f48fb4e624f849df4c67bc970331014b0d Author: Karsten Loesing karsten.loesing@gmx.net Date: Thu Jan 24 10:08:02 2019 +0100
Run modules from Java only.
Implements #29166. --- build.xml | 136 +-------------------- .../web/images => R/rserver}/no-data-available.pdf | Bin .../web/images => R/rserver}/no-data-available.png | Bin .../web/images => R/rserver}/no-data-available.xcf | Bin src/main/R/rserver/rserve-init.R | 6 +- .../torproject/metrics/stats/advbwdist/Main.java | 15 ++- .../metrics/stats/bwhist/Configuration.java | 18 --- .../org/torproject/metrics/stats/bwhist/Main.java | 31 +++-- .../torproject/metrics/stats/clients/Database.java | 10 +- .../torproject/metrics/stats/clients/Detector.java | 7 +- .../org/torproject/metrics/stats/clients/Main.java | 37 +++--- .../metrics/stats/collectdescs/Main.java | 5 +- .../metrics/stats/connbidirect/Main.java | 15 ++- .../org/torproject/metrics/stats/hidserv/Main.java | 14 ++- .../org/torproject/metrics/stats/main/Main.java | 122 ++++++++++++++++++ .../torproject/metrics/stats/onionperf/Main.java | 26 ++-- .../metrics/stats/servers/Configuration.java | 18 --- .../torproject/metrics/stats/servers/Database.java | 10 +- .../org/torproject/metrics/stats/servers/Main.java | 45 +++---- .../metrics/stats/totalcw/Configuration.java | 18 --- .../torproject/metrics/stats/totalcw/Database.java | 10 +- .../org/torproject/metrics/stats/totalcw/Main.java | 29 +++-- .../torproject/metrics/stats/webstats/Main.java | 23 ++-- 23 files changed, 296 insertions(+), 299 deletions(-)
diff --git a/build.xml b/build.xml index 42965bf..254a71f 100644 --- a/build.xml +++ b/build.xml @@ -18,7 +18,8 @@ <property name="name" value="metrics-web"/>
<property name="project-main-class" - value="org.torproject.TBD" /> + value="org.torproject.metrics.stats.main.Main" /> + <property name="jarincludes" value="logback.xml" /> <property name="additional2sign" value="${warfile}" /> <property name="tardepends" value="war" />
@@ -26,11 +27,6 @@
<property name="specdir" value="${basedir}/generated/spec" />
- <!-- Deployment base folder. - Be aware that this is also set in R scripts and web.xml, currently! --> - <property name="metrics-web.deployment.base" - value="/srv/metrics.torproject.org/metrics" /> - <!-- The coverage needs to be improved! --> <target name="coverage-check"> <cobertura-check totallinerate="0" totalbranchrate="0" > @@ -301,85 +297,6 @@ <delete file="${specdir}/${specfile}.tmp2" quiet="true" /> </target>
- <!-- This can be adapted to point at the actual work directory. --> - <property name="prepare.deployment" - value="/srv/metrics.torproject.org/metrics/work" /> - - <!-- Don't alter the following. --> - <property name="modulebase" value="${prepare.deployment}/modules" /> - - <!-- Operational tasks. --> - <target name="run-web-prepare" depends="init" > - <mkdir dir="${prepare.deployment}/modules" /> - <antcall target="collectdescs" /> - <antcall target="connbidirect" /> - <antcall target="onionperf" /> - <antcall target="bwhist" /> - <antcall target="advbwdist" /> - <antcall target="hidserv" /> - <antcall target="clients" /> - <antcall target="servers" /> - <antcall target="webstats" /> - <antcall target="totalcw" /> - <antcall target="make-data-available" /> - </target> - - <target name="collectdescs" > - <property name="module.name" value="collectdescs" /> - <antcall target="run-java" /> - </target> - - <target name="connbidirect" > - <property name="module.name" value="connbidirect" /> - <antcall target="run-java" /> - </target> - - <target name="onionperf" > - <property name="module.name" value="onionperf" /> - <antcall target="run-java" /> - </target> - - <target name="bwhist" > - <property name="module.name" value="bwhist" /> - <antcall target="run-java" /> - </target> - - <target name="advbwdist"> - <property name="module.name" value="advbwdist" /> - <antcall target="run-java" /> - </target> - - <target name="hidserv" > - <property name="module.name" value="hidserv" /> - <antcall target="run-java" /> - </target> - - <target name="clients" > - <property name="module.name" value="clients" /> - <antcall target="run-java" /> - </target> - - <target name="servers" > - <property name="module.name" value="servers" /> - <antcall target="run-java" > - <param name="module.main" - value="org.torproject.metrics.stats.servers.Main" /> - </antcall> - </target> - - <target name="webstats" > - <property name="module.name" value="webstats" /> - <antcall target="run-java" /> - </target> - - <target name="totalcw" > - <property name="module.name" value="totalcw" /> - <antcall target="run-java" > - <param name="module.main" - value="org.torproject.metrics.stats.totalcw.Main" /> - </antcall> - </target> - <!-- The run-rserver target documents a working option of configuring an R server for running metrics-web. @@ -401,55 +318,6 @@ </exec> </target>
- <target name="make-data-available" > - <property name="statsdir" value="${metrics-web.deployment.base}/shared/stats/" /> - <mkdir dir="${statsdir}" /> - <property name="rdatadir" value="${metrics-web.deployment.base}/shared/RData" /> - <mkdir dir="${rdatadir}" /> - <copy todir="${statsdir}" > - <fileset dir="${modulebase}/onionperf/stats" includes="*.csv" /> - <fileset dir="${modulebase}/connbidirect/stats" includes="connbidirect2.csv" /> - <fileset dir="${modulebase}/advbwdist/stats" includes="advbwdist.csv" /> - <fileset dir="${modulebase}/bwhist/stats" includes="*.csv" /> - <fileset dir="${modulebase}/hidserv/stats" includes="hidserv.csv" /> - <fileset dir="${modulebase}/clients/stats" - includes="clients*.csv userstats-combined.csv" /> - <fileset dir="${modulebase}/servers/stats" includes="*.csv" /> - <fileset dir="${modulebase}/webstats/stats" includes="webstats.csv" /> - <fileset dir="${modulebase}/totalcw/stats" includes="totalcw.csv" /> - </copy> - <copy todir="${rdatadir}" > - <fileset dir="${resources}/web/images/" includes="no-data-available.*" /> - </copy> - </target> - - <!-- Support tasks for operation --> - <target name="run-java"> - <echo message="Running java module ${module.name} ... " /> - <available file="${dist}/${jarfile}" property="have.jar"/> - <fail unless="have.jar" message="Please run 'ant jar' first."/> - <condition property="mainclass" - value="${module.main}" - else="org.torproject.metrics.stats.${module.name}.Main" > - <isset property="module.main"/> - </condition> - <property name="workingdir" value="${modulebase}/${module.name}" /> - <mkdir dir="${workingdir}" /> - <java dir="${workingdir}" - fork="true" - maxmemory="4g" - classname="${mainclass}"> - <classpath> - <pathelement location="${dist}/${jarfile}"/> - <pathelement location="${resources}"/> - </classpath> - <jvmarg value="-DLOGBASE=../logs"/> - <jvmarg value="-Duser.language=us" /> - <jvmarg value="-Duser.region=US" /> - </java> - <echo message="Java module ${module.name} finished. " /> - </target> - <!-- The following line adds the common targets and properties for Metrics' Java Projects. --> diff --git a/src/main/resources/web/images/no-data-available.pdf b/src/main/R/rserver/no-data-available.pdf similarity index 100% rename from src/main/resources/web/images/no-data-available.pdf rename to src/main/R/rserver/no-data-available.pdf diff --git a/src/main/resources/web/images/no-data-available.png b/src/main/R/rserver/no-data-available.png similarity index 100% rename from src/main/resources/web/images/no-data-available.png rename to src/main/R/rserver/no-data-available.png diff --git a/src/main/resources/web/images/no-data-available.xcf b/src/main/R/rserver/no-data-available.xcf similarity index 100% rename from src/main/resources/web/images/no-data-available.xcf rename to src/main/R/rserver/no-data-available.xcf diff --git a/src/main/R/rserver/rserve-init.R b/src/main/R/rserver/rserve-init.R index c412e4c..5a47550 100644 --- a/src/main/R/rserver/rserve-init.R +++ b/src/main/R/rserver/rserve-init.R @@ -333,7 +333,7 @@ copyright_notice <- "The Tor Project - https://metrics.torproject.org/"
stats_dir <- "/srv/metrics.torproject.org/metrics/shared/stats/"
-rdata_dir <- "/srv/metrics.torproject.org/metrics/shared/RData/" +no_data_available_dir <- "/srv/metrics.torproject.org/metrics/src/main/R/rserver/"
# Helper function that copies the appropriate no data object to filename. copy_no_data <- function(filename) { @@ -342,8 +342,8 @@ copy_no_data <- function(filename) { if (".csv" == extension) { write("# No data available for the given parameters.", file=filename) } else { - file.copy(paste(rdata_dir, "no-data-available", extension, sep = ""), - filename) + file.copy(paste(no_data_available_dir, "no-data-available", extension, + sep = ""), filename) } }
diff --git a/src/main/java/org/torproject/metrics/stats/advbwdist/Main.java b/src/main/java/org/torproject/metrics/stats/advbwdist/Main.java index 6c4f4ac..3afcb80 100644 --- a/src/main/java/org/torproject/metrics/stats/advbwdist/Main.java +++ b/src/main/java/org/torproject/metrics/stats/advbwdist/Main.java @@ -32,6 +32,9 @@ import java.util.TreeMap;
public class Main {
+ private static final File baseDir = new File( + org.torproject.metrics.stats.main.Main.modulesDir, "advbwdist"); + /** Executes this data-processing module. */ public static void main(String[] args) throws IOException {
@@ -41,7 +44,8 @@ public class Main { DescriptorReader descriptorReader = DescriptorSourceFactory.createDescriptorReader(); for (Descriptor descriptor : descriptorReader.readDescriptors(new File( - "../../shared/in/recent/relay-descriptors/server-descriptors"))) { + org.torproject.metrics.stats.main.Main.descriptorsDir, + "recent/relay-descriptors/server-descriptors"))) { if (!(descriptor instanceof ServerDescriptor)) { continue; } @@ -56,9 +60,9 @@ public class Main {
/* Parse consensuses, keeping a parse history. */ descriptorReader = DescriptorSourceFactory.createDescriptorReader(); - File historyFile = new File("status/parsed-consensuses"); + File historyFile = new File(baseDir, "status/parsed-consensuses"); descriptorReader.setHistoryFile(historyFile); - File resultsFile = new File("stats/advbwdist-validafter.csv"); + File resultsFile = new File(baseDir, "stats/advbwdist-validafter.csv"); resultsFile.getParentFile().mkdirs(); boolean writeHeader = !resultsFile.exists(); BufferedWriter bw = new BufferedWriter(new FileWriter(resultsFile, @@ -70,7 +74,8 @@ public class Main { "yyyy-MM-dd HH:mm:ss"); dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); for (Descriptor descriptor : descriptorReader.readDescriptors(new File( - "../../shared/in/recent/relay-descriptors/consensuses"))) { + org.torproject.metrics.stats.main.Main.descriptorsDir, + "recent/relay-descriptors/consensuses"))) { if (!(descriptor instanceof RelayNetworkStatusConsensus)) { continue; } @@ -165,7 +170,7 @@ public class Main { preAggregatedValues.get(keyWithoutTime).add(value); } } - File aggregateResultsFile = new File("stats/advbwdist.csv"); + File aggregateResultsFile = new File(baseDir, "stats/advbwdist.csv"); aggregateResultsFile.getParentFile().mkdirs(); try (BufferedWriter bw2 = new BufferedWriter( new FileWriter(aggregateResultsFile))) { diff --git a/src/main/java/org/torproject/metrics/stats/bwhist/Configuration.java b/src/main/java/org/torproject/metrics/stats/bwhist/Configuration.java deleted file mode 100644 index 2a0fbc5..0000000 --- a/src/main/java/org/torproject/metrics/stats/bwhist/Configuration.java +++ /dev/null @@ -1,18 +0,0 @@ -/* Copyright 2011--2018 The Tor Project - * See LICENSE for licensing information */ - -package org.torproject.metrics.stats.bwhist; - -/** Configuration options parsed from Java properties with reasonable hard-coded - * defaults. */ -public class Configuration { - static String descriptors = System.getProperty("bwhist.descriptors", - "../../shared/in/"); - static String database = System.getProperty("bwhist.database", - "jdbc:postgresql:tordir"); - static String history = System.getProperty("bwhist.history", - "status/read-descriptors"); - static String output = System.getProperty("bwhist.output", - "stats/"); -} - diff --git a/src/main/java/org/torproject/metrics/stats/bwhist/Main.java b/src/main/java/org/torproject/metrics/stats/bwhist/Main.java index 61c1435..30358f2 100644 --- a/src/main/java/org/torproject/metrics/stats/bwhist/Main.java +++ b/src/main/java/org/torproject/metrics/stats/bwhist/Main.java @@ -7,7 +7,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory;
import java.io.File; -import java.nio.file.Paths; import java.util.Arrays;
/** @@ -18,11 +17,19 @@ public class Main {
private static Logger log = LoggerFactory.getLogger(Main.class);
- private static String[][] paths = { - {"recent", "relay-descriptors", "consensuses"}, - {"recent", "relay-descriptors", "extra-infos"}, - {"archive", "relay-descriptors", "consensuses"}, - {"archive", "relay-descriptors", "extra-infos"}}; + private static String[] paths = { + "recent/relay-descriptors/consensuses", + "recent/relay-descriptors/extra-infos", + "archive/relay-descriptors/consensuses", + "archive/relay-descriptors/extra-infos" }; + + private static final String jdbcString = String.format( + "jdbc:postgresql://localhost/tordir?user=%s&password=%s", + System.getProperty("metrics.dbuser", "metrics"), + System.getProperty("metrics.dbpass", "password")); + + private static final File baseDir = new File( + org.torproject.metrics.stats.main.Main.modulesDir, "bwhist");
/** Executes this data-processing module. */ public static void main(String[] args) throws Exception { @@ -31,20 +38,20 @@ public class Main {
log.info("Reading descriptors and inserting relevant parts into the " + "database."); - File[] descriptorDirectories = Arrays.stream(paths).map((String[] path) - -> Paths.get(Configuration.descriptors, path).toFile()) - .toArray(File[]::new); - File historyFile = new File(Configuration.history); + File[] descriptorDirectories = Arrays.stream(paths).map((String path) + -> new File(org.torproject.metrics.stats.main.Main.descriptorsDir, + path)).toArray(File[]::new); + File historyFile = new File(baseDir, "status/read-descriptors"); RelayDescriptorDatabaseImporter database = new RelayDescriptorDatabaseImporter(descriptorDirectories, - historyFile, Configuration.database); + historyFile, jdbcString); database.importRelayDescriptors();
log.info("Aggregating database entries."); database.aggregate();
log.info("Querying aggregated statistics from the database."); - new Writer().write(Paths.get(Configuration.output, "bandwidth.csv"), + new Writer().write(new File(baseDir, "stats/bandwidth.csv").toPath(), database.queryBandwidth());
log.info("Closing database connection."); diff --git a/src/main/java/org/torproject/metrics/stats/clients/Database.java b/src/main/java/org/torproject/metrics/stats/clients/Database.java index 7e783dc..f8bca92 100644 --- a/src/main/java/org/torproject/metrics/stats/clients/Database.java +++ b/src/main/java/org/torproject/metrics/stats/clients/Database.java @@ -22,7 +22,10 @@ import java.util.TimeZone; class Database implements AutoCloseable {
/** Database connection string. */ - private String jdbcString; + private static final String jdbcString = String.format( + "jdbc:postgresql://localhost/userstats?user=%s&password=%s", + System.getProperty("metrics.dbuser", "metrics"), + System.getProperty("metrics.dbpass", "password"));
/** Connection object for all interactions with the database. */ private Connection connection; @@ -33,14 +36,13 @@ class Database implements AutoCloseable {
/** Create a new Database instance and prepare for inserting or querying * data. */ - Database(String jdbcString) throws SQLException { - this.jdbcString = jdbcString; + Database() throws SQLException { this.connect(); this.prepareStatements(); }
private void connect() throws SQLException { - this.connection = DriverManager.getConnection(this.jdbcString); + this.connection = DriverManager.getConnection(jdbcString); this.connection.setAutoCommit(false); }
diff --git a/src/main/java/org/torproject/metrics/stats/clients/Detector.java b/src/main/java/org/torproject/metrics/stats/clients/Detector.java index 1a523c2..cceac5e 100644 --- a/src/main/java/org/torproject/metrics/stats/clients/Detector.java +++ b/src/main/java/org/torproject/metrics/stats/clients/Detector.java @@ -53,7 +53,6 @@ import java.io.FileWriter; import java.io.IOException; import java.io.LineNumberReader; import java.nio.file.Path; -import java.nio.file.Paths; import java.time.LocalDate; import java.time.format.DateTimeParseException; import java.util.ArrayList; @@ -70,10 +69,12 @@ import java.util.stream.Collectors; public class Detector {
/** Input file. */ - private static final Path INPUT_PATH = Paths.get("stats", "userstats.csv"); + private static final Path INPUT_PATH = new File(Main.baseDir, + "stats/userstats.csv").toPath();
/** Output file. */ - private static final Path OUTPUT_PATH = Paths.get("stats", "clients.csv"); + private static final Path OUTPUT_PATH = new File(Main.baseDir, + "stats/clients.csv").toPath();
/** Number of largest locations to be included in the detection algorithm. */ private static final int NUM_LARGEST_LOCATIONS = 50; diff --git a/src/main/java/org/torproject/metrics/stats/clients/Main.java b/src/main/java/org/torproject/metrics/stats/clients/Main.java index 0f1087b..2f22f72 100644 --- a/src/main/java/org/torproject/metrics/stats/clients/Main.java +++ b/src/main/java/org/torproject/metrics/stats/clients/Main.java @@ -16,7 +16,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory;
import java.io.File; -import java.nio.file.Paths; import java.sql.SQLException; import java.util.Map; import java.util.SortedMap; @@ -26,18 +25,18 @@ public class Main {
private static Logger log = LoggerFactory.getLogger(Main.class);
- private static final String jdbcString - = System.getProperty("clients.database", "jdbc:postgresql:userstats"); - private static Database database;
+ static final File baseDir = new File( + org.torproject.metrics.stats.main.Main.modulesDir, "clients"); + /** Executes this data-processing module. */ public static void main(String[] args) throws Exception {
log.info("Starting clients module.");
log.info("Connecting to database."); - database = new Database(jdbcString); + database = new Database();
log.info("Reading relay descriptors and importing relevant parts into the " + "database."); @@ -52,10 +51,10 @@ public class Main { database.commit();
log.info("Querying aggregated statistics from the database."); - new Writer().write(Paths.get("stats", "userstats.csv"), + new Writer().write(new File(baseDir, "stats/userstats.csv").toPath(), database.queryEstimated()); - new Writer().write(Paths.get("stats", "userstats-combined.csv"), - database.queryCombined()); + new Writer().write(new File(baseDir, "stats/userstats-combined.csv") + .toPath(), database.queryCombined());
log.info("Disconnecting from database."); database.close(); @@ -75,13 +74,17 @@ public class Main { private static void parseRelayDescriptors() throws Exception { DescriptorReader descriptorReader = DescriptorSourceFactory.createDescriptorReader(); - File historyFile = new File("status/relay-descriptors"); + File historyFile = new File(baseDir, "status/relay-descriptors"); descriptorReader.setHistoryFile(historyFile); for (Descriptor descriptor : descriptorReader.readDescriptors( - new File("../../shared/in/recent/relay-descriptors/consensuses"), - new File("../../shared/in/recent/relay-descriptors/extra-infos"), - new File("../../shared/in/archive/relay-descriptors/consensuses"), - new File("../../shared/in/archive/relay-descriptors/extra-infos"))) { + new File(org.torproject.metrics.stats.main.Main.descriptorsDir, + "recent/relay-descriptors/consensuses"), + new File(org.torproject.metrics.stats.main.Main.descriptorsDir, + "recent/relay-descriptors/extra-infos"), + new File(org.torproject.metrics.stats.main.Main.descriptorsDir, + "archive/relay-descriptors/consensuses"), + new File(org.torproject.metrics.stats.main.Main.descriptorsDir, + "archive/relay-descriptors/extra-infos"))) { if (descriptor instanceof ExtraInfoDescriptor) { parseRelayExtraInfoDescriptor((ExtraInfoDescriptor) descriptor); } else if (descriptor instanceof RelayNetworkStatusConsensus) { @@ -209,11 +212,13 @@ public class Main { private static void parseBridgeDescriptors() throws Exception { DescriptorReader descriptorReader = DescriptorSourceFactory.createDescriptorReader(); - File historyFile = new File("status/bridge-descriptors"); + File historyFile = new File(baseDir, "status/bridge-descriptors"); descriptorReader.setHistoryFile(historyFile); for (Descriptor descriptor : descriptorReader.readDescriptors( - new File("../../shared/in/recent/bridge-descriptors"), - new File("../../shared/in/archive/bridge-descriptors"))) { + new File(org.torproject.metrics.stats.main.Main.descriptorsDir, + "recent/bridge-descriptors"), + new File(org.torproject.metrics.stats.main.Main.descriptorsDir, + "archive/bridge-descriptors"))) { if (descriptor instanceof ExtraInfoDescriptor) { parseBridgeExtraInfoDescriptor( (ExtraInfoDescriptor) descriptor); diff --git a/src/main/java/org/torproject/metrics/stats/collectdescs/Main.java b/src/main/java/org/torproject/metrics/stats/collectdescs/Main.java index a9620f5..1d0d840 100644 --- a/src/main/java/org/torproject/metrics/stats/collectdescs/Main.java +++ b/src/main/java/org/torproject/metrics/stats/collectdescs/Main.java @@ -10,6 +10,9 @@ import java.io.File;
public class Main {
+ private static final File baseDir = new File( + org.torproject.metrics.stats.main.Main.modulesDir, "collectdescs"); + /** Executes this data-processing module. */ public static void main(String[] args) { /* Fetch recent descriptors from CollecTor. */ @@ -27,7 +30,7 @@ public class Main { "/recent/relay-descriptors/votes/", "/recent/torperf/", "/recent/webstats/" - }, 0L, new File("../../shared/in"), true); + }, 0L, org.torproject.metrics.stats.main.Main.descriptorsDir, true); } }
diff --git a/src/main/java/org/torproject/metrics/stats/connbidirect/Main.java b/src/main/java/org/torproject/metrics/stats/connbidirect/Main.java index 468ffba..c7fefee 100644 --- a/src/main/java/org/torproject/metrics/stats/connbidirect/Main.java +++ b/src/main/java/org/torproject/metrics/stats/connbidirect/Main.java @@ -129,13 +129,18 @@ public class Main {
static final long ONE_DAY_IN_MILLIS = 86400000L;
+ private static final File baseDir = new File( + org.torproject.metrics.stats.main.Main.modulesDir, "connbidirect"); + /** Executes this data-processing module. */ public static void main(String[] args) throws IOException { - File parseHistoryFile = new File("stats/parse-history"); - File aggregateStatsFile = new File("stats/connbidirect2.csv"); + File parseHistoryFile = new File(baseDir, "stats/parse-history"); + File aggregateStatsFile = new File(baseDir, "stats/connbidirect2.csv"); File[] descriptorsDirectories = new File[] { - new File("../../shared/in/archive/relay-descriptors/extra-infos"), - new File("../../shared/in/recent/relay-descriptors/extra-infos")}; + new File(org.torproject.metrics.stats.main.Main.descriptorsDir, + "archive/relay-descriptors/extra-infos"), + new File(org.torproject.metrics.stats.main.Main.descriptorsDir, + "recent/relay-descriptors/extra-infos")}; SortedMap<String, Long> parseHistory = parseParseHistory( readStringFromFile(parseHistoryFile)); if (parseHistory == null) { @@ -160,7 +165,7 @@ public class Main { + "leave out those descriptors in future runs."); return; } - File rawStatsFile = new File("stats/raw-stats"); + File rawStatsFile = new File(baseDir, "stats/raw-stats"); SortedSet<RawStat> rawStats = parseRawStats( readStringFromFile(rawStatsFile)); if (rawStats == null) { diff --git a/src/main/java/org/torproject/metrics/stats/hidserv/Main.java b/src/main/java/org/torproject/metrics/stats/hidserv/Main.java index b6a4e43..5076353 100644 --- a/src/main/java/org/torproject/metrics/stats/hidserv/Main.java +++ b/src/main/java/org/torproject/metrics/stats/hidserv/Main.java @@ -16,6 +16,9 @@ public class Main {
private static Logger log = LoggerFactory.getLogger(Main.class);
+ private static final File baseDir = new File( + org.torproject.metrics.stats.main.Main.modulesDir, "hidserv"); + /** Parses new descriptors, extrapolate contained statistics using * computed network fractions, aggregate results, and writes results to * disk. */ @@ -23,9 +26,11 @@ public class Main {
/* Initialize directories and file paths. */ File[] inDirectories = new File[] { - new File("../../shared/in/recent/relay-descriptors/consensuses"), - new File("../../shared/in/recent/relay-descriptors/extra-infos") }; - File statusDirectory = new File("status"); + new File(org.torproject.metrics.stats.main.Main.descriptorsDir, + "recent/relay-descriptors/consensuses"), + new File(org.torproject.metrics.stats.main.Main.descriptorsDir, + "recent/relay-descriptors/extra-infos") }; + File statusDirectory = new File(baseDir, "status");
/* Initialize parser and read parse history to avoid parsing * descriptor files that haven't changed since the last execution. */ @@ -71,7 +76,8 @@ public class Main { * other statistics. Write the result to a .csv file that can be * processed by other tools. */ log.info("Aggregating statistics..."); - File hidservStatsExtrapolatedCsvFile = new File("stats/hidserv.csv"); + File hidservStatsExtrapolatedCsvFile = new File(baseDir, + "stats/hidserv.csv"); Aggregator aggregator = new Aggregator(statusDirectory, extrapolatedHidServStatsStore, hidservStatsExtrapolatedCsvFile); aggregator.aggregateHidServStats(); diff --git a/src/main/java/org/torproject/metrics/stats/main/Main.java b/src/main/java/org/torproject/metrics/stats/main/Main.java new file mode 100644 index 0000000..3dc2775 --- /dev/null +++ b/src/main/java/org/torproject/metrics/stats/main/Main.java @@ -0,0 +1,122 @@ +package org.torproject.metrics.stats.main; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.List; + +public class Main { + + private static final Logger log = LoggerFactory.getLogger(Main.class); + + private static final String baseDir = System.getProperty("metrics.basedir", + "/srv/metrics.torproject.org/metrics"); + + public static final File modulesDir = new File(baseDir, "work/modules"); + + public static final File descriptorsDir = new File(baseDir, "work/shared/in"); + + private static final File statsDir = new File(baseDir, "shared/stats"); + + /** Start the metrics update run. */ + public static void main(String[] args) { + + log.info("Starting metrics update run."); + + File[] outputDirs = new File[] { modulesDir, statsDir }; + for (File outputDir : outputDirs) { + if (outputDir.exists()) { + continue; + } + if (outputDir.mkdirs()) { + log.info("Successfully created module base directory {} and any " + + "nonexistent parent directories.", + outputDir.getAbsolutePath()); + } else { + log.error("Unable to create module base directory {} and any " + + "nonexistent parent directories. Exiting.", + outputDir.getAbsolutePath()); + return; + } + } + + Class<?>[] modules = new Class<?>[] { + org.torproject.metrics.stats.collectdescs.Main.class, + org.torproject.metrics.stats.connbidirect.Main.class, + org.torproject.metrics.stats.onionperf.Main.class, + org.torproject.metrics.stats.bwhist.Main.class, + org.torproject.metrics.stats.advbwdist.Main.class, + org.torproject.metrics.stats.hidserv.Main.class, + org.torproject.metrics.stats.clients.Main.class, + org.torproject.metrics.stats.servers.Main.class, + org.torproject.metrics.stats.webstats.Main.class, + org.torproject.metrics.stats.totalcw.Main.class + }; + + for (Class<?> module : modules) { + try { + log.info("Starting {} module.", module.getName()); + module.getDeclaredMethod("main", String[].class) + .invoke(null, (Object) args); + log.info("Completed {} module.", module.getName()); + } catch (NoSuchMethodException | IllegalAccessException + | InvocationTargetException e) { + log.warn("Caught an exception when invoking the main method of the {} " + + "module. Moving on to the next module, if available.", e); + } + } + + log.info("Making module data available."); + File[] moduleStatsDirs = new File[] { + new File(modulesDir, "connbidirect/stats"), + new File(modulesDir, "onionperf/stats"), + new File(modulesDir, "bwhist/stats"), + new File(modulesDir, "advbwdist/stats/advbwdist.csv"), + new File(modulesDir, "hidserv/stats"), + new File(modulesDir, "clients/stats/clients.csv"), + new File(modulesDir, "clients/stats/userstats-combined.csv"), + new File(modulesDir, "servers/stats"), + new File(modulesDir, "webstats/stats"), + new File(modulesDir, "totalcw/stats") + }; + List<String> copiedFiles = new ArrayList<>(); + for (File moduleStatsDir : moduleStatsDirs) { + if (!moduleStatsDir.exists()) { + log.warn("Skipping nonexistent module stats dir {}.", moduleStatsDir); + continue; + } + File[] moduleStatsFiles = moduleStatsDir.isDirectory() + ? moduleStatsDir.listFiles() : new File[] { moduleStatsDir }; + if (null == moduleStatsFiles) { + log.warn("Skipping nonexistent module stats dir {}.", moduleStatsDir); + continue; + } + for (File statsFile : moduleStatsFiles) { + if (!statsFile.isFile() || !statsFile.getName().endsWith(".csv")) { + continue; + } + try { + Files.copy(statsFile.toPath(), + new File(statsDir, statsFile.getName()).toPath(), + StandardCopyOption.REPLACE_EXISTING); + copiedFiles.add(statsFile.getName()); + } catch (IOException e) { + log.warn("Unable to copy module stats file {} to stats output " + + "directory {}. Skipping.", statsFile, statsDir, e); + } + } + } + if (!copiedFiles.isEmpty()) { + log.info("Successfully copied {} files to stats output directory: {}", + copiedFiles.size(), copiedFiles); + } + + log.info("Completed metrics update run."); + } +} diff --git a/src/main/java/org/torproject/metrics/stats/onionperf/Main.java b/src/main/java/org/torproject/metrics/stats/onionperf/Main.java index 02b70af..a970434 100644 --- a/src/main/java/org/torproject/metrics/stats/onionperf/Main.java +++ b/src/main/java/org/torproject/metrics/stats/onionperf/Main.java @@ -13,7 +13,6 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; @@ -37,23 +36,30 @@ public class Main { /** Logger for this class. */ private static Logger log = LoggerFactory.getLogger(Main.class);
+ private static final String jdbcString = String.format( + "jdbc:postgresql://localhost/onionperf?user=%s&password=%s", + System.getProperty("metrics.dbuser", "metrics"), + System.getProperty("metrics.dbpass", "password")); + + private static final File baseDir = new File( + org.torproject.metrics.stats.main.Main.modulesDir, "onionperf"); + /** Executes this data-processing module. */ public static void main(String[] args) throws Exception { log.info("Starting onionperf module."); - String dbUrlString = "jdbc:postgresql:onionperf"; - Connection connection = connectToDatabase(dbUrlString); + Connection connection = connectToDatabase(); importOnionPerfFiles(connection); - writeStatistics(Paths.get("stats", "torperf-1.1.csv"), + writeStatistics(new File(baseDir, "stats/torperf-1.1.csv").toPath(), queryOnionPerf(connection)); - writeStatistics(Paths.get("stats", "buildtimes.csv"), + writeStatistics(new File(baseDir, "stats/buildtimes.csv").toPath(), queryBuildTimes(connection)); - writeStatistics(Paths.get("stats", "latencies.csv"), + writeStatistics(new File(baseDir, "stats/latencies.csv").toPath(), queryLatencies(connection)); disconnectFromDatabase(connection); log.info("Terminated onionperf module."); }
- private static Connection connectToDatabase(String jdbcString) + private static Connection connectToDatabase() throws SQLException { log.info("Connecting to database."); Connection connection = DriverManager.getConnection(jdbcString); @@ -91,8 +97,10 @@ public class Main { Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); DescriptorReader dr = DescriptorSourceFactory.createDescriptorReader(); for (Descriptor d : dr.readDescriptors( - new File("../../shared/in/archive/torperf"), - new File("../../shared/in/recent/torperf"))) { + new File(org.torproject.metrics.stats.main.Main.descriptorsDir, + "archive/torperf"), + new File(org.torproject.metrics.stats.main.Main.descriptorsDir, + "recent/torperf"))) { if (!(d instanceof TorperfResult)) { continue; } diff --git a/src/main/java/org/torproject/metrics/stats/servers/Configuration.java b/src/main/java/org/torproject/metrics/stats/servers/Configuration.java deleted file mode 100644 index 1b7c217..0000000 --- a/src/main/java/org/torproject/metrics/stats/servers/Configuration.java +++ /dev/null @@ -1,18 +0,0 @@ -/* Copyright 2017--2018 The Tor Project - * See LICENSE for licensing information */ - -package org.torproject.metrics.stats.servers; - -/** Configuration options parsed from Java properties with reasonable hard-coded - * defaults. */ -class Configuration { - static String descriptors = System.getProperty("servers.descriptors", - "../../shared/in/"); - static String database = System.getProperty("servers.database", - "jdbc:postgresql:ipv6servers"); - static String history = System.getProperty("servers.history", - "status/read-descriptors"); - static String output = System.getProperty("servers.output", - "stats/"); -} - diff --git a/src/main/java/org/torproject/metrics/stats/servers/Database.java b/src/main/java/org/torproject/metrics/stats/servers/Database.java index 9c9bda3..f7f1f0f 100644 --- a/src/main/java/org/torproject/metrics/stats/servers/Database.java +++ b/src/main/java/org/torproject/metrics/stats/servers/Database.java @@ -29,7 +29,10 @@ import java.util.TimeZone; class Database implements AutoCloseable {
/** Database connection string. */ - private String jdbcString; + private static final String jdbcString = String.format( + "jdbc:postgresql://localhost/ipv6servers?user=%s&password=%s", + System.getProperty("metrics.dbuser", "metrics"), + System.getProperty("metrics.dbpass", "password"));
/** Connection object for all interactions with the database. */ private Connection connection; @@ -87,15 +90,14 @@ class Database implements AutoCloseable {
/** Create a new Database instance and prepare for inserting or querying * data. */ - Database(String jdbcString) throws SQLException { - this.jdbcString = jdbcString; + Database() throws SQLException { this.connect(); this.prepareStatements(); this.initializeCaches(); }
private void connect() throws SQLException { - this.connection = DriverManager.getConnection(this.jdbcString); + this.connection = DriverManager.getConnection(jdbcString); this.connection.setAutoCommit(false); }
diff --git a/src/main/java/org/torproject/metrics/stats/servers/Main.java b/src/main/java/org/torproject/metrics/stats/servers/Main.java index 54f44d0..30010a2 100644 --- a/src/main/java/org/torproject/metrics/stats/servers/Main.java +++ b/src/main/java/org/torproject/metrics/stats/servers/Main.java @@ -14,7 +14,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory;
import java.io.File; -import java.nio.file.Paths; import java.sql.SQLException; import java.util.Arrays;
@@ -25,15 +24,18 @@ public class Main {
private static Logger log = LoggerFactory.getLogger(Main.class);
- private static String[][] paths = { - {"recent", "relay-descriptors", "consensuses"}, - {"recent", "relay-descriptors", "server-descriptors"}, - {"recent", "bridge-descriptors", "statuses"}, - {"recent", "bridge-descriptors", "server-descriptors"}, - {"archive", "relay-descriptors", "consensuses"}, - {"archive", "relay-descriptors", "server-descriptors"}, - {"archive", "bridge-descriptors", "statuses"}, - {"archive", "bridge-descriptors", "server-descriptors"}}; + private static final File baseDir = new File( + org.torproject.metrics.stats.main.Main.modulesDir, "servers"); + + private static String[] paths = { + "recent/relay-descriptors/consensuses", + "recent/relay-descriptors/server-descriptors", + "recent/bridge-descriptors/statuses", + "recent/bridge-descriptors/server-descriptors", + "archive/relay-descriptors/consensuses", + "archive/relay-descriptors/server-descriptors", + "archive/bridge-descriptors/statuses", + "archive/bridge-descriptors/server-descriptors" };
/** Run the module. */ public static void main(String[] args) throws Exception { @@ -43,15 +45,15 @@ public class Main { log.info("Reading descriptors and inserting relevant parts into the " + "database."); DescriptorReader reader = DescriptorSourceFactory.createDescriptorReader(); - File historyFile = new File(Configuration.history); + File historyFile = new File(baseDir, "status/read-descriptors"); reader.setHistoryFile(historyFile); Parser parser = new Parser(); - try (Database database = new Database(Configuration.database)) { + try (Database database = new Database()) { try { for (Descriptor descriptor : reader.readDescriptors( - Arrays.stream(paths).map((String[] path) - -> Paths.get(Configuration.descriptors, path).toFile()) - .toArray(File[]::new))) { + Arrays.stream(paths).map((String path) -> new File( + org.torproject.metrics.stats.main.Main.descriptorsDir, path)) + .toArray(File[]::new))) { if (descriptor instanceof ServerDescriptor) { database.insertServerDescriptor(parser.parseServerDescriptor( (ServerDescriptor) descriptor)); @@ -86,17 +88,18 @@ public class Main { reader.saveHistoryFile(historyFile);
log.info("Querying aggregated statistics from the database."); - new Writer().write(Paths.get(Configuration.output, "ipv6servers.csv"), + File outputDir = new File(baseDir, "stats"); + new Writer().write(new File(outputDir, "ipv6servers.csv").toPath(), database.queryServersIpv6()); - new Writer().write(Paths.get(Configuration.output, "advbw.csv"), + new Writer().write(new File(outputDir, "advbw.csv").toPath(), database.queryAdvbw()); - new Writer().write(Paths.get(Configuration.output, "networksize.csv"), + new Writer().write(new File(outputDir, "networksize.csv").toPath(), database.queryNetworksize()); - new Writer().write(Paths.get(Configuration.output, "relayflags.csv"), + new Writer().write(new File(outputDir, "relayflags.csv").toPath(), database.queryRelayflags()); - new Writer().write(Paths.get(Configuration.output, "versions.csv"), + new Writer().write(new File(outputDir, "versions.csv").toPath(), database.queryVersions()); - new Writer().write(Paths.get(Configuration.output, "platforms.csv"), + new Writer().write(new File(outputDir, "platforms.csv").toPath(), database.queryPlatforms());
log.info("Terminating servers module."); diff --git a/src/main/java/org/torproject/metrics/stats/totalcw/Configuration.java b/src/main/java/org/torproject/metrics/stats/totalcw/Configuration.java deleted file mode 100644 index a424f84..0000000 --- a/src/main/java/org/torproject/metrics/stats/totalcw/Configuration.java +++ /dev/null @@ -1,18 +0,0 @@ -/* Copyright 2018 The Tor Project - * See LICENSE for licensing information */ - -package org.torproject.metrics.stats.totalcw; - -/** Configuration options parsed from Java properties with reasonable hard-coded - * defaults. */ -class Configuration { - static String descriptors = System.getProperty("totalcw.descriptors", - "../../shared/in/"); - static String database = System.getProperty("totalcw.database", - "jdbc:postgresql:totalcw"); - static String history = System.getProperty("totalcw.history", - "status/read-descriptors"); - static String output = System.getProperty("totalcw.output", - "stats/totalcw.csv"); -} - diff --git a/src/main/java/org/torproject/metrics/stats/totalcw/Database.java b/src/main/java/org/torproject/metrics/stats/totalcw/Database.java index e842bc6..eeb24a0 100644 --- a/src/main/java/org/torproject/metrics/stats/totalcw/Database.java +++ b/src/main/java/org/torproject/metrics/stats/totalcw/Database.java @@ -25,7 +25,10 @@ import java.util.TimeZone; class Database implements AutoCloseable {
/** Database connection string. */ - private String jdbcString; + private static final String jdbcString = String.format( + "jdbc:postgresql://localhost/totalcw?user=%s&password=%s", + System.getProperty("metrics.dbuser", "metrics"), + System.getProperty("metrics.dbpass", "password"));
/** Connection object for all interactions with the database. */ private Connection connection; @@ -46,14 +49,13 @@ class Database implements AutoCloseable {
/** Create a new Database instance and prepare for inserting or querying * data. */ - Database(String jdbcString) throws SQLException { - this.jdbcString = jdbcString; + Database() throws SQLException { this.connect(); this.prepareStatements(); }
private void connect() throws SQLException { - this.connection = DriverManager.getConnection(this.jdbcString); + this.connection = DriverManager.getConnection(jdbcString); this.connection.setAutoCommit(false); }
diff --git a/src/main/java/org/torproject/metrics/stats/totalcw/Main.java b/src/main/java/org/torproject/metrics/stats/totalcw/Main.java index e5ba8ab..54296be 100644 --- a/src/main/java/org/torproject/metrics/stats/totalcw/Main.java +++ b/src/main/java/org/torproject/metrics/stats/totalcw/Main.java @@ -13,7 +13,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory;
import java.io.File; -import java.nio.file.Paths; import java.sql.SQLException; import java.util.Arrays;
@@ -24,11 +23,14 @@ public class Main {
private static Logger log = LoggerFactory.getLogger(Main.class);
- private static String[][] paths = { - {"recent", "relay-descriptors", "consensuses"}, - {"archive", "relay-descriptors", "consensuses"}, - {"recent", "relay-descriptors", "votes"}, - {"archive", "relay-descriptors", "votes"}}; + private static final File baseDir = new File( + org.torproject.metrics.stats.main.Main.modulesDir, "totalcw"); + + private static String[] paths = { + "recent/relay-descriptors/consensuses", + "archive/relay-descriptors/consensuses", + "recent/relay-descriptors/votes", + "archive/relay-descriptors/votes" };
/** Run the module. */ public static void main(String[] args) throws Exception { @@ -38,15 +40,15 @@ public class Main { log.info("Reading consensuses and votes and inserting relevant parts into " + "the database."); DescriptorReader reader = DescriptorSourceFactory.createDescriptorReader(); - File historyFile = new File(Configuration.history); + File historyFile = new File(baseDir, "status/read-descriptors"); reader.setHistoryFile(historyFile); Parser parser = new Parser(); - try (Database database = new Database(Configuration.database)) { + try (Database database = new Database()) { try { for (Descriptor descriptor : reader.readDescriptors( - Arrays.stream(paths).map((String[] path) - -> Paths.get(Configuration.descriptors, path).toFile()) - .toArray(File[]::new))) { + Arrays.stream(paths).map((String path) -> new File( + org.torproject.metrics.stats.main.Main.descriptorsDir, path)) + .toArray(File[]::new))) { if (descriptor instanceof RelayNetworkStatusConsensus) { database.insertConsensus(parser.parseRelayNetworkStatusConsensus( (RelayNetworkStatusConsensus) descriptor)); @@ -71,9 +73,10 @@ public class Main {
log.info("Querying aggregated statistics from the database."); Iterable<OutputLine> output = database.queryTotalcw(); - log.info("Writing aggregated statistics to {}.", Configuration.output); + File outputFile = new File(baseDir, "stats/totalcw.csv"); + log.info("Writing aggregated statistics to {}.", outputFile); if (null != output) { - new Writer().write(Paths.get(Configuration.output), output); + new Writer().write(outputFile.toPath(), output); }
log.info("Terminating totalcw module."); diff --git a/src/main/java/org/torproject/metrics/stats/webstats/Main.java b/src/main/java/org/torproject/metrics/stats/webstats/Main.java index fb0a903..87b643a 100644 --- a/src/main/java/org/torproject/metrics/stats/webstats/Main.java +++ b/src/main/java/org/torproject/metrics/stats/webstats/Main.java @@ -19,7 +19,6 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.sql.Connection; import java.sql.Date; import java.sql.DriverManager; @@ -48,6 +47,11 @@ public class Main { /** Logger for this class. */ private static Logger log = LoggerFactory.getLogger(Main.class);
+ private static final String jdbcString = String.format( + "jdbc:postgresql://localhost/webstats?user=%s&password=%s", + System.getProperty("metrics.dbuser", "metrics"), + System.getProperty("metrics.dbpass", "password")); + private static final String LOG_DATE = "log_date";
private static final String REQUEST_TYPE = "request_type"; @@ -66,22 +70,27 @@ public class Main { + PLATFORM + "," + CHANNEL + "," + LOCALE + "," + INCREMENTAL + "," + COUNT;
+ private static final File baseDir = new File( + org.torproject.metrics.stats.main.Main.modulesDir, "webstats"); + /** Executes this data-processing module. */ public static void main(String[] args) throws Exception { log.info("Starting webstats module."); - String dbUrlString = "jdbc:postgresql:webstats"; - Connection connection = connectToDatabase(dbUrlString); + Connection connection = connectToDatabase(); SortedSet<String> skipFiles = queryImportedFileNames(connection); importLogFiles(connection, skipFiles, - new File("../../shared/in/recent/webstats"), - new File("../../shared/in/archive/webstats")); + new File(org.torproject.metrics.stats.main.Main.descriptorsDir, + "recent/webstats"), + new File(org.torproject.metrics.stats.main.Main.descriptorsDir, + "archive/webstats")); SortedSet<String> statistics = queryWebstats(connection); - writeStatistics(Paths.get("stats", "webstats.csv"), statistics); + writeStatistics(new File(baseDir, "stats/webstats.csv").toPath(), + statistics); disconnectFromDatabase(connection); log.info("Terminated webstats module."); }
- private static Connection connectToDatabase(String jdbcString) + private static Connection connectToDatabase() throws SQLException { log.info("Connecting to database."); Connection connection = DriverManager.getConnection(jdbcString);