[tor-commits] [metrics-db/master] Download and merge possibly truncated Torperf files.
karsten at torproject.org
karsten at torproject.org
Tue Mar 6 10:54:35 UTC 2012
commit 138b3397ec300ba64ac48a13175435f7673d65fd
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Tue Mar 6 11:53:39 2012 +0100
Download and merge possibly truncated Torperf files.
---
.gitignore | 3 +
config.template | 21 ++++
src/org/torproject/ernie/db/Configuration.java | 58 ++++++++++-
src/org/torproject/ernie/db/Main.java | 7 +
src/org/torproject/ernie/db/TorperfDownloader.java | 116 ++++++++++++++++++++
5 files changed, 204 insertions(+), 1 deletions(-)
diff --git a/.gitignore b/.gitignore
index d5a5c14..e119284 100644
--- a/.gitignore
+++ b/.gitignore
@@ -47,6 +47,9 @@ config
# Gettor files
gettor/
+# Torperf files
+torperf/
+
# Files provided via rsync
rsync/
diff --git a/config.template b/config.template
index 6cfdce5..62ff828 100644
--- a/config.template
+++ b/config.template
@@ -122,4 +122,25 @@
#
## Relative path to the directory that shall be provided via rsync
#RsyncDirectory rsync/
+#
+#
+######## Torperf downloader ########
+#
+## Download and merge Torperf .data and .extradata files
+#ProcessTorperfFiles 0
+#
+## Relative path to the directory to store Torperf files in
+#TorperfOutputDirectory torperf/
+#
+## Torperf source names and base URLs (option can be contained multiple
+## times)
+#TorperfSource torperf http://torperf.torproject.org/
+#
+## Torperf .data files available on a given source (option can be
+## contained multiple times)
+#TorperfDataFiles torperf 50kb.data 1mb.data 5mb.data
+#
+## Torperf .extradata files available on a given source (option can be
+## contained multiple times)
+#TorperfExtradataFiles torperf 50kb.extradata 1mb.extradata 5mb.extradata
diff --git a/src/org/torproject/ernie/db/Configuration.java b/src/org/torproject/ernie/db/Configuration.java
index 7197c7d..91f00b1 100644
--- a/src/org/torproject/ernie/db/Configuration.java
+++ b/src/org/torproject/ernie/db/Configuration.java
@@ -49,6 +49,11 @@ public class Configuration {
private boolean processBridgePoolAssignments = false;
private String assignmentsDirectory = "assignments/";
private String sanitizedAssignmentsDirectory = "sanitized-assignments/";
+ private boolean processTorperfFiles = false;
+ private String torperfOutputDirectory = "torperf/";
+ private SortedMap<String, String> torperfSources = null;
+ private SortedMap<String, List<String>> torperfDataFiles = null;
+ private SortedMap<String, List<String>> torperfExtradataFiles = null;
private boolean provideFilesViaRsync = false;
private String rsyncDirectory = "rsync";
public Configuration() {
@@ -170,6 +175,41 @@ public class Configuration {
this.assignmentsDirectory = line.split(" ")[1];
} else if (line.startsWith("SanitizedAssignmentsDirectory")) {
this.sanitizedAssignmentsDirectory = line.split(" ")[1];
+ } else if (line.startsWith("ProcessTorperfFiles")) {
+ this.processTorperfFiles = Integer.parseInt(line.split(" ")[1])
+ != 0;
+ } else if (line.startsWith("TorperfOutputDirectory")) {
+ } else if (line.startsWith("TorperfSource")) {
+ if (this.torperfSources == null) {
+ this.torperfSources = new TreeMap<String, String>();
+ }
+ String[] parts = line.split(" ");
+ String sourceName = parts[1];
+ String baseUrl = parts[2];
+ this.torperfSources.put(sourceName, baseUrl);
+ } else if (line.startsWith("TorperfDataFiles")) {
+ if (this.torperfDataFiles == null) {
+ this.torperfDataFiles = new TreeMap<String, List<String>>();
+ }
+ String[] parts = line.split(" ");
+ String sourceName = parts[1];
+ List<String> dataFiles = new ArrayList<String>();
+ for (int i = 2; i < parts.length; i++) {
+ dataFiles.add(parts[i]);
+ }
+ this.torperfDataFiles.put(sourceName, dataFiles);
+ } else if (line.startsWith("TorperfExtradataFiles")) {
+ if (this.torperfExtradataFiles == null) {
+ this.torperfExtradataFiles =
+ new TreeMap<String, List<String>>();
+ }
+ String[] parts = line.split(" ");
+ String sourceName = parts[1];
+ List<String> extradataFiles = new ArrayList<String>();
+ for (int i = 2; i < parts.length; i++) {
+ extradataFiles.add(parts[i]);
+ }
+ this.torperfExtradataFiles.put(sourceName, extradataFiles);
} else if (line.startsWith("ProvideFilesViaRsync")) {
this.provideFilesViaRsync = Integer.parseInt(
line.split(" ")[1]) != 0;
@@ -205,7 +245,8 @@ public class Configuration {
!this.importDirectoryArchives && !this.downloadRelayDescriptors &&
!this.importBridgeSnapshots && !this.downloadGetTorStats &&
!this.downloadExitList && !this.processBridgePoolAssignments &&
- !this.writeDirectoryArchives && !this.writeSanitizedBridges) {
+ !this.writeDirectoryArchives && !this.writeSanitizedBridges &&
+ !this.processTorperfFiles) {
logger.warning("We have not been configured to read data from any "
+ "data source or write data to any data sink. You need to "
+ "edit your config file (" + configFile.getAbsolutePath()
@@ -324,6 +365,21 @@ public class Configuration {
public String getSanitizedAssignmentsDirectory() {
return sanitizedAssignmentsDirectory;
}
+ public boolean getProcessTorperfFiles() {
+ return this.processTorperfFiles;
+ }
+ public String getTorperfOutputDirectory() {
+ return this.torperfOutputDirectory;
+ }
+ public SortedMap<String, String> getTorperfSources() {
+ return this.torperfSources;
+ }
+ public SortedMap<String, List<String>> getTorperfDataFiles() {
+ return this.torperfDataFiles;
+ }
+ public SortedMap<String, List<String>> getTorperfExtradataFiles() {
+ return this.torperfExtradataFiles;
+ }
public boolean getProvideFilesViaRsync() {
return this.provideFilesViaRsync;
}
diff --git a/src/org/torproject/ernie/db/Main.java b/src/org/torproject/ernie/db/Main.java
index ab79174..a5b5080 100644
--- a/src/org/torproject/ernie/db/Main.java
+++ b/src/org/torproject/ernie/db/Main.java
@@ -136,6 +136,13 @@ public class Main {
new File(config.getSanitizedAssignmentsDirectory()));
}
+ // Process Torperf files
+ if (config.getProcessTorperfFiles()) {
+ new TorperfDownloader(new File(config.getTorperfOutputDirectory()),
+ config.getTorperfSources(), config.getTorperfDataFiles(),
+ config.getTorperfExtradataFiles());
+ }
+
// Copy recently published files to a local directory that can then
// be served via rsync.
if (config.getProvideFilesViaRsync()) {
diff --git a/src/org/torproject/ernie/db/TorperfDownloader.java b/src/org/torproject/ernie/db/TorperfDownloader.java
new file mode 100644
index 0000000..95d2e30
--- /dev/null
+++ b/src/org/torproject/ernie/db/TorperfDownloader.java
@@ -0,0 +1,116 @@
+/* Copyright 2012 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.ernie.db;
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import java.util.logging.*;
+
+/* Download possibly truncated Torperf .data and .extradata files from
+ * configured sources and append them to the files we already have. */
+public class TorperfDownloader {
+
+ private File torperfOutputDirectory = null;
+ private SortedMap<String, String> torperfSources = null;
+ private SortedMap<String, List<String>> torperfDataFiles = null;
+ private SortedMap<String, List<String>> torperfExtradataFiles = null;
+ private Logger logger = null;
+
+ public TorperfDownloader(File torperfOutputDirectory,
+ SortedMap<String, String> torperfSources,
+ SortedMap<String, List<String>> torperfDataFiles,
+ SortedMap<String, List<String>> torperfExtradataFiles) {
+ if (torperfOutputDirectory == null) {
+ throw new IllegalArgumentException();
+ }
+ this.torperfOutputDirectory = torperfOutputDirectory;
+ this.torperfSources = torperfSources;
+ this.torperfDataFiles = torperfDataFiles;
+ this.torperfExtradataFiles = torperfExtradataFiles;
+ if (!this.torperfOutputDirectory.exists()) {
+ this.torperfOutputDirectory.mkdirs();
+ }
+ this.logger = Logger.getLogger(TorperfDownloader.class.getName());
+ this.downloadAndMergeFiles(this.torperfDataFiles, true);
+ this.downloadAndMergeFiles(this.torperfExtradataFiles, false);
+ }
+
+ private void downloadAndMergeFiles(
+ SortedMap<String, List<String>> dataOrExtradataFiles,
+ boolean isDataFile) {
+ for (Map.Entry<String, List<String>> e :
+ dataOrExtradataFiles.entrySet()) {
+ String sourceName = e.getKey();
+ String sourceBaseUrl = torperfSources.get(sourceName);
+ List<String> files = e.getValue();
+ for (String file : files) {
+ String url = sourceBaseUrl + file;
+ File outputFile = new File(torperfOutputDirectory,
+ sourceName + "-" + file);
+ this.downloadAndMergeFile(url, outputFile, isDataFile);
+ }
+ }
+ }
+
+ private void downloadAndMergeFile(String url, File outputFile,
+ boolean isDataFile) {
+ String lastTimestampLine = null;
+ int linesAfterLastTimestampLine = 0;
+ if (outputFile.exists() && outputFile.lastModified() >
+ System.currentTimeMillis() - 6L * 60L * 60L * 1000L) {
+ return;
+ } else if (outputFile.exists()) {
+ try {
+ BufferedReader br = new BufferedReader(new FileReader(
+ outputFile));
+ String line;
+ while ((line = br.readLine()) != null) {
+ if (isDataFile || line.contains(" LAUNCH")) {
+ lastTimestampLine = line;
+ linesAfterLastTimestampLine = 0;
+ } else {
+ linesAfterLastTimestampLine++;
+ }
+ }
+ br.close();
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Failed reading '"
+ + outputFile.getAbsolutePath() + "' to find the last line to "
+ + "append to.", e);
+ return;
+ }
+ }
+ try {
+ this.logger.fine("Downloading " + (isDataFile ? ".data" :
+ ".extradata") + " file from '" + url + "' and merging it into "
+ + "'" + outputFile.getAbsolutePath() + "'.");
+ URL u = new URL(url);
+ HttpURLConnection huc = (HttpURLConnection) u.openConnection();
+ huc.setRequestMethod("GET");
+ huc.connect();
+ BufferedReader br = new BufferedReader(new InputStreamReader(
+ huc.getInputStream()));
+ String line;
+ BufferedWriter bw = new BufferedWriter(new FileWriter(outputFile,
+ true));
+ boolean copyLines = lastTimestampLine == null;
+ while ((line = br.readLine()) != null) {
+ if (copyLines && linesAfterLastTimestampLine == 0) {
+ bw.write(line + "\n");
+ } else if (copyLines && linesAfterLastTimestampLine > 0) {
+ linesAfterLastTimestampLine--;
+ } else if (line.equals(lastTimestampLine)) {
+ copyLines = true;
+ }
+ }
+ bw.close();
+ br.close();
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Failed downloading and merging '" + url
+ + "'.", e);
+ return;
+ }
+ }
+}
+
More information about the tor-commits
mailing list