commit 138b3397ec300ba64ac48a13175435f7673d65fd Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Tue Mar 6 11:53:39 2012 +0100 Download and merge possibly truncated Torperf files. --- .gitignore | 3 + config.template | 21 ++++ src/org/torproject/ernie/db/Configuration.java | 58 ++++++++++- src/org/torproject/ernie/db/Main.java | 7 + src/org/torproject/ernie/db/TorperfDownloader.java | 116 ++++++++++++++++++++ 5 files changed, 204 insertions(+), 1 deletions(-) diff --git a/.gitignore b/.gitignore index d5a5c14..e119284 100644 --- a/.gitignore +++ b/.gitignore @@ -47,6 +47,9 @@ config # Gettor files gettor/ +# Torperf files +torperf/ + # Files provided via rsync rsync/ diff --git a/config.template b/config.template index 6cfdce5..62ff828 100644 --- a/config.template +++ b/config.template @@ -122,4 +122,25 @@ # ## Relative path to the directory that shall be provided via rsync #RsyncDirectory rsync/ +# +# +######## Torperf downloader ######## +# +## Download and merge Torperf .data and .extradata files +#ProcessTorperfFiles 0 +# +## Relative path to the directory to store Torperf files in +#TorperfOutputDirectory torperf/ +# +## Torperf source names and base URLs (option can be contained multiple +## times) +#TorperfSource torperf http://torperf.torproject.org/ +# +## Torperf .data files available on a given source (option can be +## contained multiple times) +#TorperfDataFiles torperf 50kb.data 1mb.data 5mb.data +# +## Torperf .extradata files available on a given source (option can be +## contained multiple times) +#TorperfExtradataFiles torperf 50kb.extradata 1mb.extradata 5mb.extradata diff --git a/src/org/torproject/ernie/db/Configuration.java b/src/org/torproject/ernie/db/Configuration.java index 7197c7d..91f00b1 100644 --- a/src/org/torproject/ernie/db/Configuration.java +++ b/src/org/torproject/ernie/db/Configuration.java @@ -49,6 +49,11 @@ public class Configuration { private boolean processBridgePoolAssignments = false; private String assignmentsDirectory = "assignments/"; private String sanitizedAssignmentsDirectory = "sanitized-assignments/"; + private boolean processTorperfFiles = false; + private String torperfOutputDirectory = "torperf/"; + private SortedMap<String, String> torperfSources = null; + private SortedMap<String, List<String>> torperfDataFiles = null; + private SortedMap<String, List<String>> torperfExtradataFiles = null; private boolean provideFilesViaRsync = false; private String rsyncDirectory = "rsync"; public Configuration() { @@ -170,6 +175,41 @@ public class Configuration { this.assignmentsDirectory = line.split(" ")[1]; } else if (line.startsWith("SanitizedAssignmentsDirectory")) { this.sanitizedAssignmentsDirectory = line.split(" ")[1]; + } else if (line.startsWith("ProcessTorperfFiles")) { + this.processTorperfFiles = Integer.parseInt(line.split(" ")[1]) + != 0; + } else if (line.startsWith("TorperfOutputDirectory")) { + } else if (line.startsWith("TorperfSource")) { + if (this.torperfSources == null) { + this.torperfSources = new TreeMap<String, String>(); + } + String[] parts = line.split(" "); + String sourceName = parts[1]; + String baseUrl = parts[2]; + this.torperfSources.put(sourceName, baseUrl); + } else if (line.startsWith("TorperfDataFiles")) { + if (this.torperfDataFiles == null) { + this.torperfDataFiles = new TreeMap<String, List<String>>(); + } + String[] parts = line.split(" "); + String sourceName = parts[1]; + List<String> dataFiles = new ArrayList<String>(); + for (int i = 2; i < parts.length; i++) { + dataFiles.add(parts[i]); + } + this.torperfDataFiles.put(sourceName, dataFiles); + } else if (line.startsWith("TorperfExtradataFiles")) { + if (this.torperfExtradataFiles == null) { + this.torperfExtradataFiles = + new TreeMap<String, List<String>>(); + } + String[] parts = line.split(" "); + String sourceName = parts[1]; + List<String> extradataFiles = new ArrayList<String>(); + for (int i = 2; i < parts.length; i++) { + extradataFiles.add(parts[i]); + } + this.torperfExtradataFiles.put(sourceName, extradataFiles); } else if (line.startsWith("ProvideFilesViaRsync")) { this.provideFilesViaRsync = Integer.parseInt( line.split(" ")[1]) != 0; @@ -205,7 +245,8 @@ public class Configuration { !this.importDirectoryArchives && !this.downloadRelayDescriptors && !this.importBridgeSnapshots && !this.downloadGetTorStats && !this.downloadExitList && !this.processBridgePoolAssignments && - !this.writeDirectoryArchives && !this.writeSanitizedBridges) { + !this.writeDirectoryArchives && !this.writeSanitizedBridges && + !this.processTorperfFiles) { logger.warning("We have not been configured to read data from any " + "data source or write data to any data sink. You need to " + "edit your config file (" + configFile.getAbsolutePath() @@ -324,6 +365,21 @@ public class Configuration { public String getSanitizedAssignmentsDirectory() { return sanitizedAssignmentsDirectory; } + public boolean getProcessTorperfFiles() { + return this.processTorperfFiles; + } + public String getTorperfOutputDirectory() { + return this.torperfOutputDirectory; + } + public SortedMap<String, String> getTorperfSources() { + return this.torperfSources; + } + public SortedMap<String, List<String>> getTorperfDataFiles() { + return this.torperfDataFiles; + } + public SortedMap<String, List<String>> getTorperfExtradataFiles() { + return this.torperfExtradataFiles; + } public boolean getProvideFilesViaRsync() { return this.provideFilesViaRsync; } diff --git a/src/org/torproject/ernie/db/Main.java b/src/org/torproject/ernie/db/Main.java index ab79174..a5b5080 100644 --- a/src/org/torproject/ernie/db/Main.java +++ b/src/org/torproject/ernie/db/Main.java @@ -136,6 +136,13 @@ public class Main { new File(config.getSanitizedAssignmentsDirectory())); } + // Process Torperf files + if (config.getProcessTorperfFiles()) { + new TorperfDownloader(new File(config.getTorperfOutputDirectory()), + config.getTorperfSources(), config.getTorperfDataFiles(), + config.getTorperfExtradataFiles()); + } + // Copy recently published files to a local directory that can then // be served via rsync. if (config.getProvideFilesViaRsync()) { diff --git a/src/org/torproject/ernie/db/TorperfDownloader.java b/src/org/torproject/ernie/db/TorperfDownloader.java new file mode 100644 index 0000000..95d2e30 --- /dev/null +++ b/src/org/torproject/ernie/db/TorperfDownloader.java @@ -0,0 +1,116 @@ +/* Copyright 2012 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.ernie.db; + +import java.io.*; +import java.net.*; +import java.util.*; +import java.util.logging.*; + +/* Download possibly truncated Torperf .data and .extradata files from + * configured sources and append them to the files we already have. */ +public class TorperfDownloader { + + private File torperfOutputDirectory = null; + private SortedMap<String, String> torperfSources = null; + private SortedMap<String, List<String>> torperfDataFiles = null; + private SortedMap<String, List<String>> torperfExtradataFiles = null; + private Logger logger = null; + + public TorperfDownloader(File torperfOutputDirectory, + SortedMap<String, String> torperfSources, + SortedMap<String, List<String>> torperfDataFiles, + SortedMap<String, List<String>> torperfExtradataFiles) { + if (torperfOutputDirectory == null) { + throw new IllegalArgumentException(); + } + this.torperfOutputDirectory = torperfOutputDirectory; + this.torperfSources = torperfSources; + this.torperfDataFiles = torperfDataFiles; + this.torperfExtradataFiles = torperfExtradataFiles; + if (!this.torperfOutputDirectory.exists()) { + this.torperfOutputDirectory.mkdirs(); + } + this.logger = Logger.getLogger(TorperfDownloader.class.getName()); + this.downloadAndMergeFiles(this.torperfDataFiles, true); + this.downloadAndMergeFiles(this.torperfExtradataFiles, false); + } + + private void downloadAndMergeFiles( + SortedMap<String, List<String>> dataOrExtradataFiles, + boolean isDataFile) { + for (Map.Entry<String, List<String>> e : + dataOrExtradataFiles.entrySet()) { + String sourceName = e.getKey(); + String sourceBaseUrl = torperfSources.get(sourceName); + List<String> files = e.getValue(); + for (String file : files) { + String url = sourceBaseUrl + file; + File outputFile = new File(torperfOutputDirectory, + sourceName + "-" + file); + this.downloadAndMergeFile(url, outputFile, isDataFile); + } + } + } + + private void downloadAndMergeFile(String url, File outputFile, + boolean isDataFile) { + String lastTimestampLine = null; + int linesAfterLastTimestampLine = 0; + if (outputFile.exists() && outputFile.lastModified() > + System.currentTimeMillis() - 6L * 60L * 60L * 1000L) { + return; + } else if (outputFile.exists()) { + try { + BufferedReader br = new BufferedReader(new FileReader( + outputFile)); + String line; + while ((line = br.readLine()) != null) { + if (isDataFile || line.contains(" LAUNCH")) { + lastTimestampLine = line; + linesAfterLastTimestampLine = 0; + } else { + linesAfterLastTimestampLine++; + } + } + br.close(); + } catch (IOException e) { + logger.log(Level.WARNING, "Failed reading '" + + outputFile.getAbsolutePath() + "' to find the last line to " + + "append to.", e); + return; + } + } + try { + this.logger.fine("Downloading " + (isDataFile ? ".data" : + ".extradata") + " file from '" + url + "' and merging it into " + + "'" + outputFile.getAbsolutePath() + "'."); + URL u = new URL(url); + HttpURLConnection huc = (HttpURLConnection) u.openConnection(); + huc.setRequestMethod("GET"); + huc.connect(); + BufferedReader br = new BufferedReader(new InputStreamReader( + huc.getInputStream())); + String line; + BufferedWriter bw = new BufferedWriter(new FileWriter(outputFile, + true)); + boolean copyLines = lastTimestampLine == null; + while ((line = br.readLine()) != null) { + if (copyLines && linesAfterLastTimestampLine == 0) { + bw.write(line + "\n"); + } else if (copyLines && linesAfterLastTimestampLine > 0) { + linesAfterLastTimestampLine--; + } else if (line.equals(lastTimestampLine)) { + copyLines = true; + } + } + bw.close(); + br.close(); + } catch (IOException e) { + logger.log(Level.WARNING, "Failed downloading and merging '" + url + + "'.", e); + return; + } + } +} +