[tor-commits] [metrics-web/master] Use metrics-lib to parse rsync'ed torperf files.
karsten at torproject.org
karsten at torproject.org
Fri Jun 1 12:54:17 UTC 2012
commit ccb8bfe5be8d41bea824239f54fddf64ba0f9a41
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Fri Jun 1 14:35:19 2012 +0200
Use metrics-lib to parse rsync'ed torperf files.
---
config.template | 2 +-
src/org/torproject/ernie/cron/Configuration.java | 2 +-
.../torproject/ernie/cron/TorperfProcessor.java | 99 ++++++++++----------
3 files changed, 53 insertions(+), 50 deletions(-)
diff --git a/config.template b/config.template
index 97cebf8..528b36f 100644
--- a/config.template
+++ b/config.template
@@ -43,7 +43,7 @@
#ImportWriteTorperfStats 0
#
## Relative path to directory to import torperf results from
-#TorperfDirectory torperf/
+#TorperfDirectory in/torperf/
#
## Process GetTor stats and import them into the database
#ProcessGetTorStats 0
diff --git a/src/org/torproject/ernie/cron/Configuration.java b/src/org/torproject/ernie/cron/Configuration.java
index 92a11b2..5cdadaf 100644
--- a/src/org/torproject/ernie/cron/Configuration.java
+++ b/src/org/torproject/ernie/cron/Configuration.java
@@ -29,7 +29,7 @@ public class Configuration {
private String relayDescriptorRawFilesDirectory = "pg-import/";
private boolean writeBridgeStats = false;
private boolean importWriteTorperfStats = false;
- private String torperfDirectory = "torperf/";
+ private String torperfDirectory = "in/torperf/";
private boolean processGetTorStats = false;
private String getTorDirectory = "in/gettor/";
private String exoneraTorDatabaseJdbc = "jdbc:postgresql:"
diff --git a/src/org/torproject/ernie/cron/TorperfProcessor.java b/src/org/torproject/ernie/cron/TorperfProcessor.java
index 377fede..ff64fb0 100644
--- a/src/org/torproject/ernie/cron/TorperfProcessor.java
+++ b/src/org/torproject/ernie/cron/TorperfProcessor.java
@@ -24,12 +24,17 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
-import java.util.Stack;
import java.util.TimeZone;
import java.util.TreeMap;
import java.util.logging.Level;
import java.util.logging.Logger;
+import org.torproject.descriptor.Descriptor;
+import org.torproject.descriptor.DescriptorFile;
+import org.torproject.descriptor.DescriptorReader;
+import org.torproject.descriptor.DescriptorSourceFactory;
+import org.torproject.descriptor.TorperfResult;
+
public class TorperfProcessor {
public TorperfProcessor(File torperfDirectory, File statsDirectory,
String connectionURL) {
@@ -80,58 +85,56 @@ public class TorperfProcessor {
}
if (torperfDirectory.exists()) {
logger.fine("Importing files in " + torperfDirectory + "/...");
- Stack<File> filesInInputDir = new Stack<File>();
- filesInInputDir.add(torperfDirectory);
- while (!filesInInputDir.isEmpty()) {
- File pop = filesInInputDir.pop();
- if (pop.isDirectory()) {
- for (File f : pop.listFiles()) {
- filesInInputDir.add(f);
+ DescriptorReader descriptorReader =
+ DescriptorSourceFactory.createDescriptorReader();
+ descriptorReader.addDirectory(torperfDirectory);
+ descriptorReader.setExcludeFiles(new File(statsDirectory,
+ "torperf-history"));
+ Iterator<DescriptorFile> descriptorFiles =
+ descriptorReader.readDescriptors();
+ while (descriptorFiles.hasNext()) {
+ DescriptorFile descriptorFile = descriptorFiles.next();
+ if (descriptorFile.getException() != null) {
+ logger.log(Level.FINE, "Error parsing file.",
+ descriptorFile.getException());
+ continue;
+ }
+ for (Descriptor descriptor : descriptorFile.getDescriptors()) {
+ if (!(descriptor instanceof TorperfResult)) {
+ continue;
}
- } else {
- String source = pop.getName().substring(0,
- pop.getName().indexOf("."));
- String size = pop.getName().split("-")[1];
- long receivedBytes = 1L;
- if (pop.getName().endsWith("kb.data")) {
- receivedBytes *= 1024L;
- } else if (pop.getName().endsWith("mb.data")) {
- receivedBytes *= 1024L * 1024L;
+ TorperfResult result = (TorperfResult) descriptor;
+ String source = result.getSource();
+ long fileSize = result.getFileSize();
+ if (fileSize == 51200) {
+ source += "-50kb";
+ } else if (fileSize == 1048576) {
+ source += "-1mb";
+ } else if (fileSize == 5242880) {
+ source += "-5mb";
} else {
- // not a valid .data file
+ logger.fine("Unexpected file size '" + fileSize
+ + "'. Skipping.");
continue;
}
- receivedBytes *= Long.parseLong(size.substring(0,
- size.length() - "xb.data".length()));
- BufferedReader br = new BufferedReader(new FileReader(pop));
- String line = null;
- while ((line = br.readLine()) != null) {
- String[] parts = line.split(" ");
- // remove defective lines as they occurred on gabelmoo
- if (parts.length >= 20 && parts[0].length() == 10) {
- long startSec = Long.parseLong(parts[0]);
- String dateTime = formatter.format(startSec * 1000L);
- long completeMillis = Long.parseLong(parts[16])
- * 1000L + Long.parseLong(parts[17]) / 1000L
- - Long.parseLong(parts[0]) * 1000L
- + Long.parseLong(parts[1]) / 1000L;
- String key = source + "," + dateTime;
- String value = key;
- if ((parts.length == 20 && parts[16].equals("0")) ||
- (parts.length >= 21 && parts[20].equals("1"))) {
- value += ",-2"; // -2 for timeout
- } else if (Long.parseLong(parts[19]) < receivedBytes) {
- value += ",-1"; // -1 for failure
- } else {
- value += "," + completeMillis;
- }
- if (!rawObs.containsKey(key)) {
- rawObs.put(key, value);
- addedRawObs++;
- }
- }
+ String dateTime = formatter.format(result.getStartMillis());
+ long completeMillis = result.getDataCompleteMillis()
+ - result.getStartMillis();
+ String key = source + "," + dateTime;
+ String value = key;
+ if ((result.didTimeout() == null &&
+ result.getDataCompleteMillis() < 1) ||
+ (result.didTimeout() != null && result.didTimeout())) {
+ value += ",-2"; // -2 for timeout
+ } else if (result.getReadBytes() < fileSize) {
+ value += ",-1"; // -1 for failure
+ } else {
+ value += "," + completeMillis;
+ }
+ if (!rawObs.containsKey(key)) {
+ rawObs.put(key, value);
+ addedRawObs++;
}
- br.close();
}
}
logger.fine("Finished importing files in " + torperfDirectory
More information about the tor-commits
mailing list