commit 9a4cc7a43d93a362af6a7311c2048cc979af7968 Author: Karsten Loesing karsten.loesing@gmx.net Date: Thu Mar 3 15:40:47 2011 +0100

Remove torperf statistics from metrics-db. --- config.template | 6 - src/org/torproject/ernie/db/Configuration.java | 18 +- .../torproject/ernie/db/ExitListDownloader.java | 2 +- src/org/torproject/ernie/db/GetTorProcessor.java | 2 +- src/org/torproject/ernie/db/Main.java | 8 - src/org/torproject/ernie/db/TorperfProcessor.java | 346 -------------------- .../ernie/test/TorperfProcessorTest.java | 37 -- 7 files changed, 4 insertions(+), 415 deletions(-)

diff --git a/config.template b/config.template index 3477801..72cc411 100644 --- a/config.template +++ b/config.template @@ -32,12 +32,6 @@ ## Relative path to directory to import bridge descriptor snapshots from #BridgeSnapshotsDirectory bridge-directories/ # -## Import torperf data, if available, and write stats to disk -#ImportWriteTorperfStats 0 -# -## Relative path to directory to import torperf results from -#TorperfDirectory torperf/ -# ## Download and process GetTor stats #DownloadProcessGetTorStats 0 # diff --git a/src/org/torproject/ernie/db/Configuration.java b/src/org/torproject/ernie/db/Configuration.java index 3554e23..4d75b59 100644 --- a/src/org/torproject/ernie/db/Configuration.java +++ b/src/org/torproject/ernie/db/Configuration.java @@ -30,8 +30,6 @@ public class Configuration { private String sanitizedBridgesWriteDirectory = "sanitized-bridges/"; private boolean importBridgeSnapshots = false; private String bridgeSnapshotsDirectory = "bridge-directories/"; - private boolean importWriteTorperfStats = false; - private String torperfDirectory = "torperf/"; private boolean downloadRelayDescriptors = false; private List<String> downloadFromDirectoryAuthorities = Arrays.asList( "86.59.21.38,194.109.206.212,80.190.246.100:8180".split(",")); @@ -105,11 +103,6 @@ public class Configuration { line.split(" ")[1]) != 0; } else if (line.startsWith("BridgeSnapshotsDirectory")) { this.bridgeSnapshotsDirectory = line.split(" ")[1]; - } else if (line.startsWith("ImportWriteTorperfStats")) { - this.importWriteTorperfStats = Integer.parseInt( - line.split(" ")[1]) != 0; - } else if (line.startsWith("TorperfDirectory")) { - this.torperfDirectory = line.split(" ")[1]; } else if (line.startsWith("DownloadRelayDescriptors")) { this.downloadRelayDescriptors = Integer.parseInt( line.split(" ")[1]) != 0; @@ -165,9 +158,8 @@ public class Configuration { /** Make some checks if configuration is valid. */ if (!this.importCachedRelayDescriptors && !this.importDirectoryArchives && !this.downloadRelayDescriptors && - !this.importBridgeSnapshots && !this.importWriteTorperfStats && - !this.downloadProcessGetTorStats && !this.downloadExitList && - !this.writeDirectoryArchives && + !this.importBridgeSnapshots && !this.downloadProcessGetTorStats && + !this.downloadExitList && !this.writeDirectoryArchives && !this.writeAggregateStatsDatabase && !this.writeSanitizedBridges) { logger.warning("We have not been configured to read data from any " @@ -252,12 +244,6 @@ public class Configuration { public String getBridgeSnapshotsDirectory() { return this.bridgeSnapshotsDirectory; } - public boolean getImportWriteTorperfStats() { - return this.importWriteTorperfStats; - } - public String getTorperfDirectory() { - return this.torperfDirectory; - } public boolean getDownloadRelayDescriptors() { return this.downloadRelayDescriptors; } diff --git a/src/org/torproject/ernie/db/ExitListDownloader.java b/src/org/torproject/ernie/db/ExitListDownloader.java index df7d87a..07d2cc4 100644 --- a/src/org/torproject/ernie/db/ExitListDownloader.java +++ b/src/org/torproject/ernie/db/ExitListDownloader.java @@ -10,7 +10,7 @@ import java.util.logging.*;

public class ExitListDownloader { public ExitListDownloader() { - Logger logger = Logger.getLogger(TorperfProcessor.class.getName()); + Logger logger = Logger.getLogger(ExitListDownloader.class.getName()); try { logger.fine("Downloading exit list..."); String exitAddressesUrl = diff --git a/src/org/torproject/ernie/db/GetTorProcessor.java b/src/org/torproject/ernie/db/GetTorProcessor.java index 8c5687c..6d9a935 100644 --- a/src/org/torproject/ernie/db/GetTorProcessor.java +++ b/src/org/torproject/ernie/db/GetTorProcessor.java @@ -10,7 +10,7 @@ import java.util.logging.*;

public class GetTorProcessor { public GetTorProcessor(String gettorStatsUrl, String connectionURL) { - Logger logger = Logger.getLogger(TorperfProcessor.class.getName()); + Logger logger = Logger.getLogger(GetTorProcessor.class.getName()); String unparsed = null; try { logger.fine("Downloading gettor stats..."); diff --git a/src/org/torproject/ernie/db/Main.java b/src/org/torproject/ernie/db/Main.java index 2dc0c86..e10b6c7 100644 --- a/src/org/torproject/ernie/db/Main.java +++ b/src/org/torproject/ernie/db/Main.java @@ -117,14 +117,6 @@ public class Main { sbw = null; }

- // Import and process torperf stats - if (config.getImportWriteTorperfStats()) { - new TorperfProcessor(new File(config.getTorperfDirectory()), - statsDirectory, - config.getWriteAggregateStatsDatabase() ? - config.getRelayDescriptorDatabaseJDBC() : null); - } - // Download and process GetTor stats if (config.getDownloadProcessGetTorStats()) { new GetTorProcessor(config.getGetTorStatsUrl(), diff --git a/src/org/torproject/ernie/db/TorperfProcessor.java b/src/org/torproject/ernie/db/TorperfProcessor.java deleted file mode 100644 index d6d2e53..0000000 --- a/src/org/torproject/ernie/db/TorperfProcessor.java +++ /dev/null @@ -1,346 +0,0 @@ -/* Copyright 2010 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.ernie.db; - -import java.io.*; -import java.sql.*; -import java.text.*; -import java.util.*; -import java.util.logging.*; - -public class TorperfProcessor { - public TorperfProcessor(File torperfDirectory, File statsDirectory, - String connectionURL) { - - if (torperfDirectory == null || statsDirectory == null) { - throw new IllegalArgumentException(); - } - - Logger logger = Logger.getLogger(TorperfProcessor.class.getName()); - File rawFile = new File(statsDirectory, "torperf-raw"); - File statsFile = new File(statsDirectory, "torperf-stats"); - SortedMap<String, String> rawObs = new TreeMap<String, String>(); - SortedMap<String, String> stats = new TreeMap<String, String>(); - int addedRawObs = 0; - try { - if (rawFile.exists()) { - logger.fine("Reading file " + rawFile.getAbsolutePath() + "..."); - BufferedReader br = new BufferedReader(new FileReader(rawFile)); - String line = br.readLine(); // ignore header - while ((line = br.readLine()) != null) { - if (line.split(",").length != 4) { - logger.warning("Corrupt line in " + rawFile.getAbsolutePath() - + "!"); - break; - } - String key = line.substring(0, line.lastIndexOf(",")); - rawObs.put(key, line); - } - br.close(); - logger.fine("Finished reading file " + rawFile.getAbsolutePath() - + "."); - } - if (statsFile.exists()) { - logger.fine("Reading file " + statsFile.getAbsolutePath() - + "..."); - BufferedReader br = new BufferedReader(new FileReader(statsFile)); - String line = br.readLine(); // ignore header - while ((line = br.readLine()) != null) { - String key = line.split(",")[0] + "," + line.split(",")[1]; - stats.put(key, line); - } - br.close(); - logger.fine("Finished reading file " + statsFile.getAbsolutePath() - + "."); - } - if (torperfDirectory.exists()) { - logger.fine("Importing files in " + torperfDirectory + "/..."); - Stack<File> filesInInputDir = new Stack<File>(); - filesInInputDir.add(torperfDirectory); - while (!filesInInputDir.isEmpty()) { - File pop = filesInInputDir.pop(); - if (pop.isDirectory()) { - for (File f : pop.listFiles()) { - filesInInputDir.add(f); - } - } else { - String source = pop.getName().substring(0, - pop.getName().indexOf(".")); - String size = pop.getName().split("-")[1]; - long receivedBytes = 1L; - if (pop.getName().endsWith("kb.data")) { - receivedBytes *= 1024L; - } else if (pop.getName().endsWith("mb.data")) { - receivedBytes *= 1024L * 1024L; - } else { - // not a valid .data file - continue; - } - receivedBytes *= Long.parseLong(size.substring(0, - size.length() - "xb.data".length())); - BufferedReader br = new BufferedReader(new FileReader(pop)); - String line = null; - SimpleDateFormat formatter = - new SimpleDateFormat("yyyy-MM-dd,HH:mm:ss"); - formatter.setTimeZone(TimeZone.getTimeZone("UTC")); - while ((line = br.readLine()) != null) { - String[] parts = line.split(" "); - // remove defective lines as they occurred on gabelmoo - if (parts.length == 20 && parts[0].length() == 10) { - long startSec = Long.parseLong(parts[0]); - String dateTime = formatter.format(startSec * 1000L); - long completeMillis = Long.parseLong(parts[16]) - * 1000L + Long.parseLong(parts[17]) / 1000L - - Long.parseLong(parts[0]) * 1000L - + Long.parseLong(parts[1]) / 1000L; - String key = source + "," + dateTime; - String value = key; - if (parts[16].equals("0")) { - value += ",-2"; // -2 for timeout - } else if (Long.parseLong(parts[19]) < receivedBytes) { - value += ",-1"; // -1 for failure - } else { - value += "," + completeMillis; - } - if (!rawObs.containsKey(key)) { - rawObs.put(key, value); - addedRawObs++; - } - } - } - br.close(); - } - } - logger.fine("Finished importing files in " + torperfDirectory - + "/."); - } - if (rawObs.size() > 0) { - logger.fine("Writing file " + rawFile.getAbsolutePath() + "..."); - rawFile.getParentFile().mkdirs(); - BufferedWriter bw = new BufferedWriter(new FileWriter(rawFile)); - bw.append("source,date,start,completemillis\n"); - String tempSourceDate = null; - Iterator<Map.Entry<String, String>> it = - rawObs.entrySet().iterator(); - List<Long> dlTimes = new ArrayList<Long>(); - boolean haveWrittenFinalLine = false; - SortedMap<String, List<Long>> dlTimesAllSources = - new TreeMap<String, List<Long>>(); - SortedMap<String, long[]> statusesAllSources = - new TreeMap<String, long[]>(); - long failures = 0, timeouts = 0, requests = 0; - while (it.hasNext() || !haveWrittenFinalLine) { - Map.Entry<String, String> next = it.hasNext() ? it.next() : null; - if (tempSourceDate != null - && (next == null || !(next.getValue().split(",")[0] + "," - + next.getValue().split(",")[1]).equals(tempSourceDate))) { - if (dlTimes.size() > 4) { - Collections.sort(dlTimes); - long q1 = dlTimes.get(dlTimes.size() / 4 - 1); - long md = dlTimes.get(dlTimes.size() / 2 - 1); - long q3 = dlTimes.get(dlTimes.size() * 3 / 4 - 1); - stats.put(tempSourceDate, tempSourceDate + "," + q1 + "," - + md + "," + q3 + "," + timeouts + "," + failures + "," - + requests); - String allSourceDate = "all" + tempSourceDate.substring( - tempSourceDate.indexOf("-")); - if (dlTimesAllSources.containsKey(allSourceDate)) { - dlTimesAllSources.get(allSourceDate).addAll(dlTimes); - } else { - dlTimesAllSources.put(allSourceDate, dlTimes); - } - if (statusesAllSources.containsKey(allSourceDate)) { - long[] status = statusesAllSources.get(allSourceDate); - status[0] += timeouts; - status[1] += failures; - status[2] += requests; - } else { - long[] status = new long[3]; - status[0] = timeouts; - status[1] = failures; - status[2] = requests; - statusesAllSources.put(allSourceDate, status); - } - } - dlTimes = new ArrayList<Long>(); - failures = timeouts = requests = 0; - if (next == null) { - haveWrittenFinalLine = true; - } - } - if (next != null) { - bw.append(next.getValue() + "\n"); - String[] parts = next.getValue().split(","); - tempSourceDate = parts[0] + "," + parts[1]; - long completeMillis = Long.parseLong(parts[3]); - if (completeMillis == -2L) { - timeouts++; - } else if (completeMillis == -1L) { - failures++; - } else { - dlTimes.add(Long.parseLong(parts[3])); - } - requests++; - } - } - bw.close(); - for (Map.Entry<String, List<Long>> e : - dlTimesAllSources.entrySet()) { - String allSourceDate = e.getKey(); - dlTimes = e.getValue(); - Collections.sort(dlTimes); - long q1 = dlTimes.get(dlTimes.size() / 4 - 1); - long md = dlTimes.get(dlTimes.size() / 2 - 1); - long q3 = dlTimes.get(dlTimes.size() * 3 / 4 - 1); - long[] status = statusesAllSources.get(allSourceDate); - timeouts = status[0]; - failures = status[1]; - requests = status[2]; - stats.put(allSourceDate, allSourceDate + "," + q1 + "," + md - + "," + q3 + "," + timeouts + "," + failures + "," - + requests); - } - logger.fine("Finished writing file " + rawFile.getAbsolutePath() - + "."); - } - if (stats.size() > 0) { - logger.fine("Writing file " + statsFile.getAbsolutePath() - + "..."); - statsFile.getParentFile().mkdirs(); - BufferedWriter bw = new BufferedWriter(new FileWriter(statsFile)); - bw.append("source,date,q1,md,q3,timeouts,failures,requests\n"); - for (String s : stats.values()) { - bw.append(s + "\n"); - } - bw.close(); - logger.fine("Finished writing file " + statsFile.getAbsolutePath() - + "."); - } - } catch (IOException e) { - logger.log(Level.WARNING, "Failed writing " - + rawFile.getAbsolutePath() + " or " - + statsFile.getAbsolutePath() + "!", e); - } - - /* Write stats. */ - StringBuilder dumpStats = new StringBuilder("Finished writing " - + "statistics on torperf results.\nAdded " + addedRawObs - + " new observations in this execution.\n" - + "Last known obserations by source and file size are:"); - String lastSource = null; - String lastLine = null; - for (String s : rawObs.keySet()) { - String[] parts = s.split(","); - if (lastSource == null) { - lastSource = parts[0]; - } else if (!parts[0].equals(lastSource)) { - dumpStats.append("\n" + lastSource + " " + lastLine.split(",")[1] - + " " + lastLine.split(",")[2]); - lastSource = parts[0]; - } - lastLine = s; - } - if (lastSource != null) { - dumpStats.append("\n" + lastSource + " " + lastLine.split(",")[1] - + " " + lastLine.split(",")[2]); - } - logger.info(dumpStats.toString()); - - /* Write results to database. */ - if (connectionURL != null) { - try { - Map<String, String> insertRows = new HashMap<String, String>(); - insertRows.putAll(stats); - Set<String> updateRows = new HashSet<String>(); - Connection conn = DriverManager.getConnection(connectionURL); - conn.setAutoCommit(false); - Statement statement = conn.createStatement(); - ResultSet rs = statement.executeQuery( - "SELECT date, source, q1, md, q3, timeouts, failures, " - + "requests FROM torperf_stats"); - while (rs.next()) { - String date = rs.getDate(1).toString(); - String source = rs.getString(2); - String key = source + "," + date; - if (insertRows.containsKey(key)) { - String insertRow = insertRows.remove(key); - String[] newStats = insertRow.split(","); - long newQ1 = Long.parseLong(newStats[2]); - long newMd = Long.parseLong(newStats[3]); - long newQ3 = Long.parseLong(newStats[4]); - long newTimeouts = Long.parseLong(newStats[5]); - long newFailures = Long.parseLong(newStats[6]); - long newRequests = Long.parseLong(newStats[7]); - long oldQ1 = rs.getLong(3); - long oldMd = rs.getLong(4); - long oldQ3 = rs.getLong(5); - long oldTimeouts = rs.getLong(6); - long oldFailures = rs.getLong(7); - long oldRequests = rs.getLong(8); - if (newQ1 != oldQ1 || newMd != oldMd || newQ3 != oldQ3 || - newTimeouts != oldTimeouts || - newFailures != oldFailures || - newRequests != oldRequests) { - updateRows.add(insertRow); - } - } - } - PreparedStatement psU = conn.prepareStatement( - "UPDATE torperf_stats SET q1 = ?, md = ?, q3 = ?, " - + "timeouts = ?, failures = ?, requests = ? " - + "WHERE date = ? AND source = ?"); - for (String row : updateRows) { - String[] newStats = row.split(","); - String source = newStats[0]; - java.sql.Date date = java.sql.Date.valueOf(newStats[1]); - long q1 = Long.parseLong(newStats[2]); - long md = Long.parseLong(newStats[3]); - long q3 = Long.parseLong(newStats[4]); - long timeouts = Long.parseLong(newStats[5]); - long failures = Long.parseLong(newStats[6]); - long requests = Long.parseLong(newStats[7]); - psU.clearParameters(); - psU.setLong(1, q1); - psU.setLong(2, md); - psU.setLong(3, q3); - psU.setLong(4, timeouts); - psU.setLong(5, failures); - psU.setLong(6, requests); - psU.setDate(7, date); - psU.setString(8, source); - psU.executeUpdate(); - } - PreparedStatement psI = conn.prepareStatement( - "INSERT INTO torperf_stats (q1, md, q3, timeouts, failures, " - + "requests, date, source) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"); - for (String row : insertRows.values()) { - String[] newStats = row.split(","); - String source = newStats[0]; - java.sql.Date date = java.sql.Date.valueOf(newStats[1]); - long q1 = Long.parseLong(newStats[2]); - long md = Long.parseLong(newStats[3]); - long q3 = Long.parseLong(newStats[4]); - long timeouts = Long.parseLong(newStats[5]); - long failures = Long.parseLong(newStats[6]); - long requests = Long.parseLong(newStats[7]); - psI.clearParameters(); - psI.setLong(1, q1); - psI.setLong(2, md); - psI.setLong(3, q3); - psI.setLong(4, timeouts); - psI.setLong(5, failures); - psI.setLong(6, requests); - psI.setDate(7, date); - psI.setString(8, source); - psI.executeUpdate(); - } - conn.commit(); - conn.close(); - } catch (SQLException e) { - logger.log(Level.WARNING, "Failed to add torperf stats to " - + "database.", e); - } - } - } -} - diff --git a/src/org/torproject/ernie/test/TorperfProcessorTest.java b/src/org/torproject/ernie/test/TorperfProcessorTest.java deleted file mode 100644 index 015eec3..0000000 --- a/src/org/torproject/ernie/test/TorperfProcessorTest.java +++ /dev/null @@ -1,37 +0,0 @@ -/* Copyright 2011 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.ernie.test; - -import org.torproject.ernie.db.*; - -import java.io.*; - -import org.junit.*; -import org.junit.rules.*; -import static org.junit.Assert.*; - -public class TorperfProcessorTest { - - private File tempTorperfDirectory; - private File tempStatsDirectory; - - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - @Before - public void createTempDirectories() { - this.tempTorperfDirectory = folder.newFolder("torperf"); - this.tempStatsDirectory = folder.newFolder("stats"); - } - - @Test(expected = IllegalArgumentException.class) - public void testTorperfDirectoryNull() { - new TorperfProcessor(null, this.tempStatsDirectory, null); - } - - @Test(expected = IllegalArgumentException.class) - public void testStatsDirectoryNull() { - new TorperfProcessor(this.tempTorperfDirectory, null, null); - } -} -