commit 8d8bdf1331699dc5ea3a97a2bc9436548ccfae3a Author: Karsten Loesing karsten.loesing@gmx.net Date: Tue Aug 7 12:46:12 2012 +0200

Remove GetTor statistics processing code. --- .gitignore | 4 - config.template | 12 -- src/org/torproject/ernie/db/Configuration.java | 25 +---- src/org/torproject/ernie/db/GetTorDownloader.java | 115 -------------------- src/org/torproject/ernie/db/Main.java | 8 -- src/org/torproject/ernie/db/RsyncDataProvider.java | 17 +--- 6 files changed, 2 insertions(+), 179 deletions(-)

diff --git a/.gitignore b/.gitignore index 8c9553c..51ae9b7 100644 --- a/.gitignore +++ b/.gitignore @@ -16,7 +16,6 @@ website/graphs/bridge-users/*.png website/graphs/descriptors/*.png website/graphs/direct-users/*.png website/graphs/exit/*.png -website/graphs/gettor/*.png website/graphs/networksize/*.png website/graphs/new-users/*.png website/graphs/torperf/*.png @@ -47,9 +46,6 @@ dist/ # Possibly modified config file config

-# Gettor files -gettor/ - # Torperf files torperf/

diff --git a/config.template b/config.template index aecc67d..52ee56f 100644 --- a/config.template +++ b/config.template @@ -96,18 +96,6 @@ #SanitizedAssignmentsDirectory sanitized-assignments/ # # -######## GetTor download statistics ######## -# -## Download GetTor stats -#DownloadGetTorStats 0 -# -## URL to download GetTor stats from -#GetTorStatsURL http://gettor.torproject.org:8080/~gettor/gettor_stats.txt -# -## Relative path to directory to store GetTor stats in -#GetTorDirectory gettor/ -# -# ######## Exit lists ######## # ## Download exit list and store it to disk diff --git a/src/org/torproject/ernie/db/Configuration.java b/src/org/torproject/ernie/db/Configuration.java index 63349c8..e130dab 100644 --- a/src/org/torproject/ernie/db/Configuration.java +++ b/src/org/torproject/ernie/db/Configuration.java @@ -48,10 +48,6 @@ public class Configuration { private boolean downloadAllServerDescriptors = false; private boolean downloadAllExtraInfoDescriptors = false; private boolean compressRelayDescriptorDownloads; - private boolean downloadGetTorStats = false; - private String getTorStatsUrl = "http://gettor.torproject.org:8080/" - + "~gettor/gettor_stats.txt"; - private String getTorDirectory = "gettor/"; private boolean downloadExitList = false; private boolean processBridgePoolAssignments = false; private String assignmentsDirectory = "assignments/"; @@ -161,16 +157,6 @@ public class Configuration { } else if (line.startsWith("CompressRelayDescriptorDownloads")) { this.compressRelayDescriptorDownloads = Integer.parseInt( line.split(" ")[1]) != 0; - } else if (line.startsWith("DownloadGetTorStats")) { - this.downloadGetTorStats = Integer.parseInt( - line.split(" ")[1]) != 0; - } else if (line.startsWith("GetTorStatsURL")) { - String newUrl = line.split(" ")[1]; - /* Test if URL has correct format. */ - new URL(newUrl); - this.getTorStatsUrl = newUrl; - } else if (line.startsWith("GetTorDirectory")) { - this.getTorDirectory = line.split(" ")[1]; } else if (line.startsWith("DownloadExitList")) { this.downloadExitList = Integer.parseInt( line.split(" ")[1]) != 0; @@ -238,7 +224,7 @@ public class Configuration { /** Make some checks if configuration is valid. */ if (!this.importCachedRelayDescriptors && !this.importDirectoryArchives && !this.downloadRelayDescriptors && - !this.importBridgeSnapshots && !this.downloadGetTorStats && + !this.importBridgeSnapshots && !this.downloadExitList && !this.processBridgePoolAssignments && !this.writeDirectoryArchives && !this.writeSanitizedBridges && !this.processTorperfFiles) { @@ -339,15 +325,6 @@ public class Configuration { public boolean getCompressRelayDescriptorDownloads() { return this.compressRelayDescriptorDownloads; } - public boolean getDownloadGetTorStats() { - return this.downloadGetTorStats; - } - public String getGetTorStatsUrl() { - return this.getTorStatsUrl; - } - public String getGetTorDirectory() { - return this.getTorDirectory; - } public boolean getDownloadExitList() { return this.downloadExitList; } diff --git a/src/org/torproject/ernie/db/GetTorDownloader.java b/src/org/torproject/ernie/db/GetTorDownloader.java deleted file mode 100644 index f1e1b77..0000000 --- a/src/org/torproject/ernie/db/GetTorDownloader.java +++ /dev/null @@ -1,115 +0,0 @@ -/* Copyright 2010--2012 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.ernie.db; - -import java.io.BufferedInputStream; -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.io.StringReader; -import java.net.HttpURLConnection; -import java.net.URL; -import java.util.SortedMap; -import java.util.TreeMap; -import java.util.logging.Level; -import java.util.logging.Logger; - -public class GetTorDownloader { - - public GetTorDownloader(String gettorStatsUrl, File getTorDirectory) { - - Logger logger = Logger.getLogger(GetTorDownloader.class.getName()); - - File getTorFile = new File(getTorDirectory, "gettor_stats.txt"); - SortedMap<String, String> getTorStats = new TreeMap<String, String>(); - - if (getTorFile.exists() && !getTorFile.isDirectory()) { - try { - logger.fine("Reading local gettor_stats.txt file..."); - BufferedReader br = new BufferedReader(new FileReader( - getTorFile)); - String line = null; - while ((line = br.readLine()) != null) { - if (line.startsWith("@type gettor 1.")) { - continue; - } - String date = line.split(" ")[0]; - getTorStats.put(date, line); - } - br.close(); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed parsing local GetTor stats!", - e); - return; - } - } - - String unparsed = null; - try { - logger.fine("Downloading GetTor stats..."); - URL u = new URL(gettorStatsUrl); - HttpURLConnection huc = (HttpURLConnection) u.openConnection(); - huc.setRequestMethod("GET"); - huc.connect(); - int response = huc.getResponseCode(); - if (response == 200) { - BufferedInputStream in = new BufferedInputStream( - huc.getInputStream()); - StringBuilder sb = new StringBuilder(); - int len; - byte[] data = new byte[1024]; - while ((len = in.read(data, 0, 1024)) >= 0) { - sb.append(new String(data, 0, len)); - } - in.close(); - unparsed = sb.toString(); - } - logger.fine("Finished downloading GetTor stats."); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed downloading GetTor stats", e); - return; - } - - try { - logger.fine("Parsing downloaded GetTor stats..."); - BufferedReader br = new BufferedReader(new StringReader(unparsed)); - String line = null; - while ((line = br.readLine()) != null) { - String date = line.split(" ")[0]; - getTorStats.put(date, line); - } - br.close(); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed parsing downloaded GetTor stats!", - e); - return; - } - - try { - logger.fine("Writing GetTor stats to local gettor_stats.txt " - + "file..."); - if (!getTorDirectory.exists()) { - getTorDirectory.mkdirs(); - } - BufferedWriter bw = new BufferedWriter(new FileWriter(getTorFile)); - bw.write("@type gettor 1.0\n"); - for (String line : getTorStats.values()) { - bw.write(line + "\n"); - } - bw.close(); - } catch (IOException e) { - logger.log(Level.WARNING, "Failed writing GetTor stats to local " - + "gettor_stats.txt file", e); - return; - } - - logger.info("Finished downloading and processing statistics on Tor " - + "packages delivered by GetTor.\nDownloaded " + unparsed.length() - + " bytes. Last date in statistics is " + getTorStats.lastKey() - + "."); - } -} - diff --git a/src/org/torproject/ernie/db/Main.java b/src/org/torproject/ernie/db/Main.java index 3fd6656..04cc868 100644 --- a/src/org/torproject/ernie/db/Main.java +++ b/src/org/torproject/ernie/db/Main.java @@ -118,12 +118,6 @@ public class Main { sbw = null; }

- // Download and process GetTor stats - if (config.getDownloadGetTorStats()) { - new GetTorDownloader(config.getGetTorStatsUrl(), - new File(config.getGetTorDirectory())); - } - // Download exit list and store it to disk if (config.getDownloadExitList()) { new ExitListDownloader(); @@ -153,8 +147,6 @@ public class Main { !config.getProcessBridgePoolAssignments() ? null : new File(config.getSanitizedAssignmentsDirectory()), config.getDownloadExitList(), - !config.getDownloadGetTorStats() ? null : - new File(config.getGetTorDirectory()), !config.getProcessTorperfFiles() ? null : new File(config.getTorperfOutputDirectory()), new File(config.getRsyncDirectory())); diff --git a/src/org/torproject/ernie/db/RsyncDataProvider.java b/src/org/torproject/ernie/db/RsyncDataProvider.java index 4cb8e99..2f9632e 100644 --- a/src/org/torproject/ernie/db/RsyncDataProvider.java +++ b/src/org/torproject/ernie/db/RsyncDataProvider.java @@ -21,7 +21,7 @@ public class RsyncDataProvider { public RsyncDataProvider(File directoryArchivesOutputDirectory, File sanitizedBridgesWriteDirectory, File sanitizedAssignmentsDirectory, - boolean downloadExitList, File getTorDirectory, + boolean downloadExitList, File torperfOutputDirectory, File rsyncDirectory) {

/* Initialize logger. */ @@ -174,21 +174,6 @@ public class RsyncDataProvider { + fileNamesInRsync.size() + " files left in " + rsyncDirectory.getAbsolutePath() + ".");

- /* Copy GetTor stats. */ - if (getTorDirectory != null) { - String getTorFileName = "gettor_stats.txt"; - File getTorFile = new File(getTorDirectory, getTorFileName); - if (getTorFile.exists() && - getTorFile.lastModified() >= cutOffMillis) { - this.copyFile(getTorFile, new File(rsyncDirectory, "gettor/" - + getTorFileName)); - fileNamesInRsync.remove(getTorFileName); - } - } - logger.info("After copying the GetTor stats file, there are still " - + fileNamesInRsync.size() + " files left in " - + rsyncDirectory.getAbsolutePath() + "."); - /* Delete all files that we didn't (over-)write in this run. */ files.add(rsyncDirectory); while (!files.isEmpty()) {