[tor-commits] [metrics-db/master] Remove GetTor statistics processing code.
karsten at torproject.org
karsten at torproject.org
Tue Aug 7 11:12:21 UTC 2012
commit 8d8bdf1331699dc5ea3a97a2bc9436548ccfae3a
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Tue Aug 7 12:46:12 2012 +0200
Remove GetTor statistics processing code.
---
.gitignore | 4 -
config.template | 12 --
src/org/torproject/ernie/db/Configuration.java | 25 +----
src/org/torproject/ernie/db/GetTorDownloader.java | 115 --------------------
src/org/torproject/ernie/db/Main.java | 8 --
src/org/torproject/ernie/db/RsyncDataProvider.java | 17 +---
6 files changed, 2 insertions(+), 179 deletions(-)
diff --git a/.gitignore b/.gitignore
index 8c9553c..51ae9b7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,7 +16,6 @@ website/graphs/bridge-users/*.png
website/graphs/descriptors/*.png
website/graphs/direct-users/*.png
website/graphs/exit/*.png
-website/graphs/gettor/*.png
website/graphs/networksize/*.png
website/graphs/new-users/*.png
website/graphs/torperf/*.png
@@ -47,9 +46,6 @@ dist/
# Possibly modified config file
config
-# Gettor files
-gettor/
-
# Torperf files
torperf/
diff --git a/config.template b/config.template
index aecc67d..52ee56f 100644
--- a/config.template
+++ b/config.template
@@ -96,18 +96,6 @@
#SanitizedAssignmentsDirectory sanitized-assignments/
#
#
-######## GetTor download statistics ########
-#
-## Download GetTor stats
-#DownloadGetTorStats 0
-#
-## URL to download GetTor stats from
-#GetTorStatsURL http://gettor.torproject.org:8080/~gettor/gettor_stats.txt
-#
-## Relative path to directory to store GetTor stats in
-#GetTorDirectory gettor/
-#
-#
######## Exit lists ########
#
## Download exit list and store it to disk
diff --git a/src/org/torproject/ernie/db/Configuration.java b/src/org/torproject/ernie/db/Configuration.java
index 63349c8..e130dab 100644
--- a/src/org/torproject/ernie/db/Configuration.java
+++ b/src/org/torproject/ernie/db/Configuration.java
@@ -48,10 +48,6 @@ public class Configuration {
private boolean downloadAllServerDescriptors = false;
private boolean downloadAllExtraInfoDescriptors = false;
private boolean compressRelayDescriptorDownloads;
- private boolean downloadGetTorStats = false;
- private String getTorStatsUrl = "http://gettor.torproject.org:8080/"
- + "~gettor/gettor_stats.txt";
- private String getTorDirectory = "gettor/";
private boolean downloadExitList = false;
private boolean processBridgePoolAssignments = false;
private String assignmentsDirectory = "assignments/";
@@ -161,16 +157,6 @@ public class Configuration {
} else if (line.startsWith("CompressRelayDescriptorDownloads")) {
this.compressRelayDescriptorDownloads = Integer.parseInt(
line.split(" ")[1]) != 0;
- } else if (line.startsWith("DownloadGetTorStats")) {
- this.downloadGetTorStats = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("GetTorStatsURL")) {
- String newUrl = line.split(" ")[1];
- /* Test if URL has correct format. */
- new URL(newUrl);
- this.getTorStatsUrl = newUrl;
- } else if (line.startsWith("GetTorDirectory")) {
- this.getTorDirectory = line.split(" ")[1];
} else if (line.startsWith("DownloadExitList")) {
this.downloadExitList = Integer.parseInt(
line.split(" ")[1]) != 0;
@@ -238,7 +224,7 @@ public class Configuration {
/** Make some checks if configuration is valid. */
if (!this.importCachedRelayDescriptors &&
!this.importDirectoryArchives && !this.downloadRelayDescriptors &&
- !this.importBridgeSnapshots && !this.downloadGetTorStats &&
+ !this.importBridgeSnapshots &&
!this.downloadExitList && !this.processBridgePoolAssignments &&
!this.writeDirectoryArchives && !this.writeSanitizedBridges &&
!this.processTorperfFiles) {
@@ -339,15 +325,6 @@ public class Configuration {
public boolean getCompressRelayDescriptorDownloads() {
return this.compressRelayDescriptorDownloads;
}
- public boolean getDownloadGetTorStats() {
- return this.downloadGetTorStats;
- }
- public String getGetTorStatsUrl() {
- return this.getTorStatsUrl;
- }
- public String getGetTorDirectory() {
- return this.getTorDirectory;
- }
public boolean getDownloadExitList() {
return this.downloadExitList;
}
diff --git a/src/org/torproject/ernie/db/GetTorDownloader.java b/src/org/torproject/ernie/db/GetTorDownloader.java
deleted file mode 100644
index f1e1b77..0000000
--- a/src/org/torproject/ernie/db/GetTorDownloader.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/* Copyright 2010--2012 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.ernie.db;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.StringReader;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-public class GetTorDownloader {
-
- public GetTorDownloader(String gettorStatsUrl, File getTorDirectory) {
-
- Logger logger = Logger.getLogger(GetTorDownloader.class.getName());
-
- File getTorFile = new File(getTorDirectory, "gettor_stats.txt");
- SortedMap<String, String> getTorStats = new TreeMap<String, String>();
-
- if (getTorFile.exists() && !getTorFile.isDirectory()) {
- try {
- logger.fine("Reading local gettor_stats.txt file...");
- BufferedReader br = new BufferedReader(new FileReader(
- getTorFile));
- String line = null;
- while ((line = br.readLine()) != null) {
- if (line.startsWith("@type gettor 1.")) {
- continue;
- }
- String date = line.split(" ")[0];
- getTorStats.put(date, line);
- }
- br.close();
- } catch (IOException e) {
- logger.log(Level.WARNING, "Failed parsing local GetTor stats!",
- e);
- return;
- }
- }
-
- String unparsed = null;
- try {
- logger.fine("Downloading GetTor stats...");
- URL u = new URL(gettorStatsUrl);
- HttpURLConnection huc = (HttpURLConnection) u.openConnection();
- huc.setRequestMethod("GET");
- huc.connect();
- int response = huc.getResponseCode();
- if (response == 200) {
- BufferedInputStream in = new BufferedInputStream(
- huc.getInputStream());
- StringBuilder sb = new StringBuilder();
- int len;
- byte[] data = new byte[1024];
- while ((len = in.read(data, 0, 1024)) >= 0) {
- sb.append(new String(data, 0, len));
- }
- in.close();
- unparsed = sb.toString();
- }
- logger.fine("Finished downloading GetTor stats.");
- } catch (IOException e) {
- logger.log(Level.WARNING, "Failed downloading GetTor stats", e);
- return;
- }
-
- try {
- logger.fine("Parsing downloaded GetTor stats...");
- BufferedReader br = new BufferedReader(new StringReader(unparsed));
- String line = null;
- while ((line = br.readLine()) != null) {
- String date = line.split(" ")[0];
- getTorStats.put(date, line);
- }
- br.close();
- } catch (IOException e) {
- logger.log(Level.WARNING, "Failed parsing downloaded GetTor stats!",
- e);
- return;
- }
-
- try {
- logger.fine("Writing GetTor stats to local gettor_stats.txt "
- + "file...");
- if (!getTorDirectory.exists()) {
- getTorDirectory.mkdirs();
- }
- BufferedWriter bw = new BufferedWriter(new FileWriter(getTorFile));
- bw.write("@type gettor 1.0\n");
- for (String line : getTorStats.values()) {
- bw.write(line + "\n");
- }
- bw.close();
- } catch (IOException e) {
- logger.log(Level.WARNING, "Failed writing GetTor stats to local "
- + "gettor_stats.txt file", e);
- return;
- }
-
- logger.info("Finished downloading and processing statistics on Tor "
- + "packages delivered by GetTor.\nDownloaded " + unparsed.length()
- + " bytes. Last date in statistics is " + getTorStats.lastKey()
- + ".");
- }
-}
-
diff --git a/src/org/torproject/ernie/db/Main.java b/src/org/torproject/ernie/db/Main.java
index 3fd6656..04cc868 100644
--- a/src/org/torproject/ernie/db/Main.java
+++ b/src/org/torproject/ernie/db/Main.java
@@ -118,12 +118,6 @@ public class Main {
sbw = null;
}
- // Download and process GetTor stats
- if (config.getDownloadGetTorStats()) {
- new GetTorDownloader(config.getGetTorStatsUrl(),
- new File(config.getGetTorDirectory()));
- }
-
// Download exit list and store it to disk
if (config.getDownloadExitList()) {
new ExitListDownloader();
@@ -153,8 +147,6 @@ public class Main {
!config.getProcessBridgePoolAssignments() ? null :
new File(config.getSanitizedAssignmentsDirectory()),
config.getDownloadExitList(),
- !config.getDownloadGetTorStats() ? null :
- new File(config.getGetTorDirectory()),
!config.getProcessTorperfFiles() ? null :
new File(config.getTorperfOutputDirectory()),
new File(config.getRsyncDirectory()));
diff --git a/src/org/torproject/ernie/db/RsyncDataProvider.java b/src/org/torproject/ernie/db/RsyncDataProvider.java
index 4cb8e99..2f9632e 100644
--- a/src/org/torproject/ernie/db/RsyncDataProvider.java
+++ b/src/org/torproject/ernie/db/RsyncDataProvider.java
@@ -21,7 +21,7 @@ public class RsyncDataProvider {
public RsyncDataProvider(File directoryArchivesOutputDirectory,
File sanitizedBridgesWriteDirectory,
File sanitizedAssignmentsDirectory,
- boolean downloadExitList, File getTorDirectory,
+ boolean downloadExitList,
File torperfOutputDirectory, File rsyncDirectory) {
/* Initialize logger. */
@@ -174,21 +174,6 @@ public class RsyncDataProvider {
+ fileNamesInRsync.size() + " files left in "
+ rsyncDirectory.getAbsolutePath() + ".");
- /* Copy GetTor stats. */
- if (getTorDirectory != null) {
- String getTorFileName = "gettor_stats.txt";
- File getTorFile = new File(getTorDirectory, getTorFileName);
- if (getTorFile.exists() &&
- getTorFile.lastModified() >= cutOffMillis) {
- this.copyFile(getTorFile, new File(rsyncDirectory, "gettor/"
- + getTorFileName));
- fileNamesInRsync.remove(getTorFileName);
- }
- }
- logger.info("After copying the GetTor stats file, there are still "
- + fileNamesInRsync.size() + " files left in "
- + rsyncDirectory.getAbsolutePath() + ".");
-
/* Delete all files that we didn't (over-)write in this run. */
files.add(rsyncDirectory);
while (!files.isEmpty()) {
More information about the tor-commits
mailing list