[or-cvs] [ernie/master] Add parser and graphs for gettor stats.

karsten at torproject.org karsten at torproject.org
Sun Feb 14 22:56:52 UTC 2010


Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Sun, 14 Feb 2010 23:56:14 +0100
Subject: Add parser and graphs for gettor stats.
Commit: ac4a99b44c7a666e6dc1bc5e3d8d6641294ba90b

---
 R/gettor.R                 |   42 +++++++++++++++++++
 R/graphs.R                 |    1 +
 src/GetTorProcessor.java   |   97 ++++++++++++++++++++++++++++++++++++++++++++
 src/Main.java              |    1 +
 website/gettor-graphs.html |   45 ++++++++++++++++++++
 website/graphs.html        |   10 +++++
 6 files changed, 196 insertions(+), 0 deletions(-)
 create mode 100644 R/gettor.R
 create mode 100644 src/GetTorProcessor.java
 create mode 100644 website/gettor-graphs.html

diff --git a/R/gettor.R b/R/gettor.R
new file mode 100644
index 0000000..b37b3ed
--- /dev/null
+++ b/R/gettor.R
@@ -0,0 +1,42 @@
+options(warn = -1)
+suppressPackageStartupMessages(library("ggplot2"))
+
+gettor <- read.csv("stats/gettor-stats", header = TRUE,
+    stringsAsFactors = FALSE);
+total <- data.frame(date = gettor$date,
+  packages = rowSums(gettor[2:length(gettor)]) - gettor$none)
+en <- data.frame(date = gettor$date,
+  packages = gettor$tor.browser.bundle_en + gettor$tor.im.browser.bundle_en)
+zh_cn <- data.frame(date = gettor$date,
+  packages = gettor$tor.browser.bundle_zh_cn +
+  gettor$tor.im.browser.bundle_zh_cn)
+fa <- data.frame(date = gettor$date,
+  packages = gettor$tor.browser.bundle_fa + gettor$tor.im.browser.bundle_fa)
+
+write.csv(data.frame(date = gettor$date,
+  total = rowSums(gettor[2:length(gettor)]) - gettor$none,
+  en = gettor$tor.browser.bundle_en + gettor$tor.im.browser.bundle_en,
+  zh_cn = gettor$tor.browser.bundle_zh_cn +
+    gettor$tor.im.browser.bundle_zh_cn,
+  fa = gettor$tor.browser.bundle_fa + gettor$tor.im.browser.bundle_fa),
+  "website/csv/gettor.csv", quote = FALSE, row.names = FALSE)
+
+plot_packages <- function(filename, title, data) {
+  ggplot(data, aes(x = as.Date(date, "%Y-%m-%d"), y = packages)) + geom_line() +
+    scale_x_date(name = "") +
+    scale_y_continuous(name = "",
+    limits = c(0, max(data$packages, na.rm = TRUE))) +
+    opts(title = paste(title, "\n", sep = ""))
+  ggsave(filename = paste("website/graphs/gettor/", filename, sep = ""),
+    width = 8, height = 5, dpi = 72)
+}
+
+plot_packages("gettor-total.png",
+  "Total packages delivered by GetTor per day", total)
+plot_packages("gettor-en.png",
+  "Tor Browser Bundles (en) delivered by GetTor per day", en)
+plot_packages("gettor-zh_cn.png",
+  "Tor Browser Bundles (zh_CN) delivered by GetTor per day", zh_cn)
+plot_packages("gettor-fa.png",
+  "Tor Browser Bundles (fa) delivered by GetTor per day", fa)
+
diff --git a/R/graphs.R b/R/graphs.R
index 8881d6c..1fc819b 100644
--- a/R/graphs.R
+++ b/R/graphs.R
@@ -2,4 +2,5 @@ source("R/consensus-stats.R");
 source("R/dirreq-stats.R");
 source("R/bridge-stats.R");
 source("R/torperf.R");
+source("R/gettor.R");
 
diff --git a/src/GetTorProcessor.java b/src/GetTorProcessor.java
new file mode 100644
index 0000000..21b9df1
--- /dev/null
+++ b/src/GetTorProcessor.java
@@ -0,0 +1,97 @@
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import java.util.logging.*;
+
+public class GetTorProcessor {
+  private final String gettorStatsUrl =
+      "http://gettor.torproject.org:8080/~gettor/gettor_stats.txt";
+  public GetTorProcessor(String statsDirectory) {
+    Logger logger = Logger.getLogger(TorperfProcessor.class.getName());
+    Calendar now = Calendar.getInstance();
+    now.setTime(new Date());
+    if (now.get(Calendar.HOUR_OF_DAY) % 6 != 0) {
+      // only download every 6th hour
+      logger.info("Skipping downloading gettor stats.");
+      return;
+    }
+    String unparsed = null;
+    try {
+      logger.info("Downloading gettor stats...");
+      URL u = new URL(gettorStatsUrl);
+      HttpURLConnection huc = (HttpURLConnection) u.openConnection();
+      huc.setRequestMethod("GET");
+      huc.connect();
+      int response = huc.getResponseCode();
+      if (response == 200) {
+        BufferedInputStream in = new BufferedInputStream(
+            huc.getInputStream());
+        StringBuilder sb = new StringBuilder();
+        int len;
+        byte[] data = new byte[1024];
+        while ((len = in.read(data, 0, 1024)) >= 0) {
+          sb.append(new String(data, 0, len));
+        }   
+        in.close();
+        unparsed = sb.toString();
+      }   
+      logger.info("Finished downloading gettor stats.");
+    } catch (IOException e) {
+      logger.log(Level.WARNING, "Failed downloading gettor stats", e);
+      return;
+    }
+
+    SortedSet<String> columns = new TreeSet<String>();
+    SortedMap<String, Map<String, Integer>> data =
+        new TreeMap<String, Map<String, Integer>>();
+    try {
+      logger.info("Parsing downloaded gettor stats...");
+      BufferedReader br = new BufferedReader(new StringReader(unparsed));
+      String line = null;
+      while ((line = br.readLine()) != null) {
+        String[] parts = line.split(" ");
+        String date = parts[0];
+        Map<String, Integer> obs = new HashMap<String, Integer>();
+        data.put(date, obs);
+        for (int i = 2; i < parts.length; i++) {
+          String key = parts[i].split(":")[0].toLowerCase();
+          Integer value = new Integer(parts[i].split(":")[1]);
+          columns.add(key);
+          obs.put(key, value);
+        }
+      }
+      br.close();
+    } catch (IOException e) {
+      logger.log(Level.WARNING, "Failed parsing gettor stats!", e);
+      return;
+    } catch (NumberFormatException e) {
+      logger.log(Level.WARNING, "Failed parsing gettor stats!", e);
+      return;
+    }
+
+    logger.info("Writing file " + statsDirectory + "/gettor-stats...");
+    try {
+      File statsFile = new File(statsDirectory + "/gettor-stats");
+      new File(statsDirectory).mkdirs();
+      BufferedWriter bw = new BufferedWriter(new FileWriter(statsFile));
+      bw.write("date");
+      for (String column : columns) {
+        bw.write("," + column);
+      }
+      bw.write("\n");
+      for (String date : data.keySet()) {
+        bw.write(date);
+        for (String column : columns) {
+          Integer value = data.get(date).get(column);
+          bw.write("," + (value == null ? "NA" : value));
+        }
+        bw.write("\n");
+      }
+      bw.close();
+    } catch (IOException e) {
+      logger.log(Level.WARNING, "Failed writing " + statsDirectory
+          + "/gettor-stats!", e);
+    }
+  }
+}
+
diff --git a/src/Main.java b/src/Main.java
index 1c78520..dd11eed 100644
--- a/src/Main.java
+++ b/src/Main.java
@@ -74,6 +74,7 @@ public class Main {
           "bridge-directories", statsDirectory, countries);
       TorperfProcessor tp = new TorperfProcessor(statsDirectory,
           "torperf");
+      GetTorProcessor gtp = new GetTorProcessor(statsDirectory);
       logger.info("Finished importing data.");
     }
 
diff --git a/website/gettor-graphs.html b/website/gettor-graphs.html
new file mode 100644
index 0000000..c69a794
--- /dev/null
+++ b/website/gettor-graphs.html
@@ -0,0 +1,45 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
+<html>
+  <head>
+    <title>Tor Metrics Portal: Packages delivered by GetTor</title>
+    <meta http-equiv=Content-Type content="text/html; charset=iso-8859-1">
+    <link href="http://www.torproject.org/stylesheet-ltr.css" type=text/css rel=stylesheet>
+    <link href="http://www.torproject.org/favicon.ico" type=image/x-icon rel="shortcut icon">
+  </head>
+  <body>
+    <div class="center">
+      <table class="banner" border="0" cellpadding="0" cellspacing="0" summary="">
+        <tr>
+          <td class="banner-left"><a href="https://www.torproject.org/"><img src="http://www.torproject.org/images/top-left.png" alt="Click to go to home page" width="193" height="79"></a></td>
+          <td class="banner-middle">
+            <a href="/">Home</a>
+            <a class="current">Graphs</a>
+            <a href="reports.html">Reports</a>
+            <a href="papers.html">Papers</a>
+            <a href="data.html">Data</a>
+            <a href="docs.html">Docs</a>
+          </td>
+          <td class="banner-right"></td>
+        </tr>
+      </table>
+      <div class="main-column">
+        <h2>Tor Metrics Portal: Graphs</h2>
+        <br/>
+        <h3>Packages delivered by GetTor</h3>
+        <br/>
+        <p>GetTor allows users to fetch Tor via email. The following
+        graphs show the number of delivered packages per day.</p>
+        <p><a href="csv/gettor.csv">CSV</a> file containing all data.</p>
+        <img src="graphs/gettor/gettor-total.png"/>
+        <img src="graphs/gettor/gettor-en.png"/>
+        <img src="graphs/gettor/gettor-zh_cn.png"/>
+        <img src="graphs/gettor/gettor-fa.png"/>
+        <br/>
+      </div>
+    </div>
+    <div class="bottom" id="bottom">
+      <p>"Tor" and the "Onion Logo" are <a href="https://www.torproject.org/trademark-faq.html.en">registered trademarks</a> of The Tor Project, Inc.</p>
+    </div>
+  </body>
+</html>
+
diff --git a/website/graphs.html b/website/graphs.html
index 113b645..219dc2b 100644
--- a/website/graphs.html
+++ b/website/graphs.html
@@ -38,6 +38,7 @@
           users</a></li>
           <li><a href="#bridgeusers">Tor users via bridges</a></li>
           <li><a href="#torperf">Time to complete requests</a></li>
+          <li><a href="#gettor">Packages delivered by GetTor</a></li>
         </ul>
         <br/>
         <a id="relays"/>
@@ -98,6 +99,15 @@
         <p>Graphs for other file sizes or time intervals can be found on a
         <a href="torperf-graphs.html">separate page</a>.</p>
         <br/>
+        <a id="gettor"/>
+        <h3>Packages delivered by GetTor</h3>
+        <br/>
+        <p>GetTor allows users to fetch Tor via email. The following
+        graphs show the number of delivered packages per day.</p>
+        <img src="graphs/gettor/gettor-total.png"/>
+        <p>More graphs about specific packages can be found on a 
+        <a href="gettor-graphs.html">separate page</a>.</p>
+        <br/>
       </div>
     </div>
     <div class="bottom" id="bottom">
-- 
1.6.5



More information about the tor-commits mailing list