- tor-commits - lists.torproject.org

[tor/master] Re-order tor_fw_helper_LDADD flags so that they work
by nickm＠torproject.org 01 Aug '11

01 Aug '11

commit 9d77f24cf5032c4a1dbf64ec2e25d7debf3758ed Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Aug 1 11:05:49 2011 -0400 Re-order tor_fw_helper_LDADD flags so that they work Patch from "blueness". --- src/tools/tor-fw-helper/Makefile.am | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/src/tools/tor-fw-helper/Makefile.am b/src/tools/tor-fw-helper/Makefile.am index 75878f2..77ff63f 100644 --- a/src/tools/tor-fw-helper/Makefile.am +++ b/src/tools/tor-fw-helper/Makefile.am @@ -34,5 +34,5 @@ miniupnpc_cppflags = endif tor_fw_helper_LDFLAGS = $(nat_pmp_ldflags) $(miniupnpc_ldflags) -tor_fw_helper_LDADD = $(nat_pmp_ldadd) $(miniupnpc_ldadd) ../../common/libor.a @TOR_LIB_WS32@ +tor_fw_helper_LDADD = ../../common/libor.a $(nat_pmp_ldadd) $(miniupnpc_ldadd) @TOR_LIB_WS32@ tor_fw_helper_CPPFLAGS = $(nat_pmp_cppflags) $(miniupnpc_cppflags)

1 0

[flashproxy/master] Gracefully handle the case when a request is so broken it doesn't have
by dcf＠torproject.org 31 Jul '11

31 Jul '11

commit 7f54b3675a9d62d930dc1842be0253d588178f0d Author: David Fifield <david(a)bamsoftware.com> Date: Thu Jul 28 16:04:33 2011 +0000 Gracefully handle the case when a request is so broken it doesn't have headers. --- facilitator.py | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diff --git a/facilitator.py b/facilitator.py index eaad855..287602d 100755 --- a/facilitator.py +++ b/facilitator.py @@ -272,7 +272,10 @@ class Handler(BaseHTTPServer.BaseHTTPRequestHandler): def log_request(self, code): addr_s = format_addr(self.client_address) - referer = self.headers.get("Referer") or "-" + try: + referer = self.headers["Referer"] + except (AttributeError, KeyError): + referer = "-" log(u"resp %s %s %d %s" % (addr_s, repr(self.requestline), code, repr(referer)))

1 0

[flashproxy/master] Turn "broken pipe" facilitator errors into a one-line log message.
by dcf＠torproject.org 31 Jul '11

31 Jul '11

commit 545b33f21b1db659f80fd1109469d25ce5ab6533 Author: David Fifield <david(a)bamsoftware.com> Date: Thu Jul 28 16:34:16 2011 +0000 Turn "broken pipe" facilitator errors into a one-line log message. --- facilitator.py | 14 ++++++++++++++ 1 files changed, 14 insertions(+), 0 deletions(-) diff --git a/facilitator.py b/facilitator.py index 287602d..b2f8304 100755 --- a/facilitator.py +++ b/facilitator.py @@ -3,6 +3,7 @@ import BaseHTTPServer import SocketServer import cgi +import errno import getopt import os import re @@ -302,6 +303,19 @@ class Handler(BaseHTTPServer.BaseHTTPRequestHandler): data["relay"] = options.relay_spec self.request.send(urllib.urlencode(data)) + # Catch "broken pipe" errors that otherwise cause a stack trace in the log. + def catch_epipe(fn): + def ret(self, *args): + try: + fn(self, *args) + except socket.error, e: + if e.errno != errno.EPIPE: + raise + log(u"%s broken pipe" % format_addr(self.client_address)) + return ret + handle = catch_epipe(BaseHTTPServer.BaseHTTPRequestHandler.handle) + finish = catch_epipe(BaseHTTPServer.BaseHTTPRequestHandler.finish) + REGS = RegSet() opts, args = getopt.gnu_getopt(sys.argv[1:], "dhl:r:",

1 0

[torperf/master] Remove unused command-line options -v and -F.
by karsten＠torproject.org 30 Jul '11

30 Jul '11

commit 9e94e666068c6d9daec1df567e44d93ffa8a6d85 Author: David Fifield <david(a)bamsoftware.com> Date: Thu Jul 28 19:17:59 2011 +0000 Remove unused command-line options -v and -F. --- trivsocks-client.c | 8 ++------ 1 files changed, 2 insertions(+), 6 deletions(-) diff --git a/trivsocks-client.c b/trivsocks-client.c index 1c3d895..70e29a7 100644 --- a/trivsocks-client.c +++ b/trivsocks-client.c @@ -449,7 +449,7 @@ main(int argc, char **argv) { uint32_t sockshost; uint16_t socksport; - int isSocks4 = 0, isVerbose = 0, isReverse = 0, force = 0; + int isSocks4 = 0, isReverse = 0; char **arg; int n_args; uint32_t result = 0; @@ -470,16 +470,12 @@ main(int argc, char **argv) } while (n_args && *arg[0] == '-') { - if (!strcmp("-v", arg[0])) - isVerbose = 1; - else if (!strcmp("-4", arg[0])) + if (!strcmp("-4", arg[0])) isSocks4 = 1; else if (!strcmp("-5", arg[0])) isSocks4 = 0; else if (!strcmp("-x", arg[0])) isReverse = 1; - else if (!strcmp("-F", arg[0])) - force = 1; else { fprintf(stderr, "Unrecognized flag '%s'\n", arg[0]); usage();

1 0

[torperf/master] Fix compilation on MacOS X
by karsten＠torproject.org 30 Jul '11

30 Jul '11

commit a8cb23f556b9391562ad0e632d9f05a332eaaf95 Author: Steven Murdoch <Steven.Murdoch(a)cl.cam.ac.uk> Date: Fri Jul 29 16:54:44 2011 +0100 Fix compilation on MacOS X - Properly cast parameters to printf - Replace GNU-specific strndup with tor_strndup --- trivsocks-client.c | 2 +- util.c | 23 ++++++++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/trivsocks-client.c b/trivsocks-client.c index 70e29a7..d68aaca 100644 --- a/trivsocks-client.c +++ b/trivsocks-client.c @@ -249,7 +249,7 @@ do_http_get(int s, const char *path, const char *hostname, size_t *read_bytes, s static int print_time(struct timeval t) { - return printf("%ld %ld ", t.tv_sec, t.tv_usec); + return printf("%ld %ld ", (long int)t.tv_sec, (long int)t.tv_usec); } // Timestamps of important events diff --git a/util.c b/util.c index 96cee0a..fcc8ef9 100644 --- a/util.c +++ b/util.c @@ -73,6 +73,27 @@ read_all(int fd, char *buf, size_t count, int isSocket) return numread; } +/** Allocate and return a new string containing the first <b>n</b> + * characters of <b>s</b>. If <b>s</b> is longer than <b>n</b> + * characters, only the first <b>n</b> are copied. The result is + * always NUL-terminated. + */ +char * +tor_strndup(const char *s, size_t n) +{ + char *dup; + dup = malloc(n+1); + /* Performance note: Ordinarily we prefer strlcpy to strncpy. But + * this function gets called a whole lot, and platform strncpy is + * much faster than strlcpy when strlen(s) is much longer than n. + */ + if (!dup) + return dup; + strncpy(dup, s, n); + dup[n]='\0'; + return dup; +} + /** * Read a 16-bit value beginning at <b>cp</b>. Equivalent to * *(uint16_t*)(cp), but will not cause segfaults on platforms that forbid @@ -207,7 +228,7 @@ parse_addr_port(int severity, const char *addrport, char **address, colon = strchr(addrport, ':'); if (colon) { - _address = strndup(addrport, colon-addrport); + _address = tor_strndup(addrport, colon-addrport); _port = (int) parse_long(colon+1,10,1,65535,NULL,NULL); if (!_port) { fprintf(stderr, "Port %s out of range\n", colon+1);

1 0

[torperf/master] Add ChangeLog entry for previous 2 commits.
by karsten＠torproject.org 30 Jul '11

30 Jul '11

commit 079de667fb0a0ed05bbb4e33de168ec13644c858 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jul 29 21:17:49 2011 -0400 Add ChangeLog entry for previous 2 commits. --- ChangeLog | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/ChangeLog b/ChangeLog index b41f1be..b0ae8c3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -3,6 +3,7 @@ Torperf change log: Changes in version 0.0.2 - 2011-??-?? - Include <arpa/inet.h> for the BSDs that don't get ntohl from any of the other includes. + - Fix compilation for GCC 4.6 and Mac OS X. Changes in version 0.0.1 - 2011-04-12 - Initial release

1 0

[metrics-web/master] Add table with top-10 relays by bandwidth.
by karsten＠torproject.org 30 Jul '11

30 Jul '11

commit e988175f5056c2b4f4eb74bc0d8b8df26886eb2c Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jul 29 20:41:04 2011 -0400 Add table with top-10 relays by bandwidth. --- rserve/tables.R | 16 ++++++++++ .../ernie/web/GraphsSubpagesServlet.java | 3 ++ .../ernie/web/TableParameterChecker.java | 1 + web/WEB-INF/network.jsp | 31 ++++++++++++++++++++ 4 files changed, 51 insertions(+), 0 deletions(-) diff --git a/rserve/tables.R b/rserve/tables.R index 6d3b5fb..aabd588 100644 --- a/rserve/tables.R +++ b/rserve/tables.R @@ -69,3 +69,19 @@ write_censorship_events <- function(start, end, path) { write.csv(r, path, quote = FALSE, row.names = FALSE) } +write_top_10_relays <- function(start, end, path) { + drv <- dbDriver("PostgreSQL") + con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) + q <- paste("SELECT fingerprint, SUM(written_sum) ", + "AS written_sum FROM bwhist WHERE date >= '", start, + "' AND date <= '", end, "' GROUP BY 1 ORDER BY 2 DESC LIMIT 10", + sep = "") + rs <- dbSendQuery(con, q) + c <- fetch(rs, n = -1) + dbDisconnect(con) + dbUnloadDriver(drv) + c <- data.frame(fingerprint = c$fingerprint, + written_sum = round(c$written_sum / 2^40, 2)) + write.csv(c, path, quote = FALSE, row.names = FALSE) +} + diff --git a/src/org/torproject/ernie/web/GraphsSubpagesServlet.java b/src/org/torproject/ernie/web/GraphsSubpagesServlet.java index c7f3056..16d0db0 100644 --- a/src/org/torproject/ernie/web/GraphsSubpagesServlet.java +++ b/src/org/torproject/ernie/web/GraphsSubpagesServlet.java @@ -36,6 +36,9 @@ public class GraphsSubpagesServlet extends HttpServlet { this.availableGraphsSubpageTables.put("users.html", new HashSet<String>(Arrays.asList( "direct-users,censorship-events".split(",")))); + this.availableGraphsSubpageTables.put("network.html", + new HashSet<String>(Arrays.asList( + "top-10-relays".split(",")))); this.knownCountries = Countries.getInstance().getCountryList(); } diff --git a/src/org/torproject/ernie/web/TableParameterChecker.java b/src/org/torproject/ernie/web/TableParameterChecker.java index 5b6f627..2c0a7be 100644 --- a/src/org/torproject/ernie/web/TableParameterChecker.java +++ b/src/org/torproject/ernie/web/TableParameterChecker.java @@ -41,6 +41,7 @@ public class TableParameterChecker { this.availableTables = new HashMap<String, String>(); this.availableTables.put("direct-users", "start,end,filename"); this.availableTables.put("censorship-events", "start,end,filename"); + this.availableTables.put("top-10-relays", "start,end,filename"); this.knownParameterValues = new HashMap<String, String>(); } diff --git a/web/WEB-INF/network.jsp b/web/WEB-INF/network.jsp index 1789cb7..9d83cf1 100644 --- a/web/WEB-INF/network.jsp +++ b/web/WEB-INF/network.jsp @@ -1,5 +1,6 @@ <%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core" %> <%@ taglib prefix="fn" uri="http://java.sun.com/jsp/jstl/functions" %> +<%@ taglib prefix="fmt" uri="http://java.sun.com/jsp/jstl/fmt" %> <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> <html> <head> @@ -230,6 +231,36 @@ in the network.</p> </form> <p><a href="csv/bandwidth.csv">CSV</a> file containing all data.</p> <br> +<a name="top-10-relays"></a> +<p><b>Top-10 relays by bandwidth history:</b></p> +<table> + <tr> + <th>Fingerprint</th> + <th>Bandwidth history</th> + </tr> + <c:forEach var="row" items="${top_10_relays_tabledata}"> + <tr> + <td><a href="https://metrics.torproject.org/relay.html?fingerprint=${row['fingerprint']}">${row['fingerprint']}</a>&emsp;</td> + <td><fmt:formatNumber type="number" minFractionDigits="2" value="${row['written_sum']}" /> TiB</td> + </tr> + </c:forEach> +</table> +<br> +<form action="network.html#top-10-relays"> + <div class="formrow"> + <input type="hidden" name="table" value="top-10-relays"> + <p> + <label>Start date (yyyy-mm-dd):</label> + <input type="text" name="start" size="10" + value="<c:choose><c:when test="${fn:length(top_10_relays_start) == 0}">${default_start_date}</c:when><c:otherwise>${top_10_relays_start[0]}</c:otherwise></c:choose>"> + <label>End date (yyyy-mm-dd):</label> + <input type="text" name="end" size="10" + value="<c:choose><c:when test="${fn:length(top_10_relays_end) == 0}">${default_end_date}</c:when><c:otherwise>${top_10_relays_end[0]}</c:otherwise></c:choose>"> + </p><p> + <input class="submit" type="submit" value="Update table"> + </p> + </div> +</form> <h3>Relay bandwidth by Exit and/or Guard flags</h3> <br>

1 0

[metrics-tasks/master] Fix missing comma in #3338 R script.
by karsten＠torproject.org 29 Jul '11

29 Jul '11

commit ddbaf5b9ea92a39fc1dda896708009da12cd0af5 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jul 29 07:33:26 2011 -0400 Fix missing comma in #3338 R script. --- task-3338/daily-users.R | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/task-3338/daily-users.R b/task-3338/daily-users.R index 58921cc..dc3583f 100644 --- a/task-3338/daily-users.R +++ b/task-3338/daily-users.R @@ -14,7 +14,7 @@ dbpassword= "password" ###### <- put in real password here! plot_dirbytes <- function(start, end, path, dpi) { drv <- dbDriver("PostgreSQL") con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) - q <- paste("SELECT date, r, bwp, brp, bwn, brn, bwr, brr " + q <- paste("SELECT date, r, bwp, brp, bwn, brn, bwr, brr ", "FROM user_stats WHERE date >= '", start, "' AND date <= '", end, "' AND date < (SELECT MAX(date) FROM user_stats) - 1 ", "AND country = 'zy' ORDER BY date", sep = "")

1 0

[metrics-web/master] Add top-10 country by possible censorship events.
by karsten＠torproject.org 29 Jul '11

29 Jul '11

commit f3044b212baab32583db8ce107796d1b2e2a705b Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Jul 28 23:00:17 2011 -0400 Add top-10 country by possible censorship events. Implements some more of #2718. --- rserve/tables.R | 39 ++++++++++++++++++++ .../ernie/web/GraphsSubpagesServlet.java | 3 +- .../ernie/web/TableParameterChecker.java | 1 + web/WEB-INF/users.jsp | 37 ++++++++++++++++++- 4 files changed, 78 insertions(+), 2 deletions(-) diff --git a/rserve/tables.R b/rserve/tables.R index 725bc31..6d3b5fb 100644 --- a/rserve/tables.R +++ b/rserve/tables.R @@ -30,3 +30,42 @@ write_direct_users <- function(start, end, path) { write.csv(d, path, quote = FALSE, row.names = FALSE) } +write_censorship_events <- function(start, end, path) { + drv <- dbDriver("PostgreSQL") + con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) + q <- paste("SELECT date, country, r, bwp, brn, bwn, brp, bwr, brr ", + "FROM user_stats WHERE date >= '", start, "' AND date <= '", end, + "' AND date < (SELECT MAX(date) FROM user_stats) - 1", + sep = "") + rs <- dbSendQuery(con, q) + u <- fetch(rs, n = -1) + dbDisconnect(con) + dbUnloadDriver(drv) + u <- data.frame(date = u$date, country = u$country, + users = u$r * (u$bwp * u$brn / u$bwn - u$brp) / + (u$bwr * u$brn / u$bwn - u$brr) / 10) + dates <- seq(from = as.Date(start, "%Y-%m-%d"), + to = as.Date(end, "%Y-%m-%d"), by="1 day") + missing <- setdiff(dates, u$date) + r <- read.csv( + "/srv/metrics.torproject.org/web/detector/direct-users-ranges.csv", + stringsAsFactors = FALSE) + r <- r[r$date >= start & r$date <= end, + c("date", "country", "minusers", "maxusers")] + r <- cast(rbind(melt(u, id.vars = c("date", "country")), + melt(r, id.vars = c("date", "country")))) + r <- na.omit(r[r$users < r$minusers | r$users > r$maxusers, ]) + r <- data.frame(date = r$date, country = r$country, + upturn = ifelse(r$users > r$maxusers, 1, 0), + downturn = ifelse(r$users < r$minusers, 1, 0)) + r <- aggregate(r[, c("upturn", "downturn")], + by = list(country = r$country), sum) + r <- r[order(r$downturn, r$upturn, decreasing = TRUE), ] + r <- r[1:10, ] + r <- data.frame(cc = r$country, + country = sub('the ', '', countrynames(as.character(r$country))), + downturns = r$downturn, + upturns = r$upturn) + write.csv(r, path, quote = FALSE, row.names = FALSE) +} + diff --git a/src/org/torproject/ernie/web/GraphsSubpagesServlet.java b/src/org/torproject/ernie/web/GraphsSubpagesServlet.java index e3845cf..c7f3056 100644 --- a/src/org/torproject/ernie/web/GraphsSubpagesServlet.java +++ b/src/org/torproject/ernie/web/GraphsSubpagesServlet.java @@ -34,7 +34,8 @@ public class GraphsSubpagesServlet extends HttpServlet { this.availableGraphsSubpageTables = new HashMap<String, Set<String>>(); this.availableGraphsSubpageTables.put("users.html", - new HashSet<String>(Arrays.asList("direct-users".split(",")))); + new HashSet<String>(Arrays.asList( + "direct-users,censorship-events".split(",")))); this.knownCountries = Countries.getInstance().getCountryList(); } diff --git a/src/org/torproject/ernie/web/TableParameterChecker.java b/src/org/torproject/ernie/web/TableParameterChecker.java index 9bf1c33..5b6f627 100644 --- a/src/org/torproject/ernie/web/TableParameterChecker.java +++ b/src/org/torproject/ernie/web/TableParameterChecker.java @@ -40,6 +40,7 @@ public class TableParameterChecker { this.availableTables = new HashMap<String, String>(); this.availableTables.put("direct-users", "start,end,filename"); + this.availableTables.put("censorship-events", "start,end,filename"); this.knownParameterValues = new HashMap<String, String>(); } diff --git a/web/WEB-INF/users.jsp b/web/WEB-INF/users.jsp index b378af1..bb4e11c 100644 --- a/web/WEB-INF/users.jsp +++ b/web/WEB-INF/users.jsp @@ -24,6 +24,7 @@ out of a few hundred directory mirrors to save bandwidth of the directory authorities. The following graphs show an estimate of recurring Tor users based on the requests seen by a few dozen directory mirrors.</p> <a name="direct-users"></a> +<p><b>Daily directly connecting users:</b></p> <img src="direct-users.png${direct_users_url}" width="576" height="360" alt="Direct users graph"> <form action="users.html#direct-users"> @@ -60,6 +61,7 @@ based on the requests seen by a few dozen directory mirrors.</p> </div> </form> <a name="direct-users-table"></a> +<p><b>Top-10 countries by directly connecting users:</b></p> <table> <tr> <th>Country</th> @@ -88,7 +90,40 @@ based on the requests seen by a few dozen directory mirrors.</p> </p> </div> </form> -<p><a href="csv/direct-users.csv">CSV</a> file containing all data.</p> +<a name="censorship-events"></a> +<p><b>Top-10 countries by possible censorship events (BETA):</b></p> +<table> + <tr> + <th>Country</th> + <th>Downturns</th> + <th>Upturns</th> + </tr> + <c:forEach var="row" items="${censorship_events_tabledata}"> + <tr> + <td><a href="users.html?graph=direct-users&country=${row['cc']}&events=on#direct-users">${row['country']}</a>&emsp;</td> + <td>${row['downturns']}</td> + <td>${row['upturns']}</td> + </tr> + </c:forEach> +</table> +<br> +<form action="users.html#censorship-events"> + <div class="formrow"> + <input type="hidden" name="table" value="censorship-events"> + <p> + <label>Start date (yyyy-mm-dd):</label> + <input type="text" name="start" size="10" + value="<c:choose><c:when test="${fn:length(censorship_events_start) == 0}">${default_start_date}</c:when><c:otherwise>${censorship_events_start[0]}</c:otherwise></c:choose>"> + <label>End date (yyyy-mm-dd):</label> + <input type="text" name="end" size="10" + value="<c:choose><c:when test="${fn:length(censorship_events_end) == 0}">${default_end_date}</c:when><c:otherwise>${censorship_events_end[0]}</c:otherwise></c:choose>"> + </p><p> + <input class="submit" type="submit" value="Update table"> + </p> + </div> +</form> +<p><a href="csv/direct-users.csv">CSV</a> file containing daily directly +connecting users by country.</p> <p><a href="csv/monthly-users-peak.csv">CSV</a> file containing peak daily Tor users (direct and bridge) per month by country.</p> <p><a href="csv/monthly-users-average.csv">CSV</a> file containing average

1 0

[metrics-web/master] Add table with top-10 countries to users.html.
by karsten＠torproject.org 29 Jul '11

29 Jul '11

commit d2078417458e63ded2bdbb28e061d3b28605f95d Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Jul 28 21:37:49 2011 -0400 Add table with top-10 countries to users.html. Implements #3624. --- rserve/rserve-init.R | 2 + rserve/tables.R | 32 ++++++ .../ernie/web/GraphsSubpagesServlet.java | 91 +++++++++++++++- src/org/torproject/ernie/web/RObjectGenerator.java | 70 +++++++++++- .../ernie/web/TableParameterChecker.java | 117 ++++++++++++++++++++ web/WEB-INF/users.jsp | 30 +++++ 6 files changed, 338 insertions(+), 4 deletions(-) diff --git a/rserve/rserve-init.R b/rserve/rserve-init.R index bafb5ba..8344705 100644 --- a/rserve/rserve-init.R +++ b/rserve/rserve-init.R @@ -15,3 +15,5 @@ dbpassword= "" source('graphs.R') source('csv.R') +source('tables.R') + diff --git a/rserve/tables.R b/rserve/tables.R new file mode 100644 index 0000000..725bc31 --- /dev/null +++ b/rserve/tables.R @@ -0,0 +1,32 @@ +countrynames <- function(countries) { + sapply(countries, countryname) +} + +write_direct_users <- function(start, end, path) { + drv <- dbDriver("PostgreSQL") + con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) + q <- paste("SELECT date, country, r, bwp, brn, bwn, brp, bwr, brr ", + "FROM user_stats WHERE date >= '", start, "' AND date <= '", end, + "' AND date < (SELECT MAX(date) FROM user_stats) - 1 ", + "ORDER BY date, country", sep = "") + rs <- dbSendQuery(con, q) + u <- fetch(rs, n = -1) + dbDisconnect(con) + dbUnloadDriver(drv) + d <- data.frame(date = u$date, country = u$country, + directusers = floor(u$r * (u$bwp * u$brn / u$bwn - u$brp) / + (u$bwr * u$brn / u$bwn - u$brr) / 10)) + d <- aggregate(d$directusers, by = list(country = d$country), mean) + total <- d[d$country == "zy", "x"] + d <- d[d$country != "zy", ] + d <- data.frame(country = d$country, directusers = d$x) + d <- d[order(d$directusers, decreasing = TRUE), ] + d <- d[1:10, ] + d <- data.frame( + cc = as.character(d$country), + country = sub('the ', '', countrynames(as.character(d$country))), + abs = round(d$directusers), + rel = round(100 * d$directusers / total, 2)) + write.csv(d, path, quote = FALSE, row.names = FALSE) +} + diff --git a/src/org/torproject/ernie/web/GraphsSubpagesServlet.java b/src/org/torproject/ernie/web/GraphsSubpagesServlet.java index f8dfba4..e3845cf 100644 --- a/src/org/torproject/ernie/web/GraphsSubpagesServlet.java +++ b/src/org/torproject/ernie/web/GraphsSubpagesServlet.java @@ -12,9 +12,15 @@ public class GraphsSubpagesServlet extends HttpServlet { * are forwarded. */ private Map<String, String> availableGraphsSubpages; + /* Available tables on graphs subpages. */ + private Map<String, Set<String>> availableGraphsSubpageTables; + /* Country codes and names for per-country graphs. */ private List<String[]> knownCountries; + /* R object generator for generating table data. */ + private RObjectGenerator rObjectGenerator; + public GraphsSubpagesServlet() { this.availableGraphsSubpages = new HashMap<String, String>(); this.availableGraphsSubpages.put("network.html", @@ -25,9 +31,21 @@ public class GraphsSubpagesServlet extends HttpServlet { this.availableGraphsSubpages.put("performance.html", "WEB-INF/performance.jsp"); + this.availableGraphsSubpageTables = + new HashMap<String, Set<String>>(); + this.availableGraphsSubpageTables.put("users.html", + new HashSet<String>(Arrays.asList("direct-users".split(",")))); + this.knownCountries = Countries.getInstance().getCountryList(); } + public void init() { + /* Get a reference to the R object generator that we need to generate + * table data. */ + this.rObjectGenerator = (RObjectGenerator) getServletContext(). + getAttribute("RObjectGenerator"); + } + public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { @@ -48,8 +66,9 @@ public class GraphsSubpagesServlet extends HttpServlet { } String jsp = availableGraphsSubpages.get(requestedPage); - /* Find out which graph type was requested, if any. */ + /* Find out which graph or table type was requested, if any. */ String requestedGraph = request.getParameter("graph"); + String requestedTable = request.getParameter("table"); if (requestedGraph != null) { /* Check if the passed parameters are valid. */ @@ -74,6 +93,76 @@ public class GraphsSubpagesServlet extends HttpServlet { url); } } + if (requestedTable != null) { + + /* Check if the passed parameters are valid. */ + Map<String, String[]> checkedParameters = TableParameterChecker. + getInstance().checkParameters(requestedTable, + request.getParameterMap()); + if (checkedParameters != null) { + + /* Set the table's attributes to the appropriate values, so that + * we can prepopulate the form. */ + for (Map.Entry<String, String[]> param : + checkedParameters.entrySet()) { + request.setAttribute(requestedTable.replaceAll("-", "_") + "_" + + param.getKey(), param.getValue()); + } + } + } + + /* Trigger generation of table data if the graphs subpage has any + * tables, regardless of whether a table update was requested. */ + if (this.availableGraphsSubpageTables.containsKey(requestedPage)) { + for (String tableName : + this.availableGraphsSubpageTables.get(requestedPage)) { + + Map<String, String[]> checkedParameters = null; + if (tableName.equals(requestedTable)) { + checkedParameters = TableParameterChecker. + getInstance().checkParameters(requestedTable, + request.getParameterMap()); + } else { + checkedParameters = TableParameterChecker. + getInstance().checkParameters(tableName, null); + } + + /* Prepare filename and R query string. */ + StringBuilder rQueryBuilder = new StringBuilder("write_" + + tableName.replaceAll("-", "_") + "("), + tableFilenameBuilder = new StringBuilder(tableName); + + for (Map.Entry<String, String[]> parameter : + checkedParameters.entrySet()) { + String parameterName = parameter.getKey(); + String[] parameterValues = parameter.getValue(); + for (String param : parameterValues) { + tableFilenameBuilder.append("-" + param); + } + if (parameterValues.length < 2) { + rQueryBuilder.append(parameterName + " = '" + + parameterValues[0] + "', "); + } else { + rQueryBuilder.append(parameterName + " = c("); + for (int i = 0; i < parameterValues.length - 1; i++) { + rQueryBuilder.append("'" + parameterValues[i] + "', "); + } + rQueryBuilder.append("'" + parameterValues[ + parameterValues.length - 1] + "'), "); + } + } + tableFilenameBuilder.append(".tbl"); + String tableFilename = tableFilenameBuilder.toString(); + rQueryBuilder.append("path = '%s')"); + String rQuery = rQueryBuilder.toString(); + + /* Generate table data and add it as request attribute. */ + List<Map<String, String>> tableData = rObjectGenerator. + generateTable(rQuery, tableFilename); + request.setAttribute(tableName.replaceAll("-", "_") + + "_tabledata", tableData); + } + } /* Pass list of known countries in case we want to display them. */ request.setAttribute("countries", this.knownCountries); diff --git a/src/org/torproject/ernie/web/RObjectGenerator.java b/src/org/torproject/ernie/web/RObjectGenerator.java index 7f152dd..10aa3b8 100644 --- a/src/org/torproject/ernie/web/RObjectGenerator.java +++ b/src/org/torproject/ernie/web/RObjectGenerator.java @@ -55,9 +55,10 @@ public class RObjectGenerator implements ServletContextListener { if (!imageFile.exists() || imageFile.lastModified() < now - this.maxCacheAge * 1000L) { - /* We do. Update the R query to contain the absolute path to the file - * to be generated, create a connection to Rserve, run the R query, - * and close the connection. The generated graph will be on disk. */ + /* We do. Update the R query to contain the absolute path to the + * file to be generated, create a connection to Rserve, run the R + * query, and close the connection. The generated graph will be on + * disk. */ rQuery = String.format(rQuery, imageFile.getAbsolutePath()); try { RConnection rc = new RConnection(rserveHost, rservePort); @@ -135,5 +136,68 @@ public class RObjectGenerator implements ServletContextListener { /* Return the csv file. */ return result; } + + /* Generate table data using the given R query and filename or read + * previously generated table data from disk if it's not too old and + * return table data. */ + public List<Map<String, String>> generateTable(String rQuery, + String tableFilename) { + + /* See if we need to generate this table. */ + File tableFile = new File(this.cachedGraphsDirectory + "/" + + tableFilename); + long now = System.currentTimeMillis(); + if (!tableFile.exists() || tableFile.lastModified() < now + - this.maxCacheAge * 1000L) { + + /* We do. Update the R query to contain the absolute path to the + * file to be generated, create a connection to Rserve, run the R + * query, and close the connection. The generated csv file will be + * on disk in the same directory as the generated graphs. */ + rQuery = String.format(rQuery, tableFile.getAbsolutePath()); + try { + RConnection rc = new RConnection(rserveHost, rservePort); + rc.eval(rQuery); + rc.close(); + } catch (RserveException e) { + return null; + } + + /* Check that we really just generated the file */ + if (!tableFile.exists() || tableFile.lastModified() < now + - this.maxCacheAge * 1000L) { + return null; + } + } + + /* Read the text file from disk and write the table content to a + * map. */ + List<Map<String, String>> result = null; + try { + result = new ArrayList<Map<String, String>>(); + BufferedReader br = new BufferedReader(new FileReader(tableFile)); + String line = br.readLine(); + if (line != null) { + List<String> headers = new ArrayList<String>(Arrays.asList( + line.split(","))); + while ((line = br.readLine()) != null) { + String[] parts = line.split(","); + if (headers.size() != parts.length) { + return null; + } + Map<String, String> row = new HashMap<String, String>(); + for (int i = 0; i < headers.size(); i++) { + row.put(headers.get(i), parts[i]); + } + result.add(row); + } + } + } catch (IOException e) { + return null; + } + + /* Return table values. */ + return result; + } } diff --git a/src/org/torproject/ernie/web/TableParameterChecker.java b/src/org/torproject/ernie/web/TableParameterChecker.java new file mode 100644 index 0000000..9bf1c33 --- /dev/null +++ b/src/org/torproject/ernie/web/TableParameterChecker.java @@ -0,0 +1,117 @@ +package org.torproject.ernie.web; + +import java.text.*; +import java.util.*; +import java.util.regex.*; + +/** + * Checks request parameters passed to generate tables. + */ +public class TableParameterChecker { + + /** + * Singleton instance of this class. + */ + private static TableParameterChecker instance = + new TableParameterChecker(); + + /** + * Returns the singleton instance of this class. + */ + public static TableParameterChecker getInstance() { + return instance; + } + + /* Date format for parsing start and end dates. */ + private SimpleDateFormat dateFormat; + + /* Available tables with corresponding parameter lists. */ + private Map<String, String> availableTables; + + /* Known parameters and parameter values. */ + private Map<String, String> knownParameterValues; + + /** + * Initializes map with valid parameters for each of the graphs. + */ + public TableParameterChecker() { + this.dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + this.dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + + this.availableTables = new HashMap<String, String>(); + this.availableTables.put("direct-users", "start,end,filename"); + + this.knownParameterValues = new HashMap<String, String>(); + } + + /** + * Checks request parameters for the given table type and returns a map + * of recognized parameters, or null if the table type doesn't exist or + * the parameters are invalid. + */ + public Map<String, String[]> checkParameters(String tableType, + Map requestParameters) { + + /* Check if the graph type exists. */ + if (tableType == null || + !this.availableTables.containsKey(tableType)) { + return null; + } + + /* Find out which other parameters are supported by this table type + * and parse them if they are given. */ + Set<String> supportedTableParameters = new HashSet<String>(Arrays. + asList(this.availableTables.get(tableType).split(","))); + Map<String, String[]> recognizedTableParameters = + new HashMap<String, String[]>(); + + /* Parse start and end dates if supported by the table type. If no end + * date is provided, set it to today. If no start date is provided, + * set it to 90 days before the end date. Make sure that start date + * precedes end date. */ + if (supportedTableParameters.contains("start") || + supportedTableParameters.contains("end")) { + String[] startParameter = null; + String[] endParameter = null; + if (requestParameters != null) { + startParameter = (String[]) requestParameters.get("start"); + endParameter = (String[]) requestParameters.get("end"); + } + long endTimestamp = System.currentTimeMillis(); + if (endParameter != null && endParameter.length > 0 && + endParameter[0].length() > 0) { + try { + endTimestamp = dateFormat.parse(endParameter[0]).getTime(); + } catch (ParseException e) { + return null; + } + if (!endParameter[0].startsWith("20")) { + return null; + } + } + endParameter = new String[] { dateFormat.format(endTimestamp) }; + long startTimestamp = endTimestamp - 90L * 24L * 60L * 60L * 1000L; + if (startParameter != null && startParameter.length > 0 && + startParameter[0].length() > 0) { + try { + startTimestamp = dateFormat.parse(startParameter[0]).getTime(); + } catch (ParseException e) { + return null; + } + if (!startParameter[0].startsWith("20")) { + return null; + } + } + startParameter = new String[] { dateFormat.format(startTimestamp) }; + if (startTimestamp > endTimestamp) { + return null; + } + recognizedTableParameters.put("start", startParameter); + recognizedTableParameters.put("end", endParameter); + } + + /* We now have a map with all required table parameters. Return it. */ + return recognizedTableParameters; + } +} + diff --git a/web/WEB-INF/users.jsp b/web/WEB-INF/users.jsp index 814c15c..b378af1 100644 --- a/web/WEB-INF/users.jsp +++ b/web/WEB-INF/users.jsp @@ -1,5 +1,6 @@ <%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core" %> <%@ taglib prefix="fn" uri="http://java.sun.com/jsp/jstl/functions" %> +<%@ taglib prefix="fmt" uri="http://java.sun.com/jsp/jstl/fmt" %> <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> <html> <head> @@ -58,6 +59,35 @@ based on the requests seen by a few dozen directory mirrors.</p> </p> </div> </form> +<a name="direct-users-table"></a> +<table> + <tr> + <th>Country</th> + <th>Mean daily users</th> + </tr> + <c:forEach var="row" items="${direct_users_tabledata}"> + <tr> + <td><a href="users.html?graph=direct-users&country=${row['cc']}#direct-users">${row['country']}</a>&emsp;</td> + <td>${row['abs']} (<fmt:formatNumber type="number" minFractionDigits="2" value="${row['rel']}" /> %)</td> + </tr> + </c:forEach> +</table> +<br> +<form action="users.html#direct-users-table"> + <div class="formrow"> + <input type="hidden" name="table" value="direct-users"> + <p> + <label>Start date (yyyy-mm-dd):</label> + <input type="text" name="start" size="10" + value="<c:choose><c:when test="${fn:length(direct_users_start) == 0}">${default_start_date}</c:when><c:otherwise>${direct_users_start[0]}</c:otherwise></c:choose>"> + <label>End date (yyyy-mm-dd):</label> + <input type="text" name="end" size="10" + value="<c:choose><c:when test="${fn:length(direct_users_end) == 0}">${default_end_date}</c:when><c:otherwise>${direct_users_end[0]}</c:otherwise></c:choose>"> + </p><p> + <input class="submit" type="submit" value="Update table"> + </p> + </div> +</form> <p><a href="csv/direct-users.csv">CSV</a> file containing all data.</p> <p><a href="csv/monthly-users-peak.csv">CSV</a> file containing peak daily Tor users (direct and bridge) per month by country.</p>

1 0