- tor-commits - lists.torproject.org

[metrics-web/master] Add table with top-10 relays by bandwidth.
by karsten＠torproject.org 30 Jul '11

30 Jul '11

commit e988175f5056c2b4f4eb74bc0d8b8df26886eb2c Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jul 29 20:41:04 2011 -0400 Add table with top-10 relays by bandwidth. --- rserve/tables.R | 16 ++++++++++ .../ernie/web/GraphsSubpagesServlet.java | 3 ++ .../ernie/web/TableParameterChecker.java | 1 + web/WEB-INF/network.jsp | 31 ++++++++++++++++++++ 4 files changed, 51 insertions(+), 0 deletions(-) diff --git a/rserve/tables.R b/rserve/tables.R index 6d3b5fb..aabd588 100644 --- a/rserve/tables.R +++ b/rserve/tables.R @@ -69,3 +69,19 @@ write_censorship_events <- function(start, end, path) { write.csv(r, path, quote = FALSE, row.names = FALSE) } +write_top_10_relays <- function(start, end, path) { + drv <- dbDriver("PostgreSQL") + con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) + q <- paste("SELECT fingerprint, SUM(written_sum) ", + "AS written_sum FROM bwhist WHERE date >= '", start, + "' AND date <= '", end, "' GROUP BY 1 ORDER BY 2 DESC LIMIT 10", + sep = "") + rs <- dbSendQuery(con, q) + c <- fetch(rs, n = -1) + dbDisconnect(con) + dbUnloadDriver(drv) + c <- data.frame(fingerprint = c$fingerprint, + written_sum = round(c$written_sum / 2^40, 2)) + write.csv(c, path, quote = FALSE, row.names = FALSE) +} + diff --git a/src/org/torproject/ernie/web/GraphsSubpagesServlet.java b/src/org/torproject/ernie/web/GraphsSubpagesServlet.java index c7f3056..16d0db0 100644 --- a/src/org/torproject/ernie/web/GraphsSubpagesServlet.java +++ b/src/org/torproject/ernie/web/GraphsSubpagesServlet.java @@ -36,6 +36,9 @@ public class GraphsSubpagesServlet extends HttpServlet { this.availableGraphsSubpageTables.put("users.html", new HashSet<String>(Arrays.asList( "direct-users,censorship-events".split(",")))); + this.availableGraphsSubpageTables.put("network.html", + new HashSet<String>(Arrays.asList( + "top-10-relays".split(",")))); this.knownCountries = Countries.getInstance().getCountryList(); } diff --git a/src/org/torproject/ernie/web/TableParameterChecker.java b/src/org/torproject/ernie/web/TableParameterChecker.java index 5b6f627..2c0a7be 100644 --- a/src/org/torproject/ernie/web/TableParameterChecker.java +++ b/src/org/torproject/ernie/web/TableParameterChecker.java @@ -41,6 +41,7 @@ public class TableParameterChecker { this.availableTables = new HashMap<String, String>(); this.availableTables.put("direct-users", "start,end,filename"); this.availableTables.put("censorship-events", "start,end,filename"); + this.availableTables.put("top-10-relays", "start,end,filename"); this.knownParameterValues = new HashMap<String, String>(); } diff --git a/web/WEB-INF/network.jsp b/web/WEB-INF/network.jsp index 1789cb7..9d83cf1 100644 --- a/web/WEB-INF/network.jsp +++ b/web/WEB-INF/network.jsp @@ -1,5 +1,6 @@ <%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core" %> <%@ taglib prefix="fn" uri="http://java.sun.com/jsp/jstl/functions" %> +<%@ taglib prefix="fmt" uri="http://java.sun.com/jsp/jstl/fmt" %> <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> <html> <head> @@ -230,6 +231,36 @@ in the network.</p> </form> <p><a href="csv/bandwidth.csv">CSV</a> file containing all data.</p> <br> +<a name="top-10-relays"></a> +<p><b>Top-10 relays by bandwidth history:</b></p> +<table> + <tr> + <th>Fingerprint</th> + <th>Bandwidth history</th> + </tr> + <c:forEach var="row" items="${top_10_relays_tabledata}"> + <tr> + <td><a href="https://metrics.torproject.org/relay.html?fingerprint=${row['fingerprint']}">${row['fingerprint']}</a>&emsp;</td> + <td><fmt:formatNumber type="number" minFractionDigits="2" value="${row['written_sum']}" /> TiB</td> + </tr> + </c:forEach> +</table> +<br> +<form action="network.html#top-10-relays"> + <div class="formrow"> + <input type="hidden" name="table" value="top-10-relays"> + <p> + <label>Start date (yyyy-mm-dd):</label> + <input type="text" name="start" size="10" + value="<c:choose><c:when test="${fn:length(top_10_relays_start) == 0}">${default_start_date}</c:when><c:otherwise>${top_10_relays_start[0]}</c:otherwise></c:choose>"> + <label>End date (yyyy-mm-dd):</label> + <input type="text" name="end" size="10" + value="<c:choose><c:when test="${fn:length(top_10_relays_end) == 0}">${default_end_date}</c:when><c:otherwise>${top_10_relays_end[0]}</c:otherwise></c:choose>"> + </p><p> + <input class="submit" type="submit" value="Update table"> + </p> + </div> +</form> <h3>Relay bandwidth by Exit and/or Guard flags</h3> <br>

1 0

[metrics-tasks/master] Fix missing comma in #3338 R script.
by karsten＠torproject.org 29 Jul '11

29 Jul '11

commit ddbaf5b9ea92a39fc1dda896708009da12cd0af5 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jul 29 07:33:26 2011 -0400 Fix missing comma in #3338 R script. --- task-3338/daily-users.R | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/task-3338/daily-users.R b/task-3338/daily-users.R index 58921cc..dc3583f 100644 --- a/task-3338/daily-users.R +++ b/task-3338/daily-users.R @@ -14,7 +14,7 @@ dbpassword= "password" ###### <- put in real password here! plot_dirbytes <- function(start, end, path, dpi) { drv <- dbDriver("PostgreSQL") con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) - q <- paste("SELECT date, r, bwp, brp, bwn, brn, bwr, brr " + q <- paste("SELECT date, r, bwp, brp, bwn, brn, bwr, brr ", "FROM user_stats WHERE date >= '", start, "' AND date <= '", end, "' AND date < (SELECT MAX(date) FROM user_stats) - 1 ", "AND country = 'zy' ORDER BY date", sep = "")

1 0

[metrics-web/master] Add top-10 country by possible censorship events.
by karsten＠torproject.org 29 Jul '11

29 Jul '11

commit f3044b212baab32583db8ce107796d1b2e2a705b Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Jul 28 23:00:17 2011 -0400 Add top-10 country by possible censorship events. Implements some more of #2718. --- rserve/tables.R | 39 ++++++++++++++++++++ .../ernie/web/GraphsSubpagesServlet.java | 3 +- .../ernie/web/TableParameterChecker.java | 1 + web/WEB-INF/users.jsp | 37 ++++++++++++++++++- 4 files changed, 78 insertions(+), 2 deletions(-) diff --git a/rserve/tables.R b/rserve/tables.R index 725bc31..6d3b5fb 100644 --- a/rserve/tables.R +++ b/rserve/tables.R @@ -30,3 +30,42 @@ write_direct_users <- function(start, end, path) { write.csv(d, path, quote = FALSE, row.names = FALSE) } +write_censorship_events <- function(start, end, path) { + drv <- dbDriver("PostgreSQL") + con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) + q <- paste("SELECT date, country, r, bwp, brn, bwn, brp, bwr, brr ", + "FROM user_stats WHERE date >= '", start, "' AND date <= '", end, + "' AND date < (SELECT MAX(date) FROM user_stats) - 1", + sep = "") + rs <- dbSendQuery(con, q) + u <- fetch(rs, n = -1) + dbDisconnect(con) + dbUnloadDriver(drv) + u <- data.frame(date = u$date, country = u$country, + users = u$r * (u$bwp * u$brn / u$bwn - u$brp) / + (u$bwr * u$brn / u$bwn - u$brr) / 10) + dates <- seq(from = as.Date(start, "%Y-%m-%d"), + to = as.Date(end, "%Y-%m-%d"), by="1 day") + missing <- setdiff(dates, u$date) + r <- read.csv( + "/srv/metrics.torproject.org/web/detector/direct-users-ranges.csv", + stringsAsFactors = FALSE) + r <- r[r$date >= start & r$date <= end, + c("date", "country", "minusers", "maxusers")] + r <- cast(rbind(melt(u, id.vars = c("date", "country")), + melt(r, id.vars = c("date", "country")))) + r <- na.omit(r[r$users < r$minusers | r$users > r$maxusers, ]) + r <- data.frame(date = r$date, country = r$country, + upturn = ifelse(r$users > r$maxusers, 1, 0), + downturn = ifelse(r$users < r$minusers, 1, 0)) + r <- aggregate(r[, c("upturn", "downturn")], + by = list(country = r$country), sum) + r <- r[order(r$downturn, r$upturn, decreasing = TRUE), ] + r <- r[1:10, ] + r <- data.frame(cc = r$country, + country = sub('the ', '', countrynames(as.character(r$country))), + downturns = r$downturn, + upturns = r$upturn) + write.csv(r, path, quote = FALSE, row.names = FALSE) +} + diff --git a/src/org/torproject/ernie/web/GraphsSubpagesServlet.java b/src/org/torproject/ernie/web/GraphsSubpagesServlet.java index e3845cf..c7f3056 100644 --- a/src/org/torproject/ernie/web/GraphsSubpagesServlet.java +++ b/src/org/torproject/ernie/web/GraphsSubpagesServlet.java @@ -34,7 +34,8 @@ public class GraphsSubpagesServlet extends HttpServlet { this.availableGraphsSubpageTables = new HashMap<String, Set<String>>(); this.availableGraphsSubpageTables.put("users.html", - new HashSet<String>(Arrays.asList("direct-users".split(",")))); + new HashSet<String>(Arrays.asList( + "direct-users,censorship-events".split(",")))); this.knownCountries = Countries.getInstance().getCountryList(); } diff --git a/src/org/torproject/ernie/web/TableParameterChecker.java b/src/org/torproject/ernie/web/TableParameterChecker.java index 9bf1c33..5b6f627 100644 --- a/src/org/torproject/ernie/web/TableParameterChecker.java +++ b/src/org/torproject/ernie/web/TableParameterChecker.java @@ -40,6 +40,7 @@ public class TableParameterChecker { this.availableTables = new HashMap<String, String>(); this.availableTables.put("direct-users", "start,end,filename"); + this.availableTables.put("censorship-events", "start,end,filename"); this.knownParameterValues = new HashMap<String, String>(); } diff --git a/web/WEB-INF/users.jsp b/web/WEB-INF/users.jsp index b378af1..bb4e11c 100644 --- a/web/WEB-INF/users.jsp +++ b/web/WEB-INF/users.jsp @@ -24,6 +24,7 @@ out of a few hundred directory mirrors to save bandwidth of the directory authorities. The following graphs show an estimate of recurring Tor users based on the requests seen by a few dozen directory mirrors.</p> <a name="direct-users"></a> +<p><b>Daily directly connecting users:</b></p> <img src="direct-users.png${direct_users_url}" width="576" height="360" alt="Direct users graph"> <form action="users.html#direct-users"> @@ -60,6 +61,7 @@ based on the requests seen by a few dozen directory mirrors.</p> </div> </form> <a name="direct-users-table"></a> +<p><b>Top-10 countries by directly connecting users:</b></p> <table> <tr> <th>Country</th> @@ -88,7 +90,40 @@ based on the requests seen by a few dozen directory mirrors.</p> </p> </div> </form> -<p><a href="csv/direct-users.csv">CSV</a> file containing all data.</p> +<a name="censorship-events"></a> +<p><b>Top-10 countries by possible censorship events (BETA):</b></p> +<table> + <tr> + <th>Country</th> + <th>Downturns</th> + <th>Upturns</th> + </tr> + <c:forEach var="row" items="${censorship_events_tabledata}"> + <tr> + <td><a href="users.html?graph=direct-users&country=${row['cc']}&events=on#direct-users">${row['country']}</a>&emsp;</td> + <td>${row['downturns']}</td> + <td>${row['upturns']}</td> + </tr> + </c:forEach> +</table> +<br> +<form action="users.html#censorship-events"> + <div class="formrow"> + <input type="hidden" name="table" value="censorship-events"> + <p> + <label>Start date (yyyy-mm-dd):</label> + <input type="text" name="start" size="10" + value="<c:choose><c:when test="${fn:length(censorship_events_start) == 0}">${default_start_date}</c:when><c:otherwise>${censorship_events_start[0]}</c:otherwise></c:choose>"> + <label>End date (yyyy-mm-dd):</label> + <input type="text" name="end" size="10" + value="<c:choose><c:when test="${fn:length(censorship_events_end) == 0}">${default_end_date}</c:when><c:otherwise>${censorship_events_end[0]}</c:otherwise></c:choose>"> + </p><p> + <input class="submit" type="submit" value="Update table"> + </p> + </div> +</form> +<p><a href="csv/direct-users.csv">CSV</a> file containing daily directly +connecting users by country.</p> <p><a href="csv/monthly-users-peak.csv">CSV</a> file containing peak daily Tor users (direct and bridge) per month by country.</p> <p><a href="csv/monthly-users-average.csv">CSV</a> file containing average

1 0

[metrics-web/master] Add table with top-10 countries to users.html.
by karsten＠torproject.org 29 Jul '11

29 Jul '11

commit d2078417458e63ded2bdbb28e061d3b28605f95d Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Jul 28 21:37:49 2011 -0400 Add table with top-10 countries to users.html. Implements #3624. --- rserve/rserve-init.R | 2 + rserve/tables.R | 32 ++++++ .../ernie/web/GraphsSubpagesServlet.java | 91 +++++++++++++++- src/org/torproject/ernie/web/RObjectGenerator.java | 70 +++++++++++- .../ernie/web/TableParameterChecker.java | 117 ++++++++++++++++++++ web/WEB-INF/users.jsp | 30 +++++ 6 files changed, 338 insertions(+), 4 deletions(-) diff --git a/rserve/rserve-init.R b/rserve/rserve-init.R index bafb5ba..8344705 100644 --- a/rserve/rserve-init.R +++ b/rserve/rserve-init.R @@ -15,3 +15,5 @@ dbpassword= "" source('graphs.R') source('csv.R') +source('tables.R') + diff --git a/rserve/tables.R b/rserve/tables.R new file mode 100644 index 0000000..725bc31 --- /dev/null +++ b/rserve/tables.R @@ -0,0 +1,32 @@ +countrynames <- function(countries) { + sapply(countries, countryname) +} + +write_direct_users <- function(start, end, path) { + drv <- dbDriver("PostgreSQL") + con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) + q <- paste("SELECT date, country, r, bwp, brn, bwn, brp, bwr, brr ", + "FROM user_stats WHERE date >= '", start, "' AND date <= '", end, + "' AND date < (SELECT MAX(date) FROM user_stats) - 1 ", + "ORDER BY date, country", sep = "") + rs <- dbSendQuery(con, q) + u <- fetch(rs, n = -1) + dbDisconnect(con) + dbUnloadDriver(drv) + d <- data.frame(date = u$date, country = u$country, + directusers = floor(u$r * (u$bwp * u$brn / u$bwn - u$brp) / + (u$bwr * u$brn / u$bwn - u$brr) / 10)) + d <- aggregate(d$directusers, by = list(country = d$country), mean) + total <- d[d$country == "zy", "x"] + d <- d[d$country != "zy", ] + d <- data.frame(country = d$country, directusers = d$x) + d <- d[order(d$directusers, decreasing = TRUE), ] + d <- d[1:10, ] + d <- data.frame( + cc = as.character(d$country), + country = sub('the ', '', countrynames(as.character(d$country))), + abs = round(d$directusers), + rel = round(100 * d$directusers / total, 2)) + write.csv(d, path, quote = FALSE, row.names = FALSE) +} + diff --git a/src/org/torproject/ernie/web/GraphsSubpagesServlet.java b/src/org/torproject/ernie/web/GraphsSubpagesServlet.java index f8dfba4..e3845cf 100644 --- a/src/org/torproject/ernie/web/GraphsSubpagesServlet.java +++ b/src/org/torproject/ernie/web/GraphsSubpagesServlet.java @@ -12,9 +12,15 @@ public class GraphsSubpagesServlet extends HttpServlet { * are forwarded. */ private Map<String, String> availableGraphsSubpages; + /* Available tables on graphs subpages. */ + private Map<String, Set<String>> availableGraphsSubpageTables; + /* Country codes and names for per-country graphs. */ private List<String[]> knownCountries; + /* R object generator for generating table data. */ + private RObjectGenerator rObjectGenerator; + public GraphsSubpagesServlet() { this.availableGraphsSubpages = new HashMap<String, String>(); this.availableGraphsSubpages.put("network.html", @@ -25,9 +31,21 @@ public class GraphsSubpagesServlet extends HttpServlet { this.availableGraphsSubpages.put("performance.html", "WEB-INF/performance.jsp"); + this.availableGraphsSubpageTables = + new HashMap<String, Set<String>>(); + this.availableGraphsSubpageTables.put("users.html", + new HashSet<String>(Arrays.asList("direct-users".split(",")))); + this.knownCountries = Countries.getInstance().getCountryList(); } + public void init() { + /* Get a reference to the R object generator that we need to generate + * table data. */ + this.rObjectGenerator = (RObjectGenerator) getServletContext(). + getAttribute("RObjectGenerator"); + } + public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { @@ -48,8 +66,9 @@ public class GraphsSubpagesServlet extends HttpServlet { } String jsp = availableGraphsSubpages.get(requestedPage); - /* Find out which graph type was requested, if any. */ + /* Find out which graph or table type was requested, if any. */ String requestedGraph = request.getParameter("graph"); + String requestedTable = request.getParameter("table"); if (requestedGraph != null) { /* Check if the passed parameters are valid. */ @@ -74,6 +93,76 @@ public class GraphsSubpagesServlet extends HttpServlet { url); } } + if (requestedTable != null) { + + /* Check if the passed parameters are valid. */ + Map<String, String[]> checkedParameters = TableParameterChecker. + getInstance().checkParameters(requestedTable, + request.getParameterMap()); + if (checkedParameters != null) { + + /* Set the table's attributes to the appropriate values, so that + * we can prepopulate the form. */ + for (Map.Entry<String, String[]> param : + checkedParameters.entrySet()) { + request.setAttribute(requestedTable.replaceAll("-", "_") + "_" + + param.getKey(), param.getValue()); + } + } + } + + /* Trigger generation of table data if the graphs subpage has any + * tables, regardless of whether a table update was requested. */ + if (this.availableGraphsSubpageTables.containsKey(requestedPage)) { + for (String tableName : + this.availableGraphsSubpageTables.get(requestedPage)) { + + Map<String, String[]> checkedParameters = null; + if (tableName.equals(requestedTable)) { + checkedParameters = TableParameterChecker. + getInstance().checkParameters(requestedTable, + request.getParameterMap()); + } else { + checkedParameters = TableParameterChecker. + getInstance().checkParameters(tableName, null); + } + + /* Prepare filename and R query string. */ + StringBuilder rQueryBuilder = new StringBuilder("write_" + + tableName.replaceAll("-", "_") + "("), + tableFilenameBuilder = new StringBuilder(tableName); + + for (Map.Entry<String, String[]> parameter : + checkedParameters.entrySet()) { + String parameterName = parameter.getKey(); + String[] parameterValues = parameter.getValue(); + for (String param : parameterValues) { + tableFilenameBuilder.append("-" + param); + } + if (parameterValues.length < 2) { + rQueryBuilder.append(parameterName + " = '" + + parameterValues[0] + "', "); + } else { + rQueryBuilder.append(parameterName + " = c("); + for (int i = 0; i < parameterValues.length - 1; i++) { + rQueryBuilder.append("'" + parameterValues[i] + "', "); + } + rQueryBuilder.append("'" + parameterValues[ + parameterValues.length - 1] + "'), "); + } + } + tableFilenameBuilder.append(".tbl"); + String tableFilename = tableFilenameBuilder.toString(); + rQueryBuilder.append("path = '%s')"); + String rQuery = rQueryBuilder.toString(); + + /* Generate table data and add it as request attribute. */ + List<Map<String, String>> tableData = rObjectGenerator. + generateTable(rQuery, tableFilename); + request.setAttribute(tableName.replaceAll("-", "_") + + "_tabledata", tableData); + } + } /* Pass list of known countries in case we want to display them. */ request.setAttribute("countries", this.knownCountries); diff --git a/src/org/torproject/ernie/web/RObjectGenerator.java b/src/org/torproject/ernie/web/RObjectGenerator.java index 7f152dd..10aa3b8 100644 --- a/src/org/torproject/ernie/web/RObjectGenerator.java +++ b/src/org/torproject/ernie/web/RObjectGenerator.java @@ -55,9 +55,10 @@ public class RObjectGenerator implements ServletContextListener { if (!imageFile.exists() || imageFile.lastModified() < now - this.maxCacheAge * 1000L) { - /* We do. Update the R query to contain the absolute path to the file - * to be generated, create a connection to Rserve, run the R query, - * and close the connection. The generated graph will be on disk. */ + /* We do. Update the R query to contain the absolute path to the + * file to be generated, create a connection to Rserve, run the R + * query, and close the connection. The generated graph will be on + * disk. */ rQuery = String.format(rQuery, imageFile.getAbsolutePath()); try { RConnection rc = new RConnection(rserveHost, rservePort); @@ -135,5 +136,68 @@ public class RObjectGenerator implements ServletContextListener { /* Return the csv file. */ return result; } + + /* Generate table data using the given R query and filename or read + * previously generated table data from disk if it's not too old and + * return table data. */ + public List<Map<String, String>> generateTable(String rQuery, + String tableFilename) { + + /* See if we need to generate this table. */ + File tableFile = new File(this.cachedGraphsDirectory + "/" + + tableFilename); + long now = System.currentTimeMillis(); + if (!tableFile.exists() || tableFile.lastModified() < now + - this.maxCacheAge * 1000L) { + + /* We do. Update the R query to contain the absolute path to the + * file to be generated, create a connection to Rserve, run the R + * query, and close the connection. The generated csv file will be + * on disk in the same directory as the generated graphs. */ + rQuery = String.format(rQuery, tableFile.getAbsolutePath()); + try { + RConnection rc = new RConnection(rserveHost, rservePort); + rc.eval(rQuery); + rc.close(); + } catch (RserveException e) { + return null; + } + + /* Check that we really just generated the file */ + if (!tableFile.exists() || tableFile.lastModified() < now + - this.maxCacheAge * 1000L) { + return null; + } + } + + /* Read the text file from disk and write the table content to a + * map. */ + List<Map<String, String>> result = null; + try { + result = new ArrayList<Map<String, String>>(); + BufferedReader br = new BufferedReader(new FileReader(tableFile)); + String line = br.readLine(); + if (line != null) { + List<String> headers = new ArrayList<String>(Arrays.asList( + line.split(","))); + while ((line = br.readLine()) != null) { + String[] parts = line.split(","); + if (headers.size() != parts.length) { + return null; + } + Map<String, String> row = new HashMap<String, String>(); + for (int i = 0; i < headers.size(); i++) { + row.put(headers.get(i), parts[i]); + } + result.add(row); + } + } + } catch (IOException e) { + return null; + } + + /* Return table values. */ + return result; + } } diff --git a/src/org/torproject/ernie/web/TableParameterChecker.java b/src/org/torproject/ernie/web/TableParameterChecker.java new file mode 100644 index 0000000..9bf1c33 --- /dev/null +++ b/src/org/torproject/ernie/web/TableParameterChecker.java @@ -0,0 +1,117 @@ +package org.torproject.ernie.web; + +import java.text.*; +import java.util.*; +import java.util.regex.*; + +/** + * Checks request parameters passed to generate tables. + */ +public class TableParameterChecker { + + /** + * Singleton instance of this class. + */ + private static TableParameterChecker instance = + new TableParameterChecker(); + + /** + * Returns the singleton instance of this class. + */ + public static TableParameterChecker getInstance() { + return instance; + } + + /* Date format for parsing start and end dates. */ + private SimpleDateFormat dateFormat; + + /* Available tables with corresponding parameter lists. */ + private Map<String, String> availableTables; + + /* Known parameters and parameter values. */ + private Map<String, String> knownParameterValues; + + /** + * Initializes map with valid parameters for each of the graphs. + */ + public TableParameterChecker() { + this.dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + this.dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + + this.availableTables = new HashMap<String, String>(); + this.availableTables.put("direct-users", "start,end,filename"); + + this.knownParameterValues = new HashMap<String, String>(); + } + + /** + * Checks request parameters for the given table type and returns a map + * of recognized parameters, or null if the table type doesn't exist or + * the parameters are invalid. + */ + public Map<String, String[]> checkParameters(String tableType, + Map requestParameters) { + + /* Check if the graph type exists. */ + if (tableType == null || + !this.availableTables.containsKey(tableType)) { + return null; + } + + /* Find out which other parameters are supported by this table type + * and parse them if they are given. */ + Set<String> supportedTableParameters = new HashSet<String>(Arrays. + asList(this.availableTables.get(tableType).split(","))); + Map<String, String[]> recognizedTableParameters = + new HashMap<String, String[]>(); + + /* Parse start and end dates if supported by the table type. If no end + * date is provided, set it to today. If no start date is provided, + * set it to 90 days before the end date. Make sure that start date + * precedes end date. */ + if (supportedTableParameters.contains("start") || + supportedTableParameters.contains("end")) { + String[] startParameter = null; + String[] endParameter = null; + if (requestParameters != null) { + startParameter = (String[]) requestParameters.get("start"); + endParameter = (String[]) requestParameters.get("end"); + } + long endTimestamp = System.currentTimeMillis(); + if (endParameter != null && endParameter.length > 0 && + endParameter[0].length() > 0) { + try { + endTimestamp = dateFormat.parse(endParameter[0]).getTime(); + } catch (ParseException e) { + return null; + } + if (!endParameter[0].startsWith("20")) { + return null; + } + } + endParameter = new String[] { dateFormat.format(endTimestamp) }; + long startTimestamp = endTimestamp - 90L * 24L * 60L * 60L * 1000L; + if (startParameter != null && startParameter.length > 0 && + startParameter[0].length() > 0) { + try { + startTimestamp = dateFormat.parse(startParameter[0]).getTime(); + } catch (ParseException e) { + return null; + } + if (!startParameter[0].startsWith("20")) { + return null; + } + } + startParameter = new String[] { dateFormat.format(startTimestamp) }; + if (startTimestamp > endTimestamp) { + return null; + } + recognizedTableParameters.put("start", startParameter); + recognizedTableParameters.put("end", endParameter); + } + + /* We now have a map with all required table parameters. Return it. */ + return recognizedTableParameters; + } +} + diff --git a/web/WEB-INF/users.jsp b/web/WEB-INF/users.jsp index 814c15c..b378af1 100644 --- a/web/WEB-INF/users.jsp +++ b/web/WEB-INF/users.jsp @@ -1,5 +1,6 @@ <%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core" %> <%@ taglib prefix="fn" uri="http://java.sun.com/jsp/jstl/functions" %> +<%@ taglib prefix="fmt" uri="http://java.sun.com/jsp/jstl/fmt" %> <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> <html> <head> @@ -58,6 +59,35 @@ based on the requests seen by a few dozen directory mirrors.</p> </p> </div> </form> +<a name="direct-users-table"></a> +<table> + <tr> + <th>Country</th> + <th>Mean daily users</th> + </tr> + <c:forEach var="row" items="${direct_users_tabledata}"> + <tr> + <td><a href="users.html?graph=direct-users&country=${row['cc']}#direct-users">${row['country']}</a>&emsp;</td> + <td>${row['abs']} (<fmt:formatNumber type="number" minFractionDigits="2" value="${row['rel']}" /> %)</td> + </tr> + </c:forEach> +</table> +<br> +<form action="users.html#direct-users-table"> + <div class="formrow"> + <input type="hidden" name="table" value="direct-users"> + <p> + <label>Start date (yyyy-mm-dd):</label> + <input type="text" name="start" size="10" + value="<c:choose><c:when test="${fn:length(direct_users_start) == 0}">${default_start_date}</c:when><c:otherwise>${direct_users_start[0]}</c:otherwise></c:choose>"> + <label>End date (yyyy-mm-dd):</label> + <input type="text" name="end" size="10" + value="<c:choose><c:when test="${fn:length(direct_users_end) == 0}">${default_end_date}</c:when><c:otherwise>${direct_users_end[0]}</c:otherwise></c:choose>"> + </p><p> + <input class="submit" type="submit" value="Update table"> + </p> + </div> +</form> <p><a href="csv/direct-users.csv">CSV</a> file containing all data.</p> <p><a href="csv/monthly-users-peak.csv">CSV</a> file containing peak daily Tor users (direct and bridge) per month by country.</p>

1 0

r24930: {website} update links to vidalia packages (website/trunk/projects/en)
by Runa Sandvik 28 Jul '11

28 Jul '11

Author: runa Date: 2011-07-28 13:56:00 +0000 (Thu, 28 Jul 2011) New Revision: 24930 Modified: website/trunk/projects/en/vidalia.wml Log: update links to vidalia packages Modified: website/trunk/projects/en/vidalia.wml =================================================================== --- website/trunk/projects/en/vidalia.wml 2011-07-25 14:57:17 UTC (rev 24929) +++ website/trunk/projects/en/vidalia.wml 2011-07-28 13:56:00 UTC (rev 24930) @@ -58,16 +58,16 @@ <p> The most recent stable release is: 0.2.12</p> <ul> <li> - <a href="../dist/vidalia-bundles/vidalia-bundle-0.2.2.24-alpha-0.2.12.exe">Windows Installer</a> - (<a href="../dist/vidalia-bundles/vidalia-bundle-0.2.2.24-alpha-0.2.12.exe.asc">sig</a>) + <a href="../dist/vidalia-bundles/vidalia-bundle-0.2.2.30-rc-0.2.12.exe">Windows Installer</a> + (<a href="../dist/vidalia-bundles/vidalia-bundle-0.2.2.30-rc-0.2.12.exe.asc">sig</a>) </li> <li> - <a href="../dist/vidalia-bundles/vidalia-bundle-0.2.2.24-alpha-0.2.12-i386.dmg">Mac OS X x86 Only</a> - (<a href="../dist/vidalia-bundles/vidalia-bundle-0.2.2.24-alpha-0.2.12-i386.dmg.asc">sig</a>) + <a href="../dist/vidalia-bundles/vidalia-bundle-0.2.2.30-rc-0.2.12-i386.dmg">Mac OS X x86 Only</a> + (<a href="../dist/vidalia-bundles/vidalia-bundle-0.2.2.30-rc-0.2.12-i386.dmg.asc">sig</a>) </li> <li> - <a href="../dist/vidalia-bundles/vidalia-bundle-0.2.2.24-alpha-0.2.12-ppc.dmg">Mac OS X PPC Only</a> - (<a href="../dist/vidalia-bundles/vidalia-bundle-0.2.2.24-alpha-0.2.12-ppc.dmg.asc">sig</a>) + <a href="../dist/vidalia-bundles/vidalia-bundle-0.2.2.30-rc-0.2.12-i386.dmg">Mac OS X PPC Only</a> + (<a href="../dist/vidalia-bundles/vidalia-bundle-0.2.2.30-rc-0.2.12-i386.dmg.asc">sig</a>) </li> <li> <a href="<page docs/debian-vidalia>">Instructions for Debian/Ubuntu/Knoppix Repositories</a>

1 0

r24929: {} Add some photographs of whiteboard scribblings (in projects/misc/rransom: . 2011 2011/dev-meeting-photos-2011-07-25-01 2011/dev-meeting-photos-2011-07-25-01/orig)
by Robert Ransom 25 Jul '11

25 Jul '11

Author: rransom Date: 2011-07-25 14:57:17 +0000 (Mon, 25 Jul 2011) New Revision: 24929 Added: projects/misc/rransom/2011/ projects/misc/rransom/2011/dev-meeting-photos-2011-07-25-01/ projects/misc/rransom/2011/dev-meeting-photos-2011-07-25-01/orig/ projects/misc/rransom/2011/dev-meeting-photos-2011-07-25-01/orig/100_2363.jpg projects/misc/rransom/2011/dev-meeting-photos-2011-07-25-01/orig/100_2364.jpg projects/misc/rransom/2011/dev-meeting-photos-2011-07-25-01/orig/100_2365.jpg projects/misc/rransom/2011/dev-meeting-photos-2011-07-25-01/orig/100_2368.jpg projects/misc/rransom/2011/dev-meeting-photos-2011-07-25-01/orig/100_2369.jpg projects/misc/rransom/2011/dev-meeting-photos-2011-07-25-01/orig/100_2372.jpg Log: Add some photographs of whiteboard scribblings Added: projects/misc/rransom/2011/dev-meeting-photos-2011-07-25-01/orig/100_2363.jpg =================================================================== (Binary files differ) Property changes on: projects/misc/rransom/2011/dev-meeting-photos-2011-07-25-01/orig/100_2363.jpg ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: projects/misc/rransom/2011/dev-meeting-photos-2011-07-25-01/orig/100_2364.jpg =================================================================== (Binary files differ) Property changes on: projects/misc/rransom/2011/dev-meeting-photos-2011-07-25-01/orig/100_2364.jpg ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: projects/misc/rransom/2011/dev-meeting-photos-2011-07-25-01/orig/100_2365.jpg =================================================================== (Binary files differ) Property changes on: projects/misc/rransom/2011/dev-meeting-photos-2011-07-25-01/orig/100_2365.jpg ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: projects/misc/rransom/2011/dev-meeting-photos-2011-07-25-01/orig/100_2368.jpg =================================================================== (Binary files differ) Property changes on: projects/misc/rransom/2011/dev-meeting-photos-2011-07-25-01/orig/100_2368.jpg ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: projects/misc/rransom/2011/dev-meeting-photos-2011-07-25-01/orig/100_2369.jpg =================================================================== (Binary files differ) Property changes on: projects/misc/rransom/2011/dev-meeting-photos-2011-07-25-01/orig/100_2369.jpg ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: projects/misc/rransom/2011/dev-meeting-photos-2011-07-25-01/orig/100_2372.jpg =================================================================== (Binary files differ) Property changes on: projects/misc/rransom/2011/dev-meeting-photos-2011-07-25-01/orig/100_2372.jpg ___________________________________________________________________ Added: svn:mime-type + application/octet-stream

1 0

r24928: {} Create a directory for rransom's miscellany (projects/misc)
by Robert Ransom 25 Jul '11

25 Jul '11

Author: rransom Date: 2011-07-25 14:41:30 +0000 (Mon, 25 Jul 2011) New Revision: 24928 Added: projects/misc/rransom/ Log: Create a directory for rransom's miscellany

1 0

[flashproxy/master] Remove new-window from remote commands.
by dcf＠torproject.org 24 Jul '11

24 Jul '11

commit 950b999f5abbf38cc70551ba1e22b877e87df772 Author: David Fifield <david(a)bamsoftware.com> Date: Sat Jul 23 02:26:05 2011 +0000 Remove new-window from remote commands. This is necessary when using the browser.link.open_newwindow=1 setting, but the default setting of 3 has the pages open in new tabs. --- experiments/01 | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/experiments/01 b/experiments/01 index 0c273e7..ff0179d 100755 --- a/experiments/01 +++ b/experiments/01 @@ -42,14 +42,14 @@ echo "starting flash proxy client side..." i=1 until [ $i -gt $NUM_CLIENTS ]; do let local_port=9000+i - firefox --remote "openURL(http://localhost:1080/swfcat.swf?local=127.0.0.1:${local_port}&client=1&facilitator=127.0.0.1:4701,new-window)" + firefox --remote "openURL(http://localhost:1080/swfcat.swf?local=127.0.0.1:${local_port}&client=1&facilitator=127.0.0.1:4701)" let i=i+1 sleep 2 done # Start the flash proxy, server side echo "starting flash proxy server side..." -firefox --remote "openURL(http://localhost:1080/swfcat.swf?facilitator=127.0.0.1:4701,new-window)" +firefox --remote "openURL(http://localhost:1080/swfcat.swf?facilitator=127.0.0.1:4701)" let query_fac_time=NUM_CLIENTS+2 sleep ${query_fac_time}

1 0

[flashproxy/master] Allow relaying data while one end of a ProxyPair is disconnected.
by dcf＠torproject.org 24 Jul '11

24 Jul '11

commit ea1618074e82e45542fbf2be79334b2517eadbd5 Author: David Fifield <david(a)bamsoftware.com> Date: Sat Jul 23 04:44:12 2011 +0000 Allow relaying data while one end of a ProxyPair is disconnected. Data could be left stranded when one end became closed after data had been read but before it had been flushed to the other side. --- ProxyPair.as | 49 +++++++++++++++++++++++++++++++++++++++---------- 1 files changed, 39 insertions(+), 10 deletions(-) diff --git a/ProxyPair.as b/ProxyPair.as index 10c4263..237ab69 100644 --- a/ProxyPair.as +++ b/ProxyPair.as @@ -86,13 +86,13 @@ package public function connect():void { s_r.addEventListener(Event.CONNECT, relay_connected); - s_r.addEventListener(Event.CLOSE, socket_error("Relay: closed", s_c)); + s_r.addEventListener(Event.CLOSE, relay_closed); s_r.addEventListener(IOErrorEvent.IO_ERROR, socket_error("Relay: I/O error", s_c)); s_r.addEventListener(SecurityErrorEvent.SECURITY_ERROR, socket_error("Relay: security error", s_c)); s_r.addEventListener(ProgressEvent.SOCKET_DATA, relay_to_client); s_c.addEventListener(Event.CONNECT, client_connected); - s_c.addEventListener(Event.CLOSE, socket_error("Client: closed", s_r)); + s_c.addEventListener(Event.CLOSE, client_closed); s_c.addEventListener(IOErrorEvent.IO_ERROR, socket_error("Client: I/O error", s_r)); s_c.addEventListener(SecurityErrorEvent.SECURITY_ERROR, socket_error("Client: security error", s_r)); s_c.addEventListener(ProgressEvent.SOCKET_DATA, client_to_relay); @@ -113,6 +113,18 @@ package log("Client: connected."); } + private function relay_closed(e:Event):void + { + log("Relay: closed."); + flush(); + } + + private function client_closed(e:Event):void + { + log("Client: closed."); + flush(); + } + private function relay_to_client(e:ProgressEvent):void { r2c_schedule.push(e.bytesLoaded); @@ -141,24 +153,41 @@ package /* Send as much data as the rate limit currently allows. */ private function flush():void { + var busy:Boolean; + if (flush_id) clearTimeout(flush_id); flush_id = undefined; - if (!(s_r.connected && s_c.connected)) - /* Can't do anything until both sockets are connected. */ + if (!s_r.connected && !s_c.connected) + /* Can't do anything while both sockets are disconnected. */ return; - while (!ui.rate_limit.is_limited() && - (r2c_schedule.length > 0 || c2r_schedule.length > 0)) { - if (r2c_schedule.length > 0) + busy = true; + while (busy && !ui.rate_limit.is_limited()) { + busy = false; + if (s_c.connected && r2c_schedule.length > 0) { transfer_chunk(s_r, s_c, r2c_schedule.shift(), "Relay"); - if (c2r_schedule.length > 0) + busy = true; + } + if (s_r.connected && c2r_schedule.length > 0) { transfer_chunk(s_c, s_r, c2r_schedule.shift(), "Client"); + busy = true; + } + } + + if (!s_r.connected && r2c_schedule.length == 0) { + log("Client: closing."); + s_c.close(); + } + if (!s_c.connected && c2r_schedule.length == 0) { + log("Relay: closing."); + s_r.close(); } - /* Call again when safe, if necessary. */ - if (r2c_schedule.length > 0 || c2r_schedule.length > 0) + if (!s_c.connected && !s_r.connected) + dispatchEvent(new Event(Event.COMPLETE)); + else if (r2c_schedule.length > 0 || c2r_schedule.length > 0) flush_id = setTimeout(flush, ui.rate_limit.when() * 1000); } }

1 0

[flashproxy/master] Make sure a firefox instance exists to begin with.
by dcf＠torproject.org 24 Jul '11

24 Jul '11

commit 29d773a87bec8344a10d7631dae365d4c8e2aec9 Author: David Fifield <david(a)bamsoftware.com> Date: Sat Jul 23 06:06:38 2011 +0000 Make sure a firefox instance exists to begin with. --- experiments/01 | 12 +++++++----- 1 files changed, 7 insertions(+), 5 deletions(-) diff --git a/experiments/01 b/experiments/01 index 1ffadaa..8718e0d 100755 --- a/experiments/01 +++ b/experiments/01 @@ -37,6 +37,12 @@ until [ $i -gt $NUM_CLIENTS ]; do let i=i+1 done +# Start the flash proxy, server side +echo "starting flash proxy server side..." +firefox & +sleep 2 +firefox --remote "openURL(http://localhost:1080/swfcat.swf?facilitator=127.0.0.1:4701)" + # Start the flash proxy, client side echo "starting flash proxy client side..." i=1 @@ -47,10 +53,6 @@ until [ $i -gt $NUM_CLIENTS ]; do sleep 2 done -# Start the flash proxy, server side -echo "starting flash proxy server side..." -firefox --remote "openURL(http://localhost:1080/swfcat.swf?facilitator=127.0.0.1:4701)" - let query_fac_time=NUM_CLIENTS+2 sleep ${query_fac_time} @@ -59,7 +61,7 @@ echo "getting the dump file..." i=2 until [ $i -gt $NUM_CLIENTS ]; do let local_port=2000+i - gnome-terminal -e "/bin/bash -c 'wget -t1 http://localhost:${local_port}/dump -O - > /dev/null; exec /bin/bash -i'" + xterm -e "/bin/bash -c 'wget -t1 http://localhost:${local_port}/dump -O - > /dev/null; exec /bin/bash -i'" & let i=i+1 sleep 2 done

1 0