commit 0189c896586e67e994cb0774b51515e115f8e8f2 Author: Karsten Loesing karsten.loesing@gmx.net Date: Tue Apr 26 14:14:09 2011 +0200

Add graph on bandwidth by Exit and/or Guard flags. --- db/tordir.sql | 36 ++++++++++++++++ etc/web.xml | 4 ++ rserve/csv.R | 14 ++++++ rserve/graphs.R | 44 ++++++++++++++++++++ src/org/torproject/ernie/web/CsvServlet.java | 1 + .../ernie/web/GraphParameterChecker.java | 1 + web/WEB-INF/network.jsp | 31 ++++++++++++++ 7 files changed, 131 insertions(+), 0 deletions(-)

diff --git a/db/tordir.sql b/db/tordir.sql index 8efba09..cfe5907 100644 --- a/db/tordir.sql +++ b/db/tordir.sql @@ -196,6 +196,16 @@ CREATE TABLE total_bwhist ( CONSTRAINT total_bwhist_pkey PRIMARY KEY(date) );

+-- TABLE bwhist_flags +CREATE TABLE bwhist_flags ( + date DATE NOT NULL, + isexit BOOLEAN NOT NULL, + isguard BOOLEAN NOT NULL, + read BIGINT, + written BIGINT, + CONSTRAINT bwhist_flags_pkey PRIMARY KEY(date, isexit, isguard) +); + -- TABLE user_stats -- Aggregate statistics on directory requests and byte histories that we -- use to estimate user numbers. @@ -591,6 +601,31 @@ CREATE OR REPLACE FUNCTION refresh_total_bwhist() RETURNS INTEGER AS $$ END; $$ LANGUAGE plpgsql;

+CREATE OR REPLACE FUNCTION refresh_bwhist_flags() RETURNS INTEGER AS $$ + BEGIN + DELETE FROM bwhist_flags WHERE date IN (SELECT date FROM updates); + INSERT INTO bwhist_flags (date, isexit, isguard, read_write_avg) + SELECT a.date, isexit, isguard, SUM(read_sum) as read, + SUM(written_sum) AS written + FROM + (SELECT DATE(validafter) AS date, + fingerprint, + BOOL_OR(isexit) AS isexit, + BOOL_OR(isguard) AS isguard + FROM statusentry + WHERE isrunning = TRUE + AND DATE(validafter) >= (SELECT MIN(date) FROM updates) + AND DATE(validafter) <= (SELECT MAX(date) FROM updates) + AND DATE(validafter) IN (SELECT date FROM updates) + GROUP BY 1, 2) a + JOIN bwhist + ON a.date = bwhist.date + AND a.fingerprint = bwhist.fingerprint + GROUP BY 1, 2, 3; + RETURN 1; + END; +$$ LANGUAGE plpgsql; + -- FUNCTION refresh_user_stats() -- This function refreshes our user statistics by weighting reported -- directory request statistics of directory mirrors with bandwidth @@ -823,6 +858,7 @@ CREATE OR REPLACE FUNCTION refresh_all() RETURNS INTEGER AS $$ PERFORM refresh_relay_versions(); PERFORM refresh_total_bandwidth(); PERFORM refresh_total_bwhist(); + PERFORM refresh_bwhist_flags(); PERFORM refresh_user_stats(); DELETE FROM scheduled_updates WHERE id IN (SELECT id FROM updates); RETURN 1; diff --git a/etc/web.xml b/etc/web.xml index a9b36bc..688e6f3 100644 --- a/etc/web.xml +++ b/etc/web.xml @@ -197,6 +197,10 @@ </servlet-mapping> <servlet-mapping> <servlet-name>GraphImage</servlet-name> + <url-pattern>/bwhist-flags.png</url-pattern> + </servlet-mapping> + <servlet-mapping> + <servlet-name>GraphImage</servlet-name> <url-pattern>/dirbytes.png</url-pattern> </servlet-mapping> <servlet-mapping> diff --git a/rserve/csv.R b/rserve/csv.R index 37a0856..17a59c2 100644 --- a/rserve/csv.R +++ b/rserve/csv.R @@ -90,6 +90,20 @@ export_bandwidth <- function(path) { write.csv(bandwidth, path, quote = FALSE, row.names = FALSE) }

+export_bwhist_flags <- function(path) { + drv <- dbDriver("PostgreSQL") + con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) + q <- paste("SELECT date, isexit, isguard, read, written", + "FROM bwhist_flags", + "WHERE date < (SELECT MAX(date) FROM bwhist_flags) - 1", + "ORDER BY date, isexit, isguard") + rs <- dbSendQuery(con, q) + bw <- fetch(rs, n = -1) + dbDisconnect(con) + dbUnloadDriver(drv) + write.csv(bw, path, quote = FALSE, row.names = FALSE) +} + export_dirbytes <- function(path) { drv <- dbDriver("PostgreSQL") con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) diff --git a/rserve/graphs.R b/rserve/graphs.R index 15daf02..f268a34 100644 --- a/rserve/graphs.R +++ b/rserve/graphs.R @@ -443,6 +443,50 @@ plot_bandwidth <- function(start, end, path, dpi) { ggsave(filename = path, width = 8, height = 5, dpi = as.numeric(dpi)) }

+plot_bwhist_flags <- function(start, end, path, dpi) { + drv <- dbDriver("PostgreSQL") + con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) + q <- paste("SELECT date, isexit, isguard, read, written ", + "FROM bwhist_flags WHERE date >= '", start, "' AND date <= '", end, + "' AND date < (SELECT MAX(date) FROM bwhist_flags) - 1 ", sep = "") + rs <- dbSendQuery(con, q) + bw <- fetch(rs, n = -1) + dbDisconnect(con) + dbUnloadDriver(drv) + dates <- seq(from = as.Date(start, "%Y-%m-%d"), + to = as.Date(end, "%Y-%m-%d"), by = "1 day") + missing <- setdiff(dates, as.Date(bw$date, origin = "1970-01-01")) + if (length(missing) > 0) + bw <- rbind(bw, + data.frame(date = as.Date(missing, origin = "1970-01-01"), + isexit = FALSE, isguard = FALSE, read = NA, written = NA), + data.frame(date = as.Date(missing, origin = "1970-01-01"), + isexit = FALSE, isguard = TRUE, read = NA, written = NA), + data.frame(date = as.Date(missing, origin = "1970-01-01"), + isexit = TRUE, isguard = FALSE, read = NA, written = NA), + data.frame(date = as.Date(missing, origin = "1970-01-01"), + isexit = TRUE, isguard = TRUE, read = NA, written = NA)) + bw <- data.frame(date = bw$date, variable = ifelse(bw$isexit, + ifelse(bw$isguard, "Guard & Exit", "Exit only"), + ifelse(bw$isguard, "Guard only", "Middle only")), + value = (bw$read + bw$written) / 2) + ggplot(bw, aes(x = as.Date(date, "%Y-%m-%d"), y = value / 2^20 / 86400, + colour = variable)) + + geom_line(size = 1) + + scale_x_date(name = paste("\nThe Tor Project - ", + "https://metrics.torproject.org/", sep = ""), format = + c("%d-%b", "%d-%b", "%b-%Y", "%b-%Y", "%Y", "%Y")[ + cut(as.numeric(max(as.Date(bw$date, "%Y-%m-%d")) - + min(as.Date(bw$date, "%Y-%m-%d"))), + c(0, 10, 56, 365, 730, 5000, Inf), labels=FALSE)]) + + scale_y_continuous(name="Bandwidth (MiB/s)", + limits = c(0, max(bw$value, na.rm = TRUE) / 2^20 / 86400)) + + scale_colour_hue(name = "") + + opts(title = "Bandwidth history by relay flags", + legend.position = "top") + ggsave(filename = path, width = 8, height = 5, dpi = as.numeric(dpi)) +} + plot_dirbytes <- function(start, end, path, dpi) { drv <- dbDriver("PostgreSQL") con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) diff --git a/src/org/torproject/ernie/web/CsvServlet.java b/src/org/torproject/ernie/web/CsvServlet.java index 0997f31..0140fa9 100644 --- a/src/org/torproject/ernie/web/CsvServlet.java +++ b/src/org/torproject/ernie/web/CsvServlet.java @@ -31,6 +31,7 @@ public class CsvServlet extends HttpServlet { this.availableCsvFiles = new HashSet<String>(); this.availableCsvFiles.add("bandwidth"); this.availableCsvFiles.add("bridge-users"); + this.availableCsvFiles.add("bwhist-flags"); this.availableCsvFiles.add("connbidirect"); this.availableCsvFiles.add("current-platform-strings"); this.availableCsvFiles.add("direct-users"); diff --git a/src/org/torproject/ernie/web/GraphParameterChecker.java b/src/org/torproject/ernie/web/GraphParameterChecker.java index 308afe5..95e4cf0 100644 --- a/src/org/torproject/ernie/web/GraphParameterChecker.java +++ b/src/org/torproject/ernie/web/GraphParameterChecker.java @@ -47,6 +47,7 @@ public class GraphParameterChecker { this.availableGraphs.put("versions", "start,end,filename,dpi"); this.availableGraphs.put("platforms", "start,end,filename,dpi"); this.availableGraphs.put("bandwidth", "start,end,filename,dpi"); + this.availableGraphs.put("bwhist-flags", "start,end,filename,dpi"); this.availableGraphs.put("dirbytes", "start,end,filename,dpi"); this.availableGraphs.put("direct-users", "start,end,country,filename,dpi"); diff --git a/web/WEB-INF/network.jsp b/web/WEB-INF/network.jsp index a1a5efe..7906789 100644 --- a/web/WEB-INF/network.jsp +++ b/web/WEB-INF/network.jsp @@ -227,6 +227,37 @@ in the network.</p> <p><a href="csv/bandwidth.csv">CSV</a> file containing all data.</p> <br>

+<h3>Relay bandwidth by Exit and/or Guard flags</h3> +<br> +<p>The following graph shows the relay bandwidth of all relays with the +Exit and/or Guard flags assigned by the directory authorities.</p> +<a name="bwhist-flags"></a> +<img src="bwhist-flags.png${bwhist_flags_url}" + width="576" height="360" alt="Relay bandwidth by flags graph"> +<form action="network.html#bwhist-flags"> + <div class="formrow"> + <input type="hidden" name="graph" value="bwhist-flags"> + <p> + <label>Start date (yyyy-mm-dd):</label> + <input type="text" name="start" size="10" + value="<c:choose><c:when test="${fn:length(bwhist_flags_start) == 0}">${default_start_date}</c:when><c:otherwise>${bwhist_flags_start[0]}</c:otherwise></c:choose>"> + <label>End date (yyyy-mm-dd):</label> + <input type="text" name="end" size="10" + value="<c:choose><c:when test="${fn:length(bwhist_flags_end) == 0}">${default_end_date}</c:when><c:otherwise>${bwhist_flags_end[0]}</c:otherwise></c:choose>"> + </p><p> + Resolution: <select name="dpi"> + <option value="72"<c:if test="${bwhist_flags_dpi[0] eq '72'}"> selected</c:if>>Screen - 576x360</option> + <option value="150"<c:if test="${bwhist_flags_dpi[0] eq '150'}"> selected</c:if>>Print low - 1200x750</option> + <option value="300"<c:if test="${bwhist_flags_dpi[0] eq '300'}"> selected</c:if>>Print high - 2400x1500</option> + </select> + </p><p> + <input class="submit" type="submit" value="Update graph"> + </p> + </div> +</form> +<p><a href="csv/bwhist-flags.csv">CSV</a> file containing all data.</p> +<br> + <h3>Number of bytes spent on answering directory requests</h3> <br> <p>Relays running on 0.2.2.15-alpha or higher report the number of bytes