- tor-commits - lists.torproject.org

[tor/master] Merge remote-tracking branch 'origin/maint-0.2.3'
by nickm＠torproject.org 31 Jul '12

31 Jul '12

commit 5919e8e561d3abb3e74b92ac1b95f05243bb9372 Merge: e6d61ee aed93f8 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Jul 31 10:29:34 2012 -0400 Merge remote-tracking branch 'origin/maint-0.2.3' changes/bug6490 | 4 ++++ src/or/config.c | 29 +++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 0 deletions(-)

1 0

[tor/maint-0.2.3] Warn when accounting is used in a way likely to link hidden services
by nickm＠torproject.org 31 Jul '12

31 Jul '12

commit 2bd45213c991df62bf184758c5e8222685bda1ba Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Jul 30 11:58:55 2012 -0400 Warn when accounting is used in a way likely to link hidden services Fix for 6490. --- changes/bug6490 | 4 ++++ src/or/config.c | 29 +++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 0 deletions(-) diff --git a/changes/bug6490 b/changes/bug6490 new file mode 100644 index 0000000..c92daad --- /dev/null +++ b/changes/bug6490 @@ -0,0 +1,4 @@ + o Minor features: + - Warn when Tor is configured to use accounting in a way that will + link a hidden service to some other hidden service or public + address. Fix for bug 6490. diff --git a/src/or/config.c b/src/or/config.c index 918efd4..a369297 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -594,6 +594,7 @@ static void option_reset(const config_format_t *fmt, or_options_t *options, const config_var_t *var, int use_defaults); static void config_free(const config_format_t *fmt, void *options); static int config_lines_eq(config_line_t *a, config_line_t *b); +static int config_count_key(const config_line_t *a, const char *key); static int option_is_same(const config_format_t *fmt, const or_options_t *o1, const or_options_t *o2, const char *name); @@ -3066,6 +3067,20 @@ config_lines_eq(config_line_t *a, config_line_t *b) return 1; } +/** Return the number of lines in a whose key is key. */ +static int +config_count_key(const config_line_t *a, const char *key) +{ + int n = 0; + while (a) { + if (!strcasecmp(a->key, key)) { + ++n; + } + a = a->next; + } + return n; +} + /** Return true iff the option name has the same value in o1 * and o2. Must not be called for LINELIST_S or OBSOLETE options. */ @@ -3848,6 +3863,20 @@ options_validate(or_options_t *old_options, or_options_t *options, if (accounting_parse_options(options, 1)<0) REJECT("Failed to parse accounting options. See logs for details."); + if (options->AccountingMax) { + if (options->RendConfigLines && server_mode(options)) { + log_warn(LD_CONFIG, "Using accounting with a hidden service and an " + "ORPort is risky: your hidden service(s) and your public " + "address will all turn off at the same time, which may alert " + "observers that they are being run by the same party."); + } else if (config_count_key(options->RendConfigLines, + "HiddenServiceDir") > 1) { + log_warn(LD_CONFIG, "Using accounting with multiple hidden services is " + "risky: they will all turn off at the same time, which may " + "alert observers that they are being run by the same party."); + } + } + if (options->HTTPProxy) { /* parse it now */ if (tor_addr_port_lookup(options->HTTPProxy, &options->HTTPProxyAddr, &options->HTTPProxyPort) < 0)

1 0

[tor/maint-0.2.3] Merge branch 'bug6490_v2' into maint-0.2.3
by nickm＠torproject.org 31 Jul '12

31 Jul '12

commit aed93f8ad918787ea787061106bcd5289c712823 Merge: d9bd0de 2bd4521 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Jul 31 10:28:43 2012 -0400 Merge branch 'bug6490_v2' into maint-0.2.3 changes/bug6490 | 4 ++++ src/or/config.c | 29 +++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 0 deletions(-)

1 0

[tor/master] Fix some manpage typos
by nickm＠torproject.org 31 Jul '12

31 Jul '12

commit d9bd0de0628ff3ac9d84409315ad6df38981f019 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Jul 31 10:16:03 2012 -0400 Fix some manpage typos This is based on a pair of patches from A. Costa. I couldn't apply those directly, since they changed the generated *roff files, not the asciidoc source. Fixes Tor bug 6500 and Debian bug 683359. --- changes/bug6500 | 2 ++ doc/tor.1.txt | 4 ++-- doc/torify.1.txt | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/changes/bug6500 b/changes/bug6500 new file mode 100644 index 0000000..cac2054 --- /dev/null +++ b/changes/bug6500 @@ -0,0 +1,2 @@ + o Minor bugfixes: + - Fix some typos in the manpages. Patch from A. Costa. Fixes bug 6500. diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 78c3487..e7ba848 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -169,7 +169,7 @@ GENERAL OPTIONS duty of that proxy to properly forward the traffic to the bridge. + + In its second form, when set along with a corresponding Bridge line, the Tor - client launches the pluggable transport proxy exacutable in + client launches the pluggable transport proxy executable in __path-to-binary__ using __options__ as its command-line options, and forwards its traffic to it. It's the duty of that proxy to properly forward the traffic to the bridge. @@ -1709,7 +1709,7 @@ DIRECTORY AUTHORITY SERVER OPTIONS **V3BandwidthsFile** __FILENAME__:: V3 authoritative directories only. Configures the location of the - bandiwdth-authority generated file storing information on relays' measured + bandwidth-authority generated file storing information on relays' measured bandwidth capacities. (Default: unset) **V3AuthUseLegacyKey** **0**|**1**:: diff --git a/doc/torify.1.txt b/doc/torify.1.txt index ca2c385..4a4be12 100644 --- a/doc/torify.1.txt +++ b/doc/torify.1.txt @@ -21,7 +21,7 @@ DESCRIPTION wrapper available on a system. It calls torsocks or tsocks with a tor specific configuration file. + -torsocks is an improved wrapper that explictly rejects UDP, safely resolves DNS +torsocks is an improved wrapper that explicitly rejects UDP, safely resolves DNS lookups and properly socksifies your TCP connections. + tsocks itself is a wrapper between the tsocks library and the application that

1 0

[tor/master] Merge remote-tracking branch 'origin/maint-0.2.3'
by nickm＠torproject.org 31 Jul '12

31 Jul '12

commit e6d61ee59430162fd7b820278f3dfa1677ad81d8 Merge: 1049d31 d9bd0de Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Jul 31 10:18:05 2012 -0400 Merge remote-tracking branch 'origin/maint-0.2.3' changes/bug6500 | 2 ++ doc/tor.1.txt | 4 ++-- doc/torify.1.txt | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-)

1 0

[tor/maint-0.2.3] Fix some manpage typos
by nickm＠torproject.org 31 Jul '12

31 Jul '12

commit d9bd0de0628ff3ac9d84409315ad6df38981f019 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Jul 31 10:16:03 2012 -0400 Fix some manpage typos This is based on a pair of patches from A. Costa. I couldn't apply those directly, since they changed the generated *roff files, not the asciidoc source. Fixes Tor bug 6500 and Debian bug 683359. --- changes/bug6500 | 2 ++ doc/tor.1.txt | 4 ++-- doc/torify.1.txt | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/changes/bug6500 b/changes/bug6500 new file mode 100644 index 0000000..cac2054 --- /dev/null +++ b/changes/bug6500 @@ -0,0 +1,2 @@ + o Minor bugfixes: + - Fix some typos in the manpages. Patch from A. Costa. Fixes bug 6500. diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 78c3487..e7ba848 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -169,7 +169,7 @@ GENERAL OPTIONS duty of that proxy to properly forward the traffic to the bridge. + + In its second form, when set along with a corresponding Bridge line, the Tor - client launches the pluggable transport proxy exacutable in + client launches the pluggable transport proxy executable in __path-to-binary__ using __options__ as its command-line options, and forwards its traffic to it. It's the duty of that proxy to properly forward the traffic to the bridge. @@ -1709,7 +1709,7 @@ DIRECTORY AUTHORITY SERVER OPTIONS **V3BandwidthsFile** __FILENAME__:: V3 authoritative directories only. Configures the location of the - bandiwdth-authority generated file storing information on relays' measured + bandwidth-authority generated file storing information on relays' measured bandwidth capacities. (Default: unset) **V3AuthUseLegacyKey** **0**|**1**:: diff --git a/doc/torify.1.txt b/doc/torify.1.txt index ca2c385..4a4be12 100644 --- a/doc/torify.1.txt +++ b/doc/torify.1.txt @@ -21,7 +21,7 @@ DESCRIPTION wrapper available on a system. It calls torsocks or tsocks with a tor specific configuration file. + -torsocks is an improved wrapper that explictly rejects UDP, safely resolves DNS +torsocks is an improved wrapper that explicitly rejects UDP, safely resolves DNS lookups and properly socksifies your TCP connections. + tsocks itself is a wrapper between the tsocks library and the application that

1 0

[metrics-web/master] Normalize bwhist by flags to B/sec, not B/day.
by karsten＠torproject.org 31 Jul '12

31 Jul '12

commit d1c7054f18bc947251293a776405880163d23d1e Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Jul 31 14:17:06 2012 +0200 Normalize bwhist by flags to B/sec, not B/day. --- rserve/csv.R | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diff --git a/rserve/csv.R b/rserve/csv.R index 8eb69fe..57f728d 100644 --- a/rserve/csv.R +++ b/rserve/csv.R @@ -94,7 +94,8 @@ export_bandwidth <- function(path) { export_bwhist_flags <- function(path) { drv <- dbDriver("PostgreSQL") con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) - q <- paste("SELECT date, isexit, isguard, read, written", + q <- paste("SELECT date, isexit, isguard, read / 86400 AS read,", + "written / 86400 AS written", "FROM bwhist_flags WHERE date < current_date - 1", "ORDER BY date, isexit, isguard") rs <- dbSendQuery(con, q)

1 0

[metrics-web/master] Cut off last 1--2 days from graphs/CSVs consistently.
by karsten＠torproject.org 31 Jul '12

31 Jul '12

commit c92d4c698b8990c78a58ae02642fc7e55b236648 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Jul 31 14:00:11 2012 +0200 Cut off last 1--2 days from graphs/CSVs consistently. --- rserve/csv.R | 55 +++++++++++++++++++++++++++++-------------------------- rserve/graphs.R | 44 +++++++++++++++++++++----------------------- rserve/tables.R | 6 ++---- 3 files changed, 52 insertions(+), 53 deletions(-) diff --git a/rserve/csv.R b/rserve/csv.R index 6d76bf7..8eb69fe 100644 --- a/rserve/csv.R +++ b/rserve/csv.R @@ -1,10 +1,12 @@ export_networksize <- function(path) { drv <- dbDriver("PostgreSQL") con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) - q <- "SELECT date, avg_running AS relays FROM network_size" + q <- paste("SELECT date, avg_running AS relays FROM network_size", + "WHERE date < current_date - 1") rs <- dbSendQuery(con, q) relays <- fetch(rs, n = -1) - q <- "SELECT date, avg_running AS bridges FROM bridge_network_size" + q <- paste("SELECT date, avg_running AS bridges", + "FROM bridge_network_size WHERE date < current_date - 1") rs <- dbSendQuery(con, q) bridges <- fetch(rs, n = -1) dbDisconnect(con) @@ -19,7 +21,8 @@ export_cloudbridges <- function(path) { drv <- dbDriver("PostgreSQL") con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) q <- paste("SELECT date, avg_running_ec2 AS cloudbridges", - "FROM bridge_network_size ORDER BY date") + "FROM bridge_network_size WHERE date < current_date - 1", + "ORDER BY date") rs <- dbSendQuery(con, q) cloudbridges <- fetch(rs, n = -1) dbDisconnect(con) @@ -31,7 +34,7 @@ export_relaycountries <- function(path) { drv <- dbDriver("PostgreSQL") con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) q <- paste("SELECT date, country, relays FROM relay_countries", - "ORDER BY date, country") + "WHERE date < current_date - 1 ORDER BY date, country") rs <- dbSendQuery(con, q) relays <- fetch(rs, n = -1) dbDisconnect(con) @@ -42,7 +45,8 @@ export_relaycountries <- function(path) { export_versions <- function(path) { drv <- dbDriver("PostgreSQL") con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) - q <- "SELECT date, version, relays FROM relay_versions" + q <- paste("SELECT date, version, relays FROM relay_versions", + "WHERE date < current_date - 1") rs <- dbSendQuery(con, q) versions <- fetch(rs, n = -1) dbDisconnect(con) @@ -57,7 +61,7 @@ export_platforms <- function(path) { con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) q <- paste("SELECT date, avg_linux AS linux, avg_darwin AS darwin,", "avg_bsd AS bsd, avg_windows AS windows, avg_other AS other", - "FROM relay_platforms ORDER BY date") + "FROM relay_platforms WHERE date < current_date - 1 ORDER BY date") rs <- dbSendQuery(con, q) platforms <- fetch(rs, n = -1) dbDisconnect(con) @@ -68,11 +72,12 @@ export_platforms <- function(path) { export_bandwidth <- function(path) { drv <- dbDriver("PostgreSQL") con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) - q <- "SELECT date, bwadvertised FROM total_bandwidth" + q <- paste("SELECT date, bwadvertised FROM total_bandwidth", + "WHERE date < current_date - 1") rs <- dbSendQuery(con, q) bw_desc <- fetch(rs, n = -1) q <- paste("SELECT date, read, written FROM total_bwhist", - "WHERE date < (SELECT MAX(date) FROM total_bwhist) - 1") + "WHERE date < current_date - 1") rs <- dbSendQuery(con, q) bw_hist <- fetch(rs, n = -1) dbDisconnect(con) @@ -90,8 +95,7 @@ export_bwhist_flags <- function(path) { drv <- dbDriver("PostgreSQL") con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) q <- paste("SELECT date, isexit, isguard, read, written", - "FROM bwhist_flags", - "WHERE date < (SELECT MAX(date) FROM bwhist_flags) - 1", + "FROM bwhist_flags WHERE date < current_date - 1", "ORDER BY date, isexit, isguard") rs <- dbSendQuery(con, q) bw <- fetch(rs, n = -1) @@ -105,7 +109,7 @@ export_dirbytes <- function(path) { con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) q <- paste("SELECT date, dr, dw, brp, bwp, brd, bwd FROM user_stats", "WHERE country = 'zy' AND bwp / bwd <= 3", - "AND date < (SELECT MAX(date) FROM user_stats) - 1 ORDER BY date") + "AND date < current_date - 1 ORDER BY date") rs <- dbSendQuery(con, q) dir <- fetch(rs, n = -1) dbDisconnect(con) @@ -122,7 +126,7 @@ export_relayflags <- function(path) { con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) q <- paste("SELECT date, avg_running AS running, avg_exit AS exit,", "avg_guard AS guard, avg_fast AS fast, avg_stable AS stable", - "FROM network_size ORDER BY date") + "FROM network_size WHERE date < current_date - 1 ORDER BY date") rs <- dbSendQuery(con, q) relayflags <- fetch(rs, n = -1) dbDisconnect(con) @@ -135,7 +139,8 @@ export_relayflags_hour <- function(path) { con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) q <- paste("SELECT validafter, avg_running AS running,", "avg_exit AS exit, avg_guard AS guard, avg_fast AS fast,", - "avg_stable AS stable FROM network_size_hour ORDER BY validafter") + "avg_stable AS stable FROM network_size_hour", + "WHERE DATE(validafter) < current_date - 1 ORDER BY validafter") rs <- dbSendQuery(con, q) relayflags <- fetch(rs, n = -1) dbDisconnect(con) @@ -147,8 +152,7 @@ export_direct_users <- function(path) { drv <- dbDriver("PostgreSQL") con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) q <- paste("SELECT date, country, r, bwp, brn, bwn, brp, bwr, brr", - "FROM user_stats", - "WHERE date < (SELECT MAX(date) FROM user_stats) - 1", + "FROM user_stats WHERE date < current_date - 1", "ORDER BY date, country") rs <- dbSendQuery(con, q) u <- fetch(rs, n = -1) @@ -166,8 +170,7 @@ export_bridge_users <- function(path) { drv <- dbDriver("PostgreSQL") con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) q <- paste("SELECT date, country, users AS bridgeusers", - "FROM bridge_stats", - "WHERE date < (SELECT MAX(date) FROM bridge_stats)", + "FROM bridge_stats WHERE date < current_date - 1", "ORDER BY date, country") rs <- dbSendQuery(con, q) bridgeusers <- fetch(rs, n = -1) @@ -181,7 +184,8 @@ export_bridge_users <- function(path) { export_gettor <- function(path) { drv <- dbDriver("PostgreSQL") con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) - q <- "SELECT date, bundle, downloads FROM gettor_stats" + q <- paste("SELECT date, bundle, downloads FROM gettor_stats", + "WHERE date < current_date - 1") rs <- dbSendQuery(con, q) downloads <- fetch(rs, n = -1) dbDisconnect(con) @@ -216,7 +220,7 @@ export_torperf <- function(path) { drv <- dbDriver("PostgreSQL") con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) q <- paste("SELECT source, date, q1, md, q3 FROM torperf_stats", - "ORDER BY source, date") + "WHERE date < current_date - 1 ORDER BY source, date") rs <- dbSendQuery(con, q) torperf <- fetch(rs, n = -1) dbDisconnect(con) @@ -228,7 +232,8 @@ export_torperf_failures <- function(path) { drv <- dbDriver("PostgreSQL") con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) q <- paste("SELECT source, date, timeouts, failures, requests", - "FROM torperf_stats ORDER BY source, date") + "FROM torperf_stats WHERE date < current_date - 1", + "ORDER BY source, date") rs <- dbSendQuery(con, q) torperf <- fetch(rs, n = -1) dbDisconnect(con) @@ -240,8 +245,7 @@ help_export_monthly_users <- function(path, aggr_fun) { drv <- dbDriver("PostgreSQL") con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) q <- paste("SELECT date, country, r, bwp, brn, bwn, brp, bwr, brr", - "FROM user_stats", - "WHERE date < (SELECT MAX(date) FROM user_stats) - 1", + "FROM user_stats WHERE date < current_date - 1", "ORDER BY date, country") rs <- dbSendQuery(con, q) u <- fetch(rs, n = -1) @@ -249,8 +253,7 @@ help_export_monthly_users <- function(path, aggr_fun) { users = u$r * (u$bwp * u$brn / u$bwn - u$brp) / (u$bwr * u$brn / u$bwn - u$brr) / 10) q <- paste("SELECT date, country, FLOOR(users) AS users", - "FROM bridge_stats", - "WHERE date < (SELECT MAX(date) FROM bridge_stats)", + "FROM bridge_stats WHERE date < current_date - 1", "ORDER BY date, country") rs <- dbSendQuery(con, q) bridge <- fetch(rs, n = -1) @@ -280,6 +283,7 @@ export_connbidirect <- function(path) { con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) q <- paste("SELECT DATE(statsend) AS date, source, belownum AS below,", "readnum AS read, writenum AS write, bothnum AS \"both\"", + "WHERE DATE(statsend) < current_date - 1", "FROM connbidirect ORDER BY 1, 2") rs <- dbSendQuery(con, q) c <- fetch(rs, n = -1) @@ -293,8 +297,7 @@ export_dirreq_stats <- function(path) { drv <- dbDriver("PostgreSQL") con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) q <- paste("SELECT date, r, bwp, brp, bwn, brn, bwr, brr ", - "FROM user_stats ", - "WHERE date < (SELECT MAX(date) FROM user_stats) - 1 ", + "FROM user_stats WHERE date < current_date - 1", "AND country = 'zy' ORDER BY date", sep = "") rs <- dbSendQuery(con, q) u <- fetch(rs, n = -1) diff --git a/rserve/graphs.R b/rserve/graphs.R index 09563a3..fac88a7 100644 --- a/rserve/graphs.R +++ b/rserve/graphs.R @@ -282,12 +282,12 @@ plot_networksize <- function(start, end, path, dpi) { con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) q <- paste("SELECT date, avg_running AS relays FROM network_size ", "WHERE date >= '", start, "' AND date <= '", end, - "' AND date < (SELECT MAX(date) FROM network_size)", sep = "") + "' AND date < current_date - 1", sep = "") rs <- dbSendQuery(con, q) relays <- fetch(rs, n = -1) q <- paste("SELECT date, avg_running AS bridges ", "FROM bridge_network_size WHERE date >= '", start, - "' AND date <= '", end, "'", sep = "") + "' AND date <= '", end, "' AND date < current_date - 1", sep = "") rs <- dbSendQuery(con, q) bridges <- fetch(rs, n = -1) dbDisconnect(con) @@ -329,7 +329,7 @@ plot_cloudbridges <- function(start, end, path, dpi) { con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) q <- paste("SELECT date, avg_running_ec2 ", "FROM bridge_network_size WHERE date >= '", start, - "' AND date <= '", end, "'", sep = "") + "' AND date <= '", end, "' AND date < current_date - 1", sep = "") rs <- dbSendQuery(con, q) bridges <- fetch(rs, n = -1) dbDisconnect(con) @@ -363,11 +363,11 @@ plot_relaycountries <- function(start, end, country, path, dpi) { if (country == "all") { q <- paste("SELECT date, avg_running AS relays FROM network_size ", "WHERE date >= '", start, "' AND date <= '", end, - "' AND date < (SELECT MAX(date) FROM network_size)", sep = "") + "' AND date < current_date - 1", sep = "") } else { q <- paste("SELECT date, relays FROM relay_countries ", "WHERE date >= '", start, "' AND date <= '", end, - "' AND date < (SELECT MAX(date) FROM relay_countries) ", + "' AND date < current_date - 1 ", "AND country = '", country, "'", sep = "") } rs <- dbSendQuery(con, q) @@ -407,7 +407,7 @@ plot_versions <- function(start, end, path, dpi) { con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) q <- paste("SELECT date, version, relays FROM relay_versions ", "WHERE date >= '", start, "' AND date <= '", end, - "' AND date < (SELECT MAX(date) FROM relay_versions)", sep = "") + "' AND date < current_date - 1", sep = "") rs <- dbSendQuery(con, q) versions <- fetch(rs, n = -1) dbDisconnect(con) @@ -443,8 +443,7 @@ plot_platforms <- function(start, end, path, dpi) { con <- dbConnect(drv, user=dbuser, password=dbpassword, dbname=db) q <- paste("SELECT date, avg_linux, avg_darwin, avg_bsd, avg_windows, ", "avg_other FROM relay_platforms WHERE date >= '", start, - "' AND date <= '", end, - "' AND date < (SELECT MAX(date) FROM relay_platforms)", sep = "") + "' AND date <= '", end, "' AND date < current_date - 1", sep = "") rs <- dbSendQuery(con, q) platforms <- fetch(rs, n = -1) dbDisconnect(con) @@ -476,12 +475,12 @@ plot_bandwidth <- function(start, end, path, dpi) { con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) q <- paste("SELECT date, bwadvertised FROM total_bandwidth ", "WHERE date >= '", start, "' AND date <= '", end, "' ", - "AND date < (SELECT MAX(date) FROM total_bandwidth) - 1 ", sep = "") + "AND date < current_date - 1", sep = "") rs <- dbSendQuery(con, q) bw_desc <- fetch(rs, n = -1) q <- paste("SELECT date, read, written FROM total_bwhist ", "WHERE date >= '", start, "' AND date <= '", end, "' ", - "AND date < (SELECT MAX(date) FROM total_bwhist) - 1 ", sep = "") + "AND date < current_date - 1", sep = "") rs <- dbSendQuery(con, q) bw_hist <- fetch(rs, n = -1) dbDisconnect(con) @@ -514,7 +513,7 @@ plot_bwhist_flags <- function(start, end, path, dpi) { con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) q <- paste("SELECT date, isexit, isguard, read, written ", "FROM bwhist_flags WHERE date >= '", start, "' AND date <= '", end, - "' AND date < (SELECT MAX(date) FROM bwhist_flags) - 1 ", sep = "") + "' AND date < current_date - 1", sep = "") rs <- dbSendQuery(con, q) bw <- fetch(rs, n = -1) dbDisconnect(con) @@ -560,9 +559,8 @@ plot_dirbytes <- function(start, end, path, dpi) { con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) q <- paste("SELECT date, dr, dw, brp, bwp, brd, bwd FROM user_stats ", "WHERE country = 'zy' AND bwp / bwd <= 3 AND date >= '", start, - "' AND date <= '", end, "' ", - "AND date < (SELECT MAX(date) FROM user_stats) - 1 ORDER BY date", - sep = "") + "' AND date <= '", end, "' AND date < current_date - 1 ", + "ORDER BY date", sep = "") rs <- dbSendQuery(con, q) dir <- fetch(rs, n = -1) dbDisconnect(con) @@ -599,7 +597,7 @@ plot_relayflags <- function(start, end, flags, granularity, path, dpi) { columns <- paste("avg_", tolower(flags), sep = "", collapse = ", ") q <- paste("SELECT date, ", columns, " FROM network_size ", "WHERE date >= '", start, "' AND date <= '", end, - "' AND date < (SELECT MAX(date) FROM network_size)", sep = "") + "' AND date < current_date - 1", sep = "") rs <- dbSendQuery(con, q) networksize <- fetch(rs, n = -1) dbDisconnect(con) @@ -639,7 +637,8 @@ plot_relayflags <- function(start, end, flags, granularity, path, dpi) { columns <- paste("avg_", tolower(flags), sep = "", collapse = ", ") q <- paste("SELECT validafter, ", columns, " FROM network_size_hour ", "WHERE DATE(validafter) >= '", start, - "' AND DATE(validafter) <= '", end, "'", sep = "") + "' AND DATE(validafter) <= '", end, "' ", + "AND DATE(validafter) < current_date - 1", sep = "") rs <- dbSendQuery(con, q) networksize <- fetch(rs, n = -1) dbDisconnect(con) @@ -672,7 +671,7 @@ plot_direct_users <- function(start, end, country, events, path, nocutoff, q <- paste("SELECT date, r, bwp, brn, bwn, brp, bwr, brr, country ", "FROM user_stats WHERE date >= '", start, "' AND date <= '", end, "' ", ifelse(nocutoff == "off", - " AND date < (SELECT MAX(date) FROM user_stats) - 1 ", ""), + " AND date < current_date - 1 ", ""), " AND (country = 'zy'", ifelse(country == "all", "", paste(" OR country = '", country, "'", sep = "")), ")", sep = "") rs <- dbSendQuery(con, q) @@ -748,7 +747,7 @@ plot_bridge_users <- function(start, end, country, path, dpi) { con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) q <- paste("SELECT date, users FROM bridge_stats ", "WHERE date >= '", start, "' AND date <= '", end, "' ", - "AND date < (SELECT MAX(date) FROM bridge_stats) - 1", + "AND date < current_date - 1", " AND country = '", ifelse(country == "all", "zy", country), "'", sep = "") rs <- dbSendQuery(con, q) @@ -790,8 +789,7 @@ plot_gettor <- function(start, end, language, path, dpi) { q <- paste("SELECT date, SUM(downloads) AS downloads ", "FROM gettor_stats WHERE bundle ", condition, " AND date >= '", start, "' AND date <= '", end, - "' AND date < (SELECT MAX(date) FROM gettor_stats) GROUP BY date", - sep = "") + "' AND date < current_date - 1 GROUP BY date", sep = "") rs <- dbSendQuery(con, q) downloads <- fetch(rs, n = -1) dbDisconnect(con) @@ -828,7 +826,7 @@ plot_torperf <- function(start, end, source, filesize, path, dpi) { q <- paste("SELECT date, q1, md, q3 FROM torperf_stats ", "WHERE source = '", paste(source, filesize, sep = "-"), "' AND date >= '", start, "' AND date <= '", end, - "' AND date < (SELECT MAX(date) FROM torperf_stats)", sep = "") + "' AND date < current_date - 1", sep = "") rs <- dbSendQuery(con, q) torperf <- fetch(rs, n = -1) dbDisconnect(con) @@ -879,7 +877,7 @@ plot_torperf_failures <- function(start, end, source, filesize, path, "FROM torperf_stats WHERE source = '", paste(source, filesize, sep = "-"), "' AND date >= '", start, "' AND date <= '", end, - "' AND date < (SELECT MAX(date) FROM torperf_stats)", sep = "") + "' AND date < current_date - 1", sep = "") rs <- dbSendQuery(con, q) torperf <- fetch(rs, n = -1) dbDisconnect(con) @@ -932,7 +930,7 @@ plot_connbidirect <- function(start, end, path, dpi) { q <- paste("SELECT DATE(statsend) AS date, readnum, writenum, bothnum ", "FROM connbidirect WHERE DATE(statsend) >= '", start, "' AND DATE(statsend) <= '", end, "' AND DATE(statsend) < ", - "(SELECT MAX(DATE(statsend)) FROM connbidirect)", sep = "") + "current_date - 1", sep = "") rs <- dbSendQuery(con, q) c <- fetch(rs, n = -1) dbDisconnect(con) diff --git a/rserve/tables.R b/rserve/tables.R index e2e270d..cb74379 100644 --- a/rserve/tables.R +++ b/rserve/tables.R @@ -7,8 +7,7 @@ write_direct_users <- function(start, end, path) { con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) q <- paste("SELECT date, country, r, bwp, brn, bwn, brp, bwr, brr ", "FROM user_stats WHERE date >= '", start, "' AND date <= '", end, - "' AND date < (SELECT MAX(date) FROM user_stats) - 1 ", - "ORDER BY date, country", sep = "") + "' AND date < current_date - 1 ORDER BY date, country", sep = "") rs <- dbSendQuery(con, q) u <- fetch(rs, n = -1) dbDisconnect(con) @@ -35,8 +34,7 @@ write_censorship_events <- function(start, end, path) { con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) q <- paste("SELECT date, country, r, bwp, brn, bwn, brp, bwr, brr ", "FROM user_stats WHERE date >= '", start, "' AND date <= '", end, - "' AND date < (SELECT MAX(date) FROM user_stats) - 1", - sep = "") + "' AND date < current_date - 1", sep = "") rs <- dbSendQuery(con, q) u <- fetch(rs, n = -1) dbDisconnect(con)

1 0

[metrics-web/master] Cache JSON-formatted graph data for performance reasons.
by karsten＠torproject.org 31 Jul '12

31 Jul '12

commit c433f5c77e0168aa2c8cbde1cea1de37327c88fc Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Jul 31 12:32:40 2012 +0200 Cache JSON-formatted graph data for performance reasons. Previously, we were only caching the .csv contents generated by R, but we were converting the .csv to .json in each request. Turns out we can save over a second per request here for large .csv files by caching the .json contents. Let's do it. --- src/org/torproject/ernie/web/CsvServlet.java | 10 ++- src/org/torproject/ernie/web/GraphDataServlet.java | 62 +++++++++++----- src/org/torproject/ernie/web/RObject.java | 11 ++- src/org/torproject/ernie/web/RObjectGenerator.java | 74 +++++--------------- 4 files changed, 76 insertions(+), 81 deletions(-) diff --git a/src/org/torproject/ernie/web/CsvServlet.java b/src/org/torproject/ernie/web/CsvServlet.java index 5e170e3..6fdcaac 100644 --- a/src/org/torproject/ernie/web/CsvServlet.java +++ b/src/org/torproject/ernie/web/CsvServlet.java @@ -2,7 +2,10 @@ * See LICENSE for licensing information */ package org.torproject.ernie.web; +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStreamReader; import java.util.SortedSet; import java.util.logging.Logger; @@ -75,16 +78,17 @@ public class CsvServlet extends HttpServlet { /* Request CSV file from R object generator, which asks Rserve to * generate it. */ - String csvFileContent = this.rObjectGenerator.generateCsv( + RObject csvFile = this.rObjectGenerator.generateCsv( requestedCsvFile, true); - /* Make sure that we have a graph to return. */ - if (csvFileContent == null) { + /* Make sure that we have a .csv file to return. */ + if (csvFile == null) { response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); return; } /* Write CSV file to response. */ + String csvFileContent = new String(csvFile.getBytes()); response.setContentType("text/csv"); response.setHeader("Content-Length", String.valueOf( csvFileContent.length())); diff --git a/src/org/torproject/ernie/web/GraphDataServlet.java b/src/org/torproject/ernie/web/GraphDataServlet.java index 209aefe..5b79c1b 100644 --- a/src/org/torproject/ernie/web/GraphDataServlet.java +++ b/src/org/torproject/ernie/web/GraphDataServlet.java @@ -7,6 +7,7 @@ import java.io.IOException; import java.io.StringReader; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.util.HashMap; import java.util.Map; import java.util.SortedMap; import java.util.SortedSet; @@ -112,16 +113,49 @@ public class GraphDataServlet extends HttpServlet { /* Request CSV file from R object generator, which may ask Rserve to * generate it. */ - String csvFileContent = this.rObjectGenerator.generateCsv( - requestedCsvFile, true); + RObject csvFile = this.rObjectGenerator.generateCsv(requestedCsvFile, + true); /* Make sure that we have a CSV to convert into JSON. */ - if (csvFileContent == null) { + if (csvFile == null) { response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); return; } - /* Convert CSV to JSON format. */ + /* Look up if we converted this CSV to JSON format before. If not, + * convert it now. */ + String jsonString; + if (!this.lastConvertedCsvFile.containsKey(requestedJsonFile) || + this.lastConvertedCsvFile.get(requestedJsonFile) < + csvFile.getLastModified()) { + jsonString = this.convertCsvToJson(requestedJsonFile, + new String(csvFile.getBytes())); + this.lastConvertedCsvFile.put(requestedJsonFile, + csvFile.getLastModified()); + this.convertedCsvFiles.put(requestedJsonFile, jsonString); + } else { + jsonString = this.convertedCsvFiles.get(requestedJsonFile); + } + + /* Make sure we have a JSON string to return. */ + if (jsonString == null) { + response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + return; + } + + /* Write JSON string to response. */ + response.setHeader("Access-Control-Allow-Origin", "*"); + response.setContentType("application/json"); + response.setCharacterEncoding("utf-8"); + response.getWriter().print(jsonString); + } + + private Map<String, Long> lastConvertedCsvFile = + new HashMap<String, Long>(); + private Map<String, String> convertedCsvFiles = + new HashMap<String, String>(); + private String convertCsvToJson(String requestedJsonFile, + String csvFileContent) { String jsonString = null; try { BufferedReader br = new BufferedReader(new StringReader( @@ -148,17 +182,14 @@ public class GraphDataServlet extends HttpServlet { } } if (columns == null || dateCol < 0 || valueCols.isEmpty()) { - response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); - return; + return null; } SortedMap<String, SortedSet<String>> graphs = new TreeMap<String, SortedSet<String>>(); while ((line = br.readLine()) != null) { String[] elements = line.split(","); if (elements.length != columns.length) { - response.sendError( - HttpServletResponse.SC_INTERNAL_SERVER_ERROR); - return; + return null; } String date = elements[dateCol]; String variable = ""; @@ -225,18 +256,11 @@ public class GraphDataServlet extends HttpServlet { br.close(); jsonString = "[" + sb.toString().substring(1) + "\n]"; } catch (IOException e) { - response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); - return; + return null; } catch (ParseException e) { - response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); - return; + return null; } - - /* Write JSON file to response. */ - response.setHeader("Access-Control-Allow-Origin", "*"); - response.setContentType("application/json"); - response.setCharacterEncoding("utf-8"); - response.getWriter().print(jsonString); + return jsonString; } } diff --git a/src/org/torproject/ernie/web/RObject.java b/src/org/torproject/ernie/web/RObject.java index 4ee5cdc..25dac52 100644 --- a/src/org/torproject/ernie/web/RObject.java +++ b/src/org/torproject/ernie/web/RObject.java @@ -5,14 +5,19 @@ package org.torproject.ernie.web; public class RObject { private byte[] bytes; private String fileName; - public RObject(byte[] bytes, String fileName) { + private long lastModified; + public RObject(byte[] bytes, String fileName, long lastModified) { this.bytes = bytes; this.fileName = fileName; + this.lastModified = lastModified; } public String getFileName() { - return fileName; + return this.fileName; } public byte[] getBytes() { - return bytes; + return this.bytes; + } + public long getLastModified() { + return this.lastModified; } } diff --git a/src/org/torproject/ernie/web/RObjectGenerator.java b/src/org/torproject/ernie/web/RObjectGenerator.java index 1b56f31..aea9389 100644 --- a/src/org/torproject/ernie/web/RObjectGenerator.java +++ b/src/org/torproject/ernie/web/RObjectGenerator.java @@ -173,69 +173,27 @@ public class RObjectGenerator implements ServletContextListener { String imageFilename = imageFilenameBuilder.toString(); rQueryBuilder.append("path = '%s')"); String rQuery = rQueryBuilder.toString(); - byte[] graphBytes = this.generateGraph(rQuery, imageFilename, - checkCache); - if (graphBytes != null) { - return new RObject(graphBytes, imageFilename); - } else { - return null; - } - } - - /* Generate a graph using the given R query that has a placeholder for - * the absolute path to the image to be created. */ - private byte[] generateGraph(String rQuery, String imageFilename, - boolean checkCache) { - - /* See if we need to generate this graph. */ File imageFile = new File(this.cachedGraphsDirectory + "/" + imageFilename); - byte[] imageBytes = this.generateRObject(rQuery, imageFile, + return this.generateRObject(rQuery, imageFile, imageFilename, checkCache); - - /* Return the graph bytes. */ - return imageBytes; } public SortedSet<String> getAvailableCsvFiles() { return this.availableCsvFiles; } - public String generateCsv(String requestedCsvFile, boolean checkCache) { + public RObject generateCsv(String requestedCsvFile, + boolean checkCache) { /* Prepare filename and R query string. */ String rQuery = "export_" + requestedCsvFile.replaceAll("-", "_") + "(path = '%s')"; String csvFilename = requestedCsvFile + ".csv"; - return this.generateCsv(rQuery, csvFilename, checkCache); - } - - /* Generate a comma-separated value file using the given R query that - * has a placeholder for the absolute path to the file to be created. */ - private String generateCsv(String rQuery, String csvFilename, - boolean checkCache) { /* See if we need to generate this .csv file. */ File csvFile = new File(this.cachedGraphsDirectory + "/" + csvFilename); - byte[] csvBytes = this.generateRObject(rQuery, csvFile, checkCache); - - /* Read the text file from disk and write it to a string. */ - String result = null; - try { - StringBuilder sb = new StringBuilder(); - BufferedReader br = new BufferedReader(new InputStreamReader( - new ByteArrayInputStream(csvBytes))); - String line = null; - while ((line = br.readLine()) != null) { - sb.append(line + "\n"); - } - result = sb.toString(); - } catch (IOException e) { - return null; - } - - /* Return the csv file content. */ - return result; + return this.generateRObject(rQuery, csvFile, csvFilename, checkCache); } public List<Map<String, String>> generateTable(String tableName, @@ -295,7 +253,7 @@ public class RObjectGenerator implements ServletContextListener { File tableFile = new File(this.cachedGraphsDirectory + "/" + tableFilename); byte[] tableBytes = this.generateRObject(rQuery, tableFile, - checkCache); + tableFilename, checkCache).getBytes(); /* Write the table content to a map. */ List<Map<String, String>> result = null; @@ -329,15 +287,15 @@ public class RObjectGenerator implements ServletContextListener { /* Generate an R object in a separate worker thread, or wait for an * already running worker thread to finish and get its result. */ - private byte[] generateRObject(String rQuery, File rObjectFile, - boolean checkCache) { + private RObject generateRObject(String rQuery, File rObjectFile, + String fileName, boolean checkCache) { RObjectGeneratorWorker worker = null; synchronized (this.rObjectGeneratorThreads) { if (this.rObjectGeneratorThreads.containsKey(rQuery)) { worker = this.rObjectGeneratorThreads.get(rQuery); } else { worker = new RObjectGeneratorWorker(rQuery, rObjectFile, - checkCache); + fileName, checkCache); this.rObjectGeneratorThreads.put(rQuery, worker); worker.start(); } @@ -352,7 +310,7 @@ public class RObjectGenerator implements ServletContextListener { this.rObjectGeneratorThreads.remove(rQuery); } } - return worker.getRObjectBytes(); + return worker.getRObject(); } private Map<String, RObjectGeneratorWorker> rObjectGeneratorThreads = @@ -362,13 +320,15 @@ public class RObjectGenerator implements ServletContextListener { private String rQuery; private File rObjectFile; + private String fileName; private boolean checkCache; - private byte[] result = null; + private RObject result = null; public RObjectGeneratorWorker(String rQuery, File rObjectFile, - boolean checkCache) { + String fileName, boolean checkCache) { this.rQuery = rQuery; this.rObjectFile = rObjectFile; + this.fileName = fileName; this.checkCache = checkCache; } @@ -401,6 +361,7 @@ public class RObjectGenerator implements ServletContextListener { } /* Read the R object from disk and write it to a byte array. */ + long lastModified = this.rObjectFile.lastModified(); try { BufferedInputStream bis = new BufferedInputStream( new FileInputStream(this.rObjectFile), 1024); @@ -411,14 +372,15 @@ public class RObjectGenerator implements ServletContextListener { baos.write(buffer, 0, length); } bis.close(); - this.result = baos.toByteArray(); + this.result = new RObject(baos.toByteArray(), this.fileName, + lastModified); } catch (IOException e) { return; } } - public byte[] getRObjectBytes() { - return result; + public RObject getRObject() { + return this.result; } } }

1 0

[tor/release-0.2.3] bump to 0.2.3.19-rc-dev
by arma＠torproject.org 31 Jul '12

31 Jul '12

commit ab4c6b6543b98389f252bdb21f8c59f57b7a4973 Author: Roger Dingledine <arma(a)torproject.org> Date: Tue Jul 31 05:43:41 2012 -0400 bump to 0.2.3.19-rc-dev --- configure.in | 2 +- contrib/tor-mingw.nsi.in | 2 +- src/win32/orconfig.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configure.in b/configure.in index 344cf28..816d29e 100644 --- a/configure.in +++ b/configure.in @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2012, The Tor Project, Inc. dnl See LICENSE for licensing information AC_INIT -AM_INIT_AUTOMAKE(tor, 0.2.3.19-rc) +AM_INIT_AUTOMAKE(tor, 0.2.3.19-rc-dev) AM_CONFIG_HEADER(orconfig.h) AC_CANONICAL_HOST diff --git a/contrib/tor-mingw.nsi.in b/contrib/tor-mingw.nsi.in index 26442af..bd191d4 100644 --- a/contrib/tor-mingw.nsi.in +++ b/contrib/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.2.3.19-rc" +!define VERSION "0.2.3.19-rc-dev" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index e410dde..37d3631 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -232,7 +232,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.2.3.19-rc" +#define VERSION "0.2.3.19-rc-dev"

1 0