- tor-commits - lists.torproject.org

[pytorctl/master] rewrite query for mysql compatibility attempt 1
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit a416402b0a4c6b5406448a5f87538081f26d9368 Author: aagbsn <aagbsn(a)extc.org> Date: Thu Apr 21 18:28:17 2011 -0700 rewrite query for mysql compatibility attempt 1 --- SQLSupport.py | 11 ++++++++++- 1 files changed, 10 insertions(+), 1 deletions(-) diff --git a/SQLSupport.py b/SQLSupport.py index 9e1a4cb..8ef8b68 100644 --- a/SQLSupport.py +++ b/SQLSupport.py @@ -405,9 +405,18 @@ class RouterStats(Entity): #min_avg_rank = select([func.min(RouterStats.avg_rank)]).as_scalar() max_avg_rank = select([func.max(RouterStats.avg_rank)]).as_scalar() + # this query breaks MySQL! + #RouterStats.table.update(values= + # {RouterStats.table.c.percentile: + # (100.0*RouterStats.table.c.avg_rank)/max_avg_rank}).execute() + + # I think the problem is the reference to RouterStats.table.c.avg_rank -- let's just replace it with the query used above. I think the nested query will be OK + #query('UPDATE routerstats SET percentile=((%s * routerstats.avg_rank) / (SELECT max(routerstats.avg_rank) AS max_1 \nFROM routerstats))' (100.0,)) RouterStats.table.update(values= {RouterStats.table.c.percentile: - (100.0*RouterStats.table.c.avg_rank)/max_avg_rank}).execute() + (100.0*avg_r)/max_avg_rank}).execute() + + tc_session.commit() _compute_ranks = Callable(_compute_ranks)

1 0

[pytorctl/master] fixes for divide-by-zeros
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit fbf2fcf865bc0d9b65841076fd88e31f321638de Author: aagbsn <aagbsn(a)extc.org> Date: Fri Apr 29 16:21:40 2011 -0700 fixes for divide-by-zeros Postgres doesn't ignore divide-by-zeros like MySQL CASE statement added to set the result to NULL if the denominator is zero --- SQLSupport.py | 15 ++++++++++----- 1 files changed, 10 insertions(+), 5 deletions(-) diff --git a/SQLSupport.py b/SQLSupport.py index ec29bda..e8feed9 100644 --- a/SQLSupport.py +++ b/SQLSupport.py @@ -25,7 +25,7 @@ import sqlalchemy import sqlalchemy.orm.exc from sqlalchemy.orm import scoped_session, sessionmaker, eagerload, lazyload, eagerload_all from sqlalchemy import create_engine, and_, or_, not_, func -from sqlalchemy.sql import func,select,alias +from sqlalchemy.sql import func,select,alias,case from sqlalchemy.schema import ThreadLocalMetaData,MetaData from elixir import * from elixir import options @@ -328,15 +328,20 @@ class RouterStats(Entity): RouterStats.table.c.circ_fail_from:f_from_s, RouterStats.table.c.avg_first_ext:avg_ext}).execute() + # added case() to set NULL and avoid divide-by-zeros (Postgres) RouterStats.table.update(stats_clause, values= {RouterStats.table.c.circ_from_rate: - RouterStats.table.c.circ_fail_from/RouterStats.table.c.circ_try_from, + case([(RouterStats.table.c.circ_try_from == 0, None)], + else_=(RouterStats.table.c.circ_fail_from/RouterStats.table.c.circ_try_from)), RouterStats.table.c.circ_to_rate: - RouterStats.table.c.circ_fail_to/RouterStats.table.c.circ_try_to, + case([(RouterStats.table.c.circ_try_to == 0, None)], + else_=(RouterStats.table.c.circ_fail_to/RouterStats.table.c.circ_try_to)), RouterStats.table.c.circ_bi_rate: - (RouterStats.table.c.circ_fail_to+RouterStats.table.c.circ_fail_from) + case([(RouterStats.table.c.circ_try_to+RouterStats.table.c.circ_try_from == 0, None)], + else_=((RouterStats.table.c.circ_fail_to+RouterStats.table.c.circ_fail_from) / - (RouterStats.table.c.circ_try_to+RouterStats.table.c.circ_try_from)}).execute() + (RouterStats.table.c.circ_try_to+RouterStats.table.c.circ_try_from))), + }).execute() # TODO: Give the streams relation table a sane name and reduce this too

1 0

[pytorctl/master] rewrite query for mysql compatibility attempt 2
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit 3a2b44864417908bae45e835e0502297671e6c7b Author: aagbsn <aagbsn(a)extc.org> Date: Sun Apr 24 21:12:56 2011 -0700 rewrite query for mysql compatibility attempt 2 this actually appears to work --- SQLSupport.py | 49 +++++++++++++++++++++++++++++-------------------- 1 files changed, 29 insertions(+), 20 deletions(-) diff --git a/SQLSupport.py b/SQLSupport.py index 8ef8b68..ec29bda 100644 --- a/SQLSupport.py +++ b/SQLSupport.py @@ -25,7 +25,7 @@ import sqlalchemy import sqlalchemy.orm.exc from sqlalchemy.orm import scoped_session, sessionmaker, eagerload, lazyload, eagerload_all from sqlalchemy import create_engine, and_, or_, not_, func -from sqlalchemy.sql import func,select +from sqlalchemy.sql import func,select,alias from sqlalchemy.schema import ThreadLocalMetaData,MetaData from elixir import * from elixir import options @@ -403,35 +403,44 @@ class RouterStats(Entity): RouterStats.table.c.avg_desc_bw:avg_desc_bw}).execute() #min_avg_rank = select([func.min(RouterStats.avg_rank)]).as_scalar() - max_avg_rank = select([func.max(RouterStats.avg_rank)]).as_scalar() - # this query breaks MySQL! - #RouterStats.table.update(values= - # {RouterStats.table.c.percentile: - # (100.0*RouterStats.table.c.avg_rank)/max_avg_rank}).execute() + # the commented query breaks mysql because UPDATE cannot reference + # target table in the FROM clause. So we throw in an anonymous alias and wrap + # another select around it in order to get the nested SELECT stored into a + # temporary table. + # FIXME: performance? no idea + #max_avg_rank = select([func.max(RouterStats.avg_rank)]).as_scalar() + max_avg_rank = select([alias(select([func.max(RouterStats.avg_rank)]))]).as_scalar() - # I think the problem is the reference to RouterStats.table.c.avg_rank -- let's just replace it with the query used above. I think the nested query will be OK - #query('UPDATE routerstats SET percentile=((%s * routerstats.avg_rank) / (SELECT max(routerstats.avg_rank) AS max_1 \nFROM routerstats))' (100.0,)) RouterStats.table.update(values= {RouterStats.table.c.percentile: - (100.0*avg_r)/max_avg_rank}).execute() - + (100.0*RouterStats.table.c.avg_rank)/max_avg_rank}).execute() tc_session.commit() _compute_ranks = Callable(_compute_ranks) def _compute_ratios(stats_clause): tc_session.expunge_all() - avg_from_rate = select([func.avg(RouterStats.circ_from_rate)], - stats_clause).as_scalar() - avg_to_rate = select([func.avg(RouterStats.circ_to_rate)], - stats_clause).as_scalar() - avg_bi_rate = select([func.avg(RouterStats.circ_bi_rate)], - stats_clause).as_scalar() - avg_ext = select([func.avg(RouterStats.avg_first_ext)], - stats_clause).as_scalar() - avg_sbw = select([func.avg(RouterStats.sbw)], - stats_clause).as_scalar() + avg_from_rate = select([alias( + select([func.avg(RouterStats.circ_from_rate)], + stats_clause) + )]).as_scalar() + avg_to_rate = select([alias( + select([func.avg(RouterStats.circ_to_rate)], + stats_clause) + )]).as_scalar() + avg_bi_rate = select([alias( + select([func.avg(RouterStats.circ_bi_rate)], + stats_clause) + )]).as_scalar() + avg_ext = select([alias( + select([func.avg(RouterStats.avg_first_ext)], + stats_clause) + )]).as_scalar() + avg_sbw = select([alias( + select([func.avg(RouterStats.sbw)], + stats_clause) + )]).as_scalar() RouterStats.table.update(stats_clause, values= {RouterStats.table.c.circ_from_ratio:

1 0

[pytorctl/master] generalize db message, we now support postgres and mysql too
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit ffb68994a4d440db9deca2624ed33075f3fb7125 Author: aagbsn <aagbsn(a)extc.org> Date: Fri Jun 17 10:53:05 2011 -0700 generalize db message, we now support postgres and mysql too --- ScanSupport.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/ScanSupport.py b/ScanSupport.py index d102c78..f2a39d9 100644 --- a/ScanSupport.py +++ b/ScanSupport.py @@ -182,7 +182,7 @@ class SQLScanHandler(ScanHandler): ScanHandler.__init__(self, c, selmgr, RouterClass, strm_selector) def attach_sql_listener(self, db_uri): - plog("DEBUG", "Got sqlite: "+db_uri) + plog("DEBUG", "Got db: "+db_uri) SQLSupport.setup_db(db_uri, echo=False, drop=True) self.sql_consensus_listener = SQLSupport.ConsensusTrackerListener() self.add_event_listener(self.sql_consensus_listener)

1 0

[pytorctl/master] add case for long
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit ef0b770e9fa3ddb63ec52dfa4796c7dff27e8709 Author: aagbsn <aagbsn(a)extc.org> Date: Sat Jun 18 16:36:24 2011 -0700 add case for long --- SQLSupport.py | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/SQLSupport.py b/SQLSupport.py index e8feed9..8eefa20 100644 --- a/SQLSupport.py +++ b/SQLSupport.py @@ -568,6 +568,7 @@ class RouterStats(Entity): def cvt(a,b,c=1): if type(a) == float: return round(a/c,b) elif type(a) == int: return a + elif type(a) == long: return a elif type(a) == type(None): return "None" else: return type(a)

1 0

[pytorctl/master] SQLAlchemey and Elixir upgrade
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit 5161c64139f18b55288dab07e74f9f3f1c5bb704 Author: aagbsn <aagbsn(a)extc.org> Date: Sun Apr 10 23:24:23 2011 -0700 SQLAlchemey and Elixir upgrade enabled elixir migration aid options. renamed a few function calls, as per SQLAlchemy upgrade docs: session.clear() is removed. use session.remove_all() --- SQLSupport.py | 12 ++++++++---- 1 files changed, 8 insertions(+), 4 deletions(-) diff --git a/SQLSupport.py b/SQLSupport.py index 2532973..2bd0a8f 100644 --- a/SQLSupport.py +++ b/SQLSupport.py @@ -28,6 +28,10 @@ from sqlalchemy import create_engine, and_, or_, not_, func from sqlalchemy.sql import func,select from sqlalchemy.schema import ThreadLocalMetaData,MetaData from elixir import * +from elixir import options +# migrate from elixir 06 to 07 +options.MIGRATION_TO_07_AID = True + # Nodes with a ratio below this value will be removed from consideration # for higher-valued nodes @@ -291,7 +295,7 @@ class RouterStats(Entity): _compute_stats_relation = Callable(_compute_stats_relation) def _compute_stats_query(stats_clause): - tc_session.clear() + tc_session.expunge_all() # http://www.sqlalchemy.org/docs/04/sqlexpression.html#sql_update to_s = select([func.count(Extension.id)], and_(stats_clause, Extension.table.c.to_node_idhex @@ -404,7 +408,7 @@ class RouterStats(Entity): _compute_ranks = Callable(_compute_ranks) def _compute_ratios(stats_clause): - tc_session.clear() + tc_session.expunge_all() avg_from_rate = select([func.avg(RouterStats.circ_from_rate)], stats_clause).as_scalar() avg_to_rate = select([func.avg(RouterStats.circ_to_rate)], @@ -476,7 +480,7 @@ class RouterStats(Entity): _compute_filtered_ratios = Callable(_compute_filtered_ratios) def reset(): - tc_session.clear() + tc_session.expunge_all() RouterStats.table.drop() RouterStats.table.create() for r in Router.query.all(): @@ -637,7 +641,7 @@ def reset_all(): tc_session.add(r) tc_session.commit() - tc_session.clear() + tc_session.expunge_all() BwHistory.table.drop() # Will drop subclasses Extension.table.drop()

1 0

[metrics-tasks/master] Add code for second #3338 graph.
by karsten＠torproject.org 24 Jun '11

24 Jun '11

commit 4d17699bf8450a9d6d8aff2e8eb94b935b73df6b Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jun 24 20:40:10 2011 +0200 Add code for second #3338 graph. --- task-3338/.gitignore | 2 + task-3338/README | 2 +- task-3338/dir-bytes-estimate.R | 63 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 1 deletions(-) diff --git a/task-3338/.gitignore b/task-3338/.gitignore new file mode 100644 index 0000000..bd98a73 --- /dev/null +++ b/task-3338/.gitignore @@ -0,0 +1,2 @@ +*.pdf + diff --git a/task-3338/README b/task-3338/README index 4db2c65..85fda1b 100644 --- a/task-3338/README +++ b/task-3338/README @@ -16,5 +16,5 @@ and run daily-users.R. 2. Plot directory bytes estimate -(TODO) + $ R --slave -f dir-bytes-estimate.R diff --git a/task-3338/dir-bytes-estimate.R b/task-3338/dir-bytes-estimate.R new file mode 100644 index 0000000..de74dcf --- /dev/null +++ b/task-3338/dir-bytes-estimate.R @@ -0,0 +1,63 @@ +library("RPostgreSQL") +library("DBI") +library("ggplot2") +library("proto") +library("grid") +library("reshape") +library("plyr") +library("digest") + +db = "tordir" +dbuser = "metrics" +dbpassword= "password" ###### <- put in real password here! + +plot_dirbytes <- function(start, end, path, dpi) { + drv <- dbDriver("PostgreSQL") + con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) + q <- paste("SELECT date, r, bwp, brp, bwn, brn, bwr, brr, dw, bwd ", + "FROM user_stats WHERE date >= '", start, "' AND date <= '", end, + "' AND date < (SELECT MAX(date) FROM user_stats) - 1 ", + "AND country = 'zy' ORDER BY date", sep = "") + rs <- dbSendQuery(con, q) + u <- fetch(rs, n = -1) + dbDisconnect(con) + dbUnloadDriver(drv) + u <- data.frame(date = u$date, + estimated = floor((u$bwp * u$brn / u$bwn - u$brp) / 86400), + estimatedreq = floor((u$bwr * u$brn / u$bwn - u$brr) / 86400), + extrapolated = floor(u$dw * u$bwp / u$bwd / 86400)) + dates <- seq(from = as.Date(start, "%Y-%m-%d"), + to = as.Date(end, "%Y-%m-%d"), by="1 day") + missing <- setdiff(dates, u$date) + if (length(missing) > 0) + u <- rbind(u, + data.frame(date = as.Date(missing, origin = "1970-01-01"), + estimated = NA, estimatedreq = NA, extrapolated = NA)) + u <- melt(u, id.vars = 1) + highest <- u[u$date %in% as.Date(c("2011-04-10", "2011-04-17", + "2011-04-24", "2011-05-29")) & u$variable == "estimatedreq", ] + date_breaks <- date_breaks( + as.numeric(max(as.Date(u$date, "%Y-%m-%d")) - + min(as.Date(u$date, "%Y-%m-%d")))) + ggplot(u, aes(x = as.Date(date, "%Y-%m-%d"), y = value / 2^20, + colour = variable)) + + geom_line() + + scale_x_date(name = paste("\nThe Tor Project - ", + "https://metrics.torproject.org/", sep = "")) + + scale_y_continuous(name = "", + limits = c(0, max(u$value, na.rm = TRUE) / 2^20)) + + scale_colour_hue(name = "", + breaks = c("extrapolated", "estimated", "estimatedreq"), + labels = c(paste("extrapolated from reported directory bytes (all", + "directory mirrors)"), + "estimated from total bytes (all directory mirrors)", + paste("estimated from total bytes (only directory mirrors", + "reporting stats)"))) + + geom_point(data = highest, size = 3, colour = alpha("purple", 0.35)) + + opts(title = paste("Estimated vs. extrapolated written directory", + "bytes in MiB/s per day"), + legend.position = "top") + ggsave(filename = path, width = 8, height = 5, dpi = as.numeric(dpi)) +} +plot_dirbytes("2009-08-19", "2011-06-23", "dir-bytes-estimate.pdf", 300) +

1 0

[pytorctl/master] Eliminate a warn about a consensus miscount.
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit 3c438e013b2301f2c5e04bb1aceb907ae5f96576 Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Fri Jun 24 10:46:59 2011 -0700 Eliminate a warn about a consensus miscount. By fixing the miscount, of course. --- TorCtl.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diff --git a/TorCtl.py b/TorCtl.py index edf9408..ac49574 100755 --- a/TorCtl.py +++ b/TorCtl.py @@ -1772,8 +1772,9 @@ class ConsensusTracker(EventHandler): r = r[0] ns = ns[0] - if ns.idhex in self.routers and self.routers[ns.idhex].orhash == r.orhash: - plog("NOTICE", + if ns.idhex in self.routers: + if self.routers[ns.idhex].orhash == r.orhash: + plog("NOTICE", "Got extra NEWDESC event for router "+ns.nickname+"="+ns.idhex) else: self.consensus_count += 1

1 0

r24837: {website} Remove links to torstatus.kgprog.com, the site has been down (in website/trunk: about/en docs/en)
by Sebastian Hahn 24 Jun '11

24 Jun '11

Author: sebastian Date: 2011-06-24 15:56:56 +0000 (Fri, 24 Jun 2011) New Revision: 24837 Modified: website/trunk/about/en/volunteers.wml website/trunk/docs/en/documentation.wml Log: Remove links to torstatus.kgprog.com, the site has been down for a while Modified: website/trunk/about/en/volunteers.wml =================================================================== --- website/trunk/about/en/volunteers.wml 2011-06-23 11:30:47 UTC (rev 24836) +++ website/trunk/about/en/volunteers.wml 2011-06-24 15:56:56 UTC (rev 24837) @@ -18,8 +18,7 @@ tirelessly answers the phone when somebody in the world has a legal question about Tor.</dd> <dt>Marco Bonetti</dt><dd>Focusing on MobileTor for the iPhone.</dd> -<dt>Kasimir Gabert</dt><dd>Maintains the <a -href="https://torstatus.kgprog.com/">TorStatus</a> statistics pages.</dd> +<dt>Kasimir Gabert</dt><dd>Maintains TorStatus.</dd> <dt>Andreas Jonsson</dt><dd>Works on OS X Tor Browser Bundle sandbox technology.</dd> <dt>Fabian Keil</dt><dd>One of the core Privoxy developers, and also a Modified: website/trunk/docs/en/documentation.wml =================================================================== --- website/trunk/docs/en/documentation.wml 2011-06-23 11:30:47 UTC (rev 24836) +++ website/trunk/docs/en/documentation.wml 2011-06-24 15:56:56 UTC (rev 24837) @@ -250,8 +250,7 @@ Tor detector</a> or <a href="http://torcheck.xenobite.eu/">the other Tor detector</a> try to guess if you're using Tor or not.</li> <li>Check out one of the Tor status pages, such as <a - href="http://torstatus.blutmagie.de/">blutmagie's</a>, or <a - href="http://trunk.torstatus.kgprog.com/index.php">kgprog's</a>, or + href="http://torstatus.blutmagie.de/">blutmagie's</a>, or Xenobite's <a href="https://torstat.xenobite.eu/">Tor node status</a> page. Remember that these lists may not be as accurate as what your Tor client uses, because your client fetches its own directory information and

1 0

[tor/master] reach back in time and name a config option correctly
by arma＠torproject.org 24 Jun '11

24 Jun '11

commit 694cd7ade985e402b6c62a2f8a970c83313ca267 Author: Roger Dingledine <arma(a)torproject.org> Date: Thu Jun 23 23:54:34 2011 -0400 reach back in time and name a config option correctly --- ChangeLog | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 2a445e9..05a0587 100644 --- a/ChangeLog +++ b/ChangeLog @@ -366,9 +366,9 @@ Changes in version 0.2.3.1-alpha - 2011-05-05 by downloading microdescriptors instead of router descriptors. o Minor features: - - Make logging resolution configurable with a new LogGranularity - option, and change the default from 1 millisecond to 1 - second. Implements enhancement 1668. + - Make logging resolution configurable with a new LogTimeGranularity + option, and change the default from 1 millisecond to 1 second. + Implements enhancement 1668. - We log which torrc file we're using on startup. Implements ticket 2444. - Ordinarily, Tor does not count traffic from private addresses (like

1 0