June 2011 - tor-commits - lists.torproject.org

[pytorctl/master] Added refresh_all() and warnings to reset_all()
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit eb736ba5e24d1e4259db921bda52ab918c043553 Author: aagbsn <aagbsn(a)extc.org> Date: Thu Jun 23 15:40:05 2011 -0700 Added refresh_all() and warnings to reset_all() SQLSupport.refresh_all() is required to keep Elixir and tc_session in sync. Otherwise it is possible for routers added by the consensus update to not show up in queries using the Elixir model i.e. Router.query.all() Also, warnings have been added to SQLSupport.reset_all() because this does not work properly -- in some cases relation tables were not being reset properly (this resulted in old bw measurements being repeated in future output!). Finally, even when reset_all() works properly, bwauthority memory usage continues to grow. --- SQLSupport.py | 27 +++++++++++++++++++++++++++ ScanSupport.py | 1 + 2 files changed, 28 insertions(+), 0 deletions(-) diff --git a/SQLSupport.py b/SQLSupport.py index 8eefa20..cf2c753 100644 --- a/SQLSupport.py +++ b/SQLSupport.py @@ -657,7 +657,13 @@ class RouterStats(Entity): #################### Model Support ################ def reset_all(): + plog("WARN", "SQLSupport.reset_all() called. See SQLSupport.py for details") # Need to keep routers around.. + # WARNING! + # Must keep the routers around because circ_status_event may + # reference old Routers that are no longer in consensus + # and will raise an ObjectDeletedError. See function circ_status_event in + # class CircuitListener in SQLSupport.py for r in Router.query.all(): # This appears to be needed. the relation tables do not get dropped # automatically. @@ -671,6 +677,16 @@ def reset_all(): tc_session.commit() tc_session.expunge_all() + # WARNING! + # May not clear relation all tables! (SQLAlchemy or Elixir bug) + # Try: + # tc_session.execute('delete from router_streams__stream;') + + # WARNING! + # This will cause Postgres databases to hang + # on DROP TABLE. Possibly an issue with cascade. + # Sqlite works though. + BwHistory.table.drop() # Will drop subclasses Extension.table.drop() Stream.table.drop() @@ -691,6 +707,17 @@ def reset_all(): plog("NOTICE", "Reset all SQL stats") +def refresh_all(): + # necessary to keep all sessions synchronized + # This is probably a bug. See reset_all() above. + # Call this after update_consensus(), _update_rank_history() + # See: ScanSupport.reset_stats() + # Could be a cascade problem too, see: + # http://stackoverflow.com/questions/3481976/sqlalchemy-objectdeletederror-in… + # Also see: + # http://groups.google.com/group/sqlalchemy/browse_thread/thread/c9099eaaffd7… + [tc_session.refresh(r) for r in Router.query.all()] + ##################### End Model Support #################### class ConsensusTrackerListener(TorCtl.DualEventListener): diff --git a/ScanSupport.py b/ScanSupport.py index 97c43a8..8985fd7 100644 --- a/ScanSupport.py +++ b/ScanSupport.py @@ -268,6 +268,7 @@ class SQLScanHandler(ScanHandler): SQLSupport.reset_all() this.sql_consensus_listener.update_consensus() this.sql_consensus_listener._update_rank_history(this.sql_consensus_listener.consensus.ns_map.iterkeys()) + SQLSupport.refresh_all() cond.notify() cond.release() cond.acquire()

1 0

[pytorctl/master] backward compatibility with SQLAlchemy 0.5.x
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit c8fee8b7b5ea6c584bb5a75647ee44737e60b66d Author: aagbsn <aagbsn(a)extc.org> Date: Sun Apr 10 23:59:09 2011 -0700 backward compatibility with SQLAlchemy 0.5.x --- SQLSupport.py | 6 +++++- 1 files changed, 5 insertions(+), 1 deletions(-) diff --git a/SQLSupport.py b/SQLSupport.py index 2bd0a8f..9e1a4cb 100644 --- a/SQLSupport.py +++ b/SQLSupport.py @@ -63,6 +63,10 @@ def setup_db(db_uri, echo=False, drop=False): # wouldn't kill you, you know. tc_session.add = tc_session.save_or_update + if sqlalchemy.__version__ < "0.6.0": + # clear() replaced with expunge_all + tc_session.clear = tc_session.expunge_all + class Router(Entity): using_options(shortnames=True, order_by='-published', session=tc_session, metadata=tc_metadata) using_mapper_options(save_on_init=False) @@ -374,7 +378,7 @@ class RouterStats(Entity): _compute_stats = Callable(_compute_stats) def _compute_ranks(): - tc_session.clear() + tc_session.expunge_all() min_r = select([func.min(BwHistory.rank)], BwHistory.table.c.router_idhex == RouterStats.table.c.router_idhex).as_scalar()

1 0

[pytorctl/master] rewrite query for mysql compatibility attempt 1
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit a416402b0a4c6b5406448a5f87538081f26d9368 Author: aagbsn <aagbsn(a)extc.org> Date: Thu Apr 21 18:28:17 2011 -0700 rewrite query for mysql compatibility attempt 1 --- SQLSupport.py | 11 ++++++++++- 1 files changed, 10 insertions(+), 1 deletions(-) diff --git a/SQLSupport.py b/SQLSupport.py index 9e1a4cb..8ef8b68 100644 --- a/SQLSupport.py +++ b/SQLSupport.py @@ -405,9 +405,18 @@ class RouterStats(Entity): #min_avg_rank = select([func.min(RouterStats.avg_rank)]).as_scalar() max_avg_rank = select([func.max(RouterStats.avg_rank)]).as_scalar() + # this query breaks MySQL! + #RouterStats.table.update(values= + # {RouterStats.table.c.percentile: + # (100.0*RouterStats.table.c.avg_rank)/max_avg_rank}).execute() + + # I think the problem is the reference to RouterStats.table.c.avg_rank -- let's just replace it with the query used above. I think the nested query will be OK + #query('UPDATE routerstats SET percentile=((%s * routerstats.avg_rank) / (SELECT max(routerstats.avg_rank) AS max_1 \nFROM routerstats))' (100.0,)) RouterStats.table.update(values= {RouterStats.table.c.percentile: - (100.0*RouterStats.table.c.avg_rank)/max_avg_rank}).execute() + (100.0*avg_r)/max_avg_rank}).execute() + + tc_session.commit() _compute_ranks = Callable(_compute_ranks)

1 0

[pytorctl/master] fixes for divide-by-zeros
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit fbf2fcf865bc0d9b65841076fd88e31f321638de Author: aagbsn <aagbsn(a)extc.org> Date: Fri Apr 29 16:21:40 2011 -0700 fixes for divide-by-zeros Postgres doesn't ignore divide-by-zeros like MySQL CASE statement added to set the result to NULL if the denominator is zero --- SQLSupport.py | 15 ++++++++++----- 1 files changed, 10 insertions(+), 5 deletions(-) diff --git a/SQLSupport.py b/SQLSupport.py index ec29bda..e8feed9 100644 --- a/SQLSupport.py +++ b/SQLSupport.py @@ -25,7 +25,7 @@ import sqlalchemy import sqlalchemy.orm.exc from sqlalchemy.orm import scoped_session, sessionmaker, eagerload, lazyload, eagerload_all from sqlalchemy import create_engine, and_, or_, not_, func -from sqlalchemy.sql import func,select,alias +from sqlalchemy.sql import func,select,alias,case from sqlalchemy.schema import ThreadLocalMetaData,MetaData from elixir import * from elixir import options @@ -328,15 +328,20 @@ class RouterStats(Entity): RouterStats.table.c.circ_fail_from:f_from_s, RouterStats.table.c.avg_first_ext:avg_ext}).execute() + # added case() to set NULL and avoid divide-by-zeros (Postgres) RouterStats.table.update(stats_clause, values= {RouterStats.table.c.circ_from_rate: - RouterStats.table.c.circ_fail_from/RouterStats.table.c.circ_try_from, + case([(RouterStats.table.c.circ_try_from == 0, None)], + else_=(RouterStats.table.c.circ_fail_from/RouterStats.table.c.circ_try_from)), RouterStats.table.c.circ_to_rate: - RouterStats.table.c.circ_fail_to/RouterStats.table.c.circ_try_to, + case([(RouterStats.table.c.circ_try_to == 0, None)], + else_=(RouterStats.table.c.circ_fail_to/RouterStats.table.c.circ_try_to)), RouterStats.table.c.circ_bi_rate: - (RouterStats.table.c.circ_fail_to+RouterStats.table.c.circ_fail_from) + case([(RouterStats.table.c.circ_try_to+RouterStats.table.c.circ_try_from == 0, None)], + else_=((RouterStats.table.c.circ_fail_to+RouterStats.table.c.circ_fail_from) / - (RouterStats.table.c.circ_try_to+RouterStats.table.c.circ_try_from)}).execute() + (RouterStats.table.c.circ_try_to+RouterStats.table.c.circ_try_from))), + }).execute() # TODO: Give the streams relation table a sane name and reduce this too

1 0

[pytorctl/master] rewrite query for mysql compatibility attempt 2
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit 3a2b44864417908bae45e835e0502297671e6c7b Author: aagbsn <aagbsn(a)extc.org> Date: Sun Apr 24 21:12:56 2011 -0700 rewrite query for mysql compatibility attempt 2 this actually appears to work --- SQLSupport.py | 49 +++++++++++++++++++++++++++++-------------------- 1 files changed, 29 insertions(+), 20 deletions(-) diff --git a/SQLSupport.py b/SQLSupport.py index 8ef8b68..ec29bda 100644 --- a/SQLSupport.py +++ b/SQLSupport.py @@ -25,7 +25,7 @@ import sqlalchemy import sqlalchemy.orm.exc from sqlalchemy.orm import scoped_session, sessionmaker, eagerload, lazyload, eagerload_all from sqlalchemy import create_engine, and_, or_, not_, func -from sqlalchemy.sql import func,select +from sqlalchemy.sql import func,select,alias from sqlalchemy.schema import ThreadLocalMetaData,MetaData from elixir import * from elixir import options @@ -403,35 +403,44 @@ class RouterStats(Entity): RouterStats.table.c.avg_desc_bw:avg_desc_bw}).execute() #min_avg_rank = select([func.min(RouterStats.avg_rank)]).as_scalar() - max_avg_rank = select([func.max(RouterStats.avg_rank)]).as_scalar() - # this query breaks MySQL! - #RouterStats.table.update(values= - # {RouterStats.table.c.percentile: - # (100.0*RouterStats.table.c.avg_rank)/max_avg_rank}).execute() + # the commented query breaks mysql because UPDATE cannot reference + # target table in the FROM clause. So we throw in an anonymous alias and wrap + # another select around it in order to get the nested SELECT stored into a + # temporary table. + # FIXME: performance? no idea + #max_avg_rank = select([func.max(RouterStats.avg_rank)]).as_scalar() + max_avg_rank = select([alias(select([func.max(RouterStats.avg_rank)]))]).as_scalar() - # I think the problem is the reference to RouterStats.table.c.avg_rank -- let's just replace it with the query used above. I think the nested query will be OK - #query('UPDATE routerstats SET percentile=((%s * routerstats.avg_rank) / (SELECT max(routerstats.avg_rank) AS max_1 \nFROM routerstats))' (100.0,)) RouterStats.table.update(values= {RouterStats.table.c.percentile: - (100.0*avg_r)/max_avg_rank}).execute() - + (100.0*RouterStats.table.c.avg_rank)/max_avg_rank}).execute() tc_session.commit() _compute_ranks = Callable(_compute_ranks) def _compute_ratios(stats_clause): tc_session.expunge_all() - avg_from_rate = select([func.avg(RouterStats.circ_from_rate)], - stats_clause).as_scalar() - avg_to_rate = select([func.avg(RouterStats.circ_to_rate)], - stats_clause).as_scalar() - avg_bi_rate = select([func.avg(RouterStats.circ_bi_rate)], - stats_clause).as_scalar() - avg_ext = select([func.avg(RouterStats.avg_first_ext)], - stats_clause).as_scalar() - avg_sbw = select([func.avg(RouterStats.sbw)], - stats_clause).as_scalar() + avg_from_rate = select([alias( + select([func.avg(RouterStats.circ_from_rate)], + stats_clause) + )]).as_scalar() + avg_to_rate = select([alias( + select([func.avg(RouterStats.circ_to_rate)], + stats_clause) + )]).as_scalar() + avg_bi_rate = select([alias( + select([func.avg(RouterStats.circ_bi_rate)], + stats_clause) + )]).as_scalar() + avg_ext = select([alias( + select([func.avg(RouterStats.avg_first_ext)], + stats_clause) + )]).as_scalar() + avg_sbw = select([alias( + select([func.avg(RouterStats.sbw)], + stats_clause) + )]).as_scalar() RouterStats.table.update(stats_clause, values= {RouterStats.table.c.circ_from_ratio:

1 0

[pytorctl/master] generalize db message, we now support postgres and mysql too
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit ffb68994a4d440db9deca2624ed33075f3fb7125 Author: aagbsn <aagbsn(a)extc.org> Date: Fri Jun 17 10:53:05 2011 -0700 generalize db message, we now support postgres and mysql too --- ScanSupport.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/ScanSupport.py b/ScanSupport.py index d102c78..f2a39d9 100644 --- a/ScanSupport.py +++ b/ScanSupport.py @@ -182,7 +182,7 @@ class SQLScanHandler(ScanHandler): ScanHandler.__init__(self, c, selmgr, RouterClass, strm_selector) def attach_sql_listener(self, db_uri): - plog("DEBUG", "Got sqlite: "+db_uri) + plog("DEBUG", "Got db: "+db_uri) SQLSupport.setup_db(db_uri, echo=False, drop=True) self.sql_consensus_listener = SQLSupport.ConsensusTrackerListener() self.add_event_listener(self.sql_consensus_listener)

1 0

[pytorctl/master] add case for long
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit ef0b770e9fa3ddb63ec52dfa4796c7dff27e8709 Author: aagbsn <aagbsn(a)extc.org> Date: Sat Jun 18 16:36:24 2011 -0700 add case for long --- SQLSupport.py | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/SQLSupport.py b/SQLSupport.py index e8feed9..8eefa20 100644 --- a/SQLSupport.py +++ b/SQLSupport.py @@ -568,6 +568,7 @@ class RouterStats(Entity): def cvt(a,b,c=1): if type(a) == float: return round(a/c,b) elif type(a) == int: return a + elif type(a) == long: return a elif type(a) == type(None): return "None" else: return type(a)

1 0

[pytorctl/master] SQLAlchemey and Elixir upgrade
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit 5161c64139f18b55288dab07e74f9f3f1c5bb704 Author: aagbsn <aagbsn(a)extc.org> Date: Sun Apr 10 23:24:23 2011 -0700 SQLAlchemey and Elixir upgrade enabled elixir migration aid options. renamed a few function calls, as per SQLAlchemy upgrade docs: session.clear() is removed. use session.remove_all() --- SQLSupport.py | 12 ++++++++---- 1 files changed, 8 insertions(+), 4 deletions(-) diff --git a/SQLSupport.py b/SQLSupport.py index 2532973..2bd0a8f 100644 --- a/SQLSupport.py +++ b/SQLSupport.py @@ -28,6 +28,10 @@ from sqlalchemy import create_engine, and_, or_, not_, func from sqlalchemy.sql import func,select from sqlalchemy.schema import ThreadLocalMetaData,MetaData from elixir import * +from elixir import options +# migrate from elixir 06 to 07 +options.MIGRATION_TO_07_AID = True + # Nodes with a ratio below this value will be removed from consideration # for higher-valued nodes @@ -291,7 +295,7 @@ class RouterStats(Entity): _compute_stats_relation = Callable(_compute_stats_relation) def _compute_stats_query(stats_clause): - tc_session.clear() + tc_session.expunge_all() # http://www.sqlalchemy.org/docs/04/sqlexpression.html#sql_update to_s = select([func.count(Extension.id)], and_(stats_clause, Extension.table.c.to_node_idhex @@ -404,7 +408,7 @@ class RouterStats(Entity): _compute_ranks = Callable(_compute_ranks) def _compute_ratios(stats_clause): - tc_session.clear() + tc_session.expunge_all() avg_from_rate = select([func.avg(RouterStats.circ_from_rate)], stats_clause).as_scalar() avg_to_rate = select([func.avg(RouterStats.circ_to_rate)], @@ -476,7 +480,7 @@ class RouterStats(Entity): _compute_filtered_ratios = Callable(_compute_filtered_ratios) def reset(): - tc_session.clear() + tc_session.expunge_all() RouterStats.table.drop() RouterStats.table.create() for r in Router.query.all(): @@ -637,7 +641,7 @@ def reset_all(): tc_session.add(r) tc_session.commit() - tc_session.clear() + tc_session.expunge_all() BwHistory.table.drop() # Will drop subclasses Extension.table.drop()

1 0

[metrics-tasks/master] Add code for second #3338 graph.
by karsten＠torproject.org 24 Jun '11

24 Jun '11

commit 4d17699bf8450a9d6d8aff2e8eb94b935b73df6b Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jun 24 20:40:10 2011 +0200 Add code for second #3338 graph. --- task-3338/.gitignore | 2 + task-3338/README | 2 +- task-3338/dir-bytes-estimate.R | 63 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 1 deletions(-) diff --git a/task-3338/.gitignore b/task-3338/.gitignore new file mode 100644 index 0000000..bd98a73 --- /dev/null +++ b/task-3338/.gitignore @@ -0,0 +1,2 @@ +*.pdf + diff --git a/task-3338/README b/task-3338/README index 4db2c65..85fda1b 100644 --- a/task-3338/README +++ b/task-3338/README @@ -16,5 +16,5 @@ and run daily-users.R. 2. Plot directory bytes estimate -(TODO) + $ R --slave -f dir-bytes-estimate.R diff --git a/task-3338/dir-bytes-estimate.R b/task-3338/dir-bytes-estimate.R new file mode 100644 index 0000000..de74dcf --- /dev/null +++ b/task-3338/dir-bytes-estimate.R @@ -0,0 +1,63 @@ +library("RPostgreSQL") +library("DBI") +library("ggplot2") +library("proto") +library("grid") +library("reshape") +library("plyr") +library("digest") + +db = "tordir" +dbuser = "metrics" +dbpassword= "password" ###### <- put in real password here! + +plot_dirbytes <- function(start, end, path, dpi) { + drv <- dbDriver("PostgreSQL") + con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) + q <- paste("SELECT date, r, bwp, brp, bwn, brn, bwr, brr, dw, bwd ", + "FROM user_stats WHERE date >= '", start, "' AND date <= '", end, + "' AND date < (SELECT MAX(date) FROM user_stats) - 1 ", + "AND country = 'zy' ORDER BY date", sep = "") + rs <- dbSendQuery(con, q) + u <- fetch(rs, n = -1) + dbDisconnect(con) + dbUnloadDriver(drv) + u <- data.frame(date = u$date, + estimated = floor((u$bwp * u$brn / u$bwn - u$brp) / 86400), + estimatedreq = floor((u$bwr * u$brn / u$bwn - u$brr) / 86400), + extrapolated = floor(u$dw * u$bwp / u$bwd / 86400)) + dates <- seq(from = as.Date(start, "%Y-%m-%d"), + to = as.Date(end, "%Y-%m-%d"), by="1 day") + missing <- setdiff(dates, u$date) + if (length(missing) > 0) + u <- rbind(u, + data.frame(date = as.Date(missing, origin = "1970-01-01"), + estimated = NA, estimatedreq = NA, extrapolated = NA)) + u <- melt(u, id.vars = 1) + highest <- u[u$date %in% as.Date(c("2011-04-10", "2011-04-17", + "2011-04-24", "2011-05-29")) & u$variable == "estimatedreq", ] + date_breaks <- date_breaks( + as.numeric(max(as.Date(u$date, "%Y-%m-%d")) - + min(as.Date(u$date, "%Y-%m-%d")))) + ggplot(u, aes(x = as.Date(date, "%Y-%m-%d"), y = value / 2^20, + colour = variable)) + + geom_line() + + scale_x_date(name = paste("\nThe Tor Project - ", + "https://metrics.torproject.org/", sep = "")) + + scale_y_continuous(name = "", + limits = c(0, max(u$value, na.rm = TRUE) / 2^20)) + + scale_colour_hue(name = "", + breaks = c("extrapolated", "estimated", "estimatedreq"), + labels = c(paste("extrapolated from reported directory bytes (all", + "directory mirrors)"), + "estimated from total bytes (all directory mirrors)", + paste("estimated from total bytes (only directory mirrors", + "reporting stats)"))) + + geom_point(data = highest, size = 3, colour = alpha("purple", 0.35)) + + opts(title = paste("Estimated vs. extrapolated written directory", + "bytes in MiB/s per day"), + legend.position = "top") + ggsave(filename = path, width = 8, height = 5, dpi = as.numeric(dpi)) +} +plot_dirbytes("2009-08-19", "2011-06-23", "dir-bytes-estimate.pdf", 300) +

1 0

[pytorctl/master] Eliminate a warn about a consensus miscount.
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit 3c438e013b2301f2c5e04bb1aceb907ae5f96576 Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Fri Jun 24 10:46:59 2011 -0700 Eliminate a warn about a consensus miscount. By fixing the miscount, of course. --- TorCtl.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diff --git a/TorCtl.py b/TorCtl.py index edf9408..ac49574 100755 --- a/TorCtl.py +++ b/TorCtl.py @@ -1772,8 +1772,9 @@ class ConsensusTracker(EventHandler): r = r[0] ns = ns[0] - if ns.idhex in self.routers and self.routers[ns.idhex].orhash == r.orhash: - plog("NOTICE", + if ns.idhex in self.routers: + if self.routers[ns.idhex].orhash == r.orhash: + plog("NOTICE", "Got extra NEWDESC event for router "+ns.nickname+"="+ns.idhex) else: self.consensus_count += 1

1 0