- tor-commits - lists.torproject.org

[pytorctl/master] Merge remote branch 'aagbsn/testing'
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit d0f593f2ae245e951323ff7fc3a65fa6a1f16b47 Merge: 3c438e0 eb736ba Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Fri Jun 24 11:57:47 2011 -0700 Merge remote branch 'aagbsn/testing' SQLSupport.py | 101 ++++++++++++++++++++++++++++++++++++++++++++------------ ScanSupport.py | 18 +++++++++- 2 files changed, 96 insertions(+), 23 deletions(-)

1 0

[pytorctl/master] added reset_stats() to Scansupport.py
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit 13307d9cc0ed54bf5602c2c9b5e8db86113f3a45 Author: aagbsn <aagbsn(a)extc.org> Date: Mon Apr 18 10:38:50 2011 -0700 added reset_stats() to Scansupport.py calls parent and SQLSupport reset functions Tests show that SQLSupport.reset_all() may clear too much because if BwAuthority calls Scansupport.reset_stats() after each speedrace() run only the first slice is properly recorded; the rest are empty. See: https://trac.torproject.org/projects/tor/ticket/2947 --- ScanSupport.py | 13 ++++++++++++- 1 files changed, 12 insertions(+), 1 deletions(-) diff --git a/ScanSupport.py b/ScanSupport.py index f2a39d9..c425fd2 100644 --- a/ScanSupport.py +++ b/ScanSupport.py @@ -260,4 +260,15 @@ class SQLScanHandler(ScanHandler): cond.release() plog("INFO", "Consensus OK") - + def reset_stats(self): + cond = threading.Condition() + def notlambda(this): + cond.acquire() + ScanHandler.reset_stats(self) + SQLSupport.reset_all() + cond.notify() + cond.release() + cond.acquire() + self.schedule_low_prio(notlambda) + cond.wait() + cond.release()

1 0

[pytorctl/master] update consensus after resetting stats within the same job
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit e1266cb4d716a8a066ea3d92fb3d19305939a058 Author: aagbsn <aagbsn(a)extc.org> Date: Thu May 19 14:17:12 2011 -0700 update consensus after resetting stats within the same job --- ScanSupport.py | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/ScanSupport.py b/ScanSupport.py index c425fd2..97c43a8 100644 --- a/ScanSupport.py +++ b/ScanSupport.py @@ -266,6 +266,8 @@ class SQLScanHandler(ScanHandler): cond.acquire() ScanHandler.reset_stats(self) SQLSupport.reset_all() + this.sql_consensus_listener.update_consensus() + this.sql_consensus_listener._update_rank_history(this.sql_consensus_listener.consensus.ns_map.iterkeys()) cond.notify() cond.release() cond.acquire()

1 0

[pytorctl/master] Added refresh_all() and warnings to reset_all()
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit eb736ba5e24d1e4259db921bda52ab918c043553 Author: aagbsn <aagbsn(a)extc.org> Date: Thu Jun 23 15:40:05 2011 -0700 Added refresh_all() and warnings to reset_all() SQLSupport.refresh_all() is required to keep Elixir and tc_session in sync. Otherwise it is possible for routers added by the consensus update to not show up in queries using the Elixir model i.e. Router.query.all() Also, warnings have been added to SQLSupport.reset_all() because this does not work properly -- in some cases relation tables were not being reset properly (this resulted in old bw measurements being repeated in future output!). Finally, even when reset_all() works properly, bwauthority memory usage continues to grow. --- SQLSupport.py | 27 +++++++++++++++++++++++++++ ScanSupport.py | 1 + 2 files changed, 28 insertions(+), 0 deletions(-) diff --git a/SQLSupport.py b/SQLSupport.py index 8eefa20..cf2c753 100644 --- a/SQLSupport.py +++ b/SQLSupport.py @@ -657,7 +657,13 @@ class RouterStats(Entity): #################### Model Support ################ def reset_all(): + plog("WARN", "SQLSupport.reset_all() called. See SQLSupport.py for details") # Need to keep routers around.. + # WARNING! + # Must keep the routers around because circ_status_event may + # reference old Routers that are no longer in consensus + # and will raise an ObjectDeletedError. See function circ_status_event in + # class CircuitListener in SQLSupport.py for r in Router.query.all(): # This appears to be needed. the relation tables do not get dropped # automatically. @@ -671,6 +677,16 @@ def reset_all(): tc_session.commit() tc_session.expunge_all() + # WARNING! + # May not clear relation all tables! (SQLAlchemy or Elixir bug) + # Try: + # tc_session.execute('delete from router_streams__stream;') + + # WARNING! + # This will cause Postgres databases to hang + # on DROP TABLE. Possibly an issue with cascade. + # Sqlite works though. + BwHistory.table.drop() # Will drop subclasses Extension.table.drop() Stream.table.drop() @@ -691,6 +707,17 @@ def reset_all(): plog("NOTICE", "Reset all SQL stats") +def refresh_all(): + # necessary to keep all sessions synchronized + # This is probably a bug. See reset_all() above. + # Call this after update_consensus(), _update_rank_history() + # See: ScanSupport.reset_stats() + # Could be a cascade problem too, see: + # http://stackoverflow.com/questions/3481976/sqlalchemy-objectdeletederror-in… + # Also see: + # http://groups.google.com/group/sqlalchemy/browse_thread/thread/c9099eaaffd7… + [tc_session.refresh(r) for r in Router.query.all()] + ##################### End Model Support #################### class ConsensusTrackerListener(TorCtl.DualEventListener): diff --git a/ScanSupport.py b/ScanSupport.py index 97c43a8..8985fd7 100644 --- a/ScanSupport.py +++ b/ScanSupport.py @@ -268,6 +268,7 @@ class SQLScanHandler(ScanHandler): SQLSupport.reset_all() this.sql_consensus_listener.update_consensus() this.sql_consensus_listener._update_rank_history(this.sql_consensus_listener.consensus.ns_map.iterkeys()) + SQLSupport.refresh_all() cond.notify() cond.release() cond.acquire()

1 0

[pytorctl/master] backward compatibility with SQLAlchemy 0.5.x
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit c8fee8b7b5ea6c584bb5a75647ee44737e60b66d Author: aagbsn <aagbsn(a)extc.org> Date: Sun Apr 10 23:59:09 2011 -0700 backward compatibility with SQLAlchemy 0.5.x --- SQLSupport.py | 6 +++++- 1 files changed, 5 insertions(+), 1 deletions(-) diff --git a/SQLSupport.py b/SQLSupport.py index 2bd0a8f..9e1a4cb 100644 --- a/SQLSupport.py +++ b/SQLSupport.py @@ -63,6 +63,10 @@ def setup_db(db_uri, echo=False, drop=False): # wouldn't kill you, you know. tc_session.add = tc_session.save_or_update + if sqlalchemy.__version__ < "0.6.0": + # clear() replaced with expunge_all + tc_session.clear = tc_session.expunge_all + class Router(Entity): using_options(shortnames=True, order_by='-published', session=tc_session, metadata=tc_metadata) using_mapper_options(save_on_init=False) @@ -374,7 +378,7 @@ class RouterStats(Entity): _compute_stats = Callable(_compute_stats) def _compute_ranks(): - tc_session.clear() + tc_session.expunge_all() min_r = select([func.min(BwHistory.rank)], BwHistory.table.c.router_idhex == RouterStats.table.c.router_idhex).as_scalar()

1 0

[pytorctl/master] rewrite query for mysql compatibility attempt 1
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit a416402b0a4c6b5406448a5f87538081f26d9368 Author: aagbsn <aagbsn(a)extc.org> Date: Thu Apr 21 18:28:17 2011 -0700 rewrite query for mysql compatibility attempt 1 --- SQLSupport.py | 11 ++++++++++- 1 files changed, 10 insertions(+), 1 deletions(-) diff --git a/SQLSupport.py b/SQLSupport.py index 9e1a4cb..8ef8b68 100644 --- a/SQLSupport.py +++ b/SQLSupport.py @@ -405,9 +405,18 @@ class RouterStats(Entity): #min_avg_rank = select([func.min(RouterStats.avg_rank)]).as_scalar() max_avg_rank = select([func.max(RouterStats.avg_rank)]).as_scalar() + # this query breaks MySQL! + #RouterStats.table.update(values= + # {RouterStats.table.c.percentile: + # (100.0*RouterStats.table.c.avg_rank)/max_avg_rank}).execute() + + # I think the problem is the reference to RouterStats.table.c.avg_rank -- let's just replace it with the query used above. I think the nested query will be OK + #query('UPDATE routerstats SET percentile=((%s * routerstats.avg_rank) / (SELECT max(routerstats.avg_rank) AS max_1 \nFROM routerstats))' (100.0,)) RouterStats.table.update(values= {RouterStats.table.c.percentile: - (100.0*RouterStats.table.c.avg_rank)/max_avg_rank}).execute() + (100.0*avg_r)/max_avg_rank}).execute() + + tc_session.commit() _compute_ranks = Callable(_compute_ranks)

1 0

[pytorctl/master] fixes for divide-by-zeros
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit fbf2fcf865bc0d9b65841076fd88e31f321638de Author: aagbsn <aagbsn(a)extc.org> Date: Fri Apr 29 16:21:40 2011 -0700 fixes for divide-by-zeros Postgres doesn't ignore divide-by-zeros like MySQL CASE statement added to set the result to NULL if the denominator is zero --- SQLSupport.py | 15 ++++++++++----- 1 files changed, 10 insertions(+), 5 deletions(-) diff --git a/SQLSupport.py b/SQLSupport.py index ec29bda..e8feed9 100644 --- a/SQLSupport.py +++ b/SQLSupport.py @@ -25,7 +25,7 @@ import sqlalchemy import sqlalchemy.orm.exc from sqlalchemy.orm import scoped_session, sessionmaker, eagerload, lazyload, eagerload_all from sqlalchemy import create_engine, and_, or_, not_, func -from sqlalchemy.sql import func,select,alias +from sqlalchemy.sql import func,select,alias,case from sqlalchemy.schema import ThreadLocalMetaData,MetaData from elixir import * from elixir import options @@ -328,15 +328,20 @@ class RouterStats(Entity): RouterStats.table.c.circ_fail_from:f_from_s, RouterStats.table.c.avg_first_ext:avg_ext}).execute() + # added case() to set NULL and avoid divide-by-zeros (Postgres) RouterStats.table.update(stats_clause, values= {RouterStats.table.c.circ_from_rate: - RouterStats.table.c.circ_fail_from/RouterStats.table.c.circ_try_from, + case([(RouterStats.table.c.circ_try_from == 0, None)], + else_=(RouterStats.table.c.circ_fail_from/RouterStats.table.c.circ_try_from)), RouterStats.table.c.circ_to_rate: - RouterStats.table.c.circ_fail_to/RouterStats.table.c.circ_try_to, + case([(RouterStats.table.c.circ_try_to == 0, None)], + else_=(RouterStats.table.c.circ_fail_to/RouterStats.table.c.circ_try_to)), RouterStats.table.c.circ_bi_rate: - (RouterStats.table.c.circ_fail_to+RouterStats.table.c.circ_fail_from) + case([(RouterStats.table.c.circ_try_to+RouterStats.table.c.circ_try_from == 0, None)], + else_=((RouterStats.table.c.circ_fail_to+RouterStats.table.c.circ_fail_from) / - (RouterStats.table.c.circ_try_to+RouterStats.table.c.circ_try_from)}).execute() + (RouterStats.table.c.circ_try_to+RouterStats.table.c.circ_try_from))), + }).execute() # TODO: Give the streams relation table a sane name and reduce this too

1 0

[pytorctl/master] rewrite query for mysql compatibility attempt 2
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit 3a2b44864417908bae45e835e0502297671e6c7b Author: aagbsn <aagbsn(a)extc.org> Date: Sun Apr 24 21:12:56 2011 -0700 rewrite query for mysql compatibility attempt 2 this actually appears to work --- SQLSupport.py | 49 +++++++++++++++++++++++++++++-------------------- 1 files changed, 29 insertions(+), 20 deletions(-) diff --git a/SQLSupport.py b/SQLSupport.py index 8ef8b68..ec29bda 100644 --- a/SQLSupport.py +++ b/SQLSupport.py @@ -25,7 +25,7 @@ import sqlalchemy import sqlalchemy.orm.exc from sqlalchemy.orm import scoped_session, sessionmaker, eagerload, lazyload, eagerload_all from sqlalchemy import create_engine, and_, or_, not_, func -from sqlalchemy.sql import func,select +from sqlalchemy.sql import func,select,alias from sqlalchemy.schema import ThreadLocalMetaData,MetaData from elixir import * from elixir import options @@ -403,35 +403,44 @@ class RouterStats(Entity): RouterStats.table.c.avg_desc_bw:avg_desc_bw}).execute() #min_avg_rank = select([func.min(RouterStats.avg_rank)]).as_scalar() - max_avg_rank = select([func.max(RouterStats.avg_rank)]).as_scalar() - # this query breaks MySQL! - #RouterStats.table.update(values= - # {RouterStats.table.c.percentile: - # (100.0*RouterStats.table.c.avg_rank)/max_avg_rank}).execute() + # the commented query breaks mysql because UPDATE cannot reference + # target table in the FROM clause. So we throw in an anonymous alias and wrap + # another select around it in order to get the nested SELECT stored into a + # temporary table. + # FIXME: performance? no idea + #max_avg_rank = select([func.max(RouterStats.avg_rank)]).as_scalar() + max_avg_rank = select([alias(select([func.max(RouterStats.avg_rank)]))]).as_scalar() - # I think the problem is the reference to RouterStats.table.c.avg_rank -- let's just replace it with the query used above. I think the nested query will be OK - #query('UPDATE routerstats SET percentile=((%s * routerstats.avg_rank) / (SELECT max(routerstats.avg_rank) AS max_1 \nFROM routerstats))' (100.0,)) RouterStats.table.update(values= {RouterStats.table.c.percentile: - (100.0*avg_r)/max_avg_rank}).execute() - + (100.0*RouterStats.table.c.avg_rank)/max_avg_rank}).execute() tc_session.commit() _compute_ranks = Callable(_compute_ranks) def _compute_ratios(stats_clause): tc_session.expunge_all() - avg_from_rate = select([func.avg(RouterStats.circ_from_rate)], - stats_clause).as_scalar() - avg_to_rate = select([func.avg(RouterStats.circ_to_rate)], - stats_clause).as_scalar() - avg_bi_rate = select([func.avg(RouterStats.circ_bi_rate)], - stats_clause).as_scalar() - avg_ext = select([func.avg(RouterStats.avg_first_ext)], - stats_clause).as_scalar() - avg_sbw = select([func.avg(RouterStats.sbw)], - stats_clause).as_scalar() + avg_from_rate = select([alias( + select([func.avg(RouterStats.circ_from_rate)], + stats_clause) + )]).as_scalar() + avg_to_rate = select([alias( + select([func.avg(RouterStats.circ_to_rate)], + stats_clause) + )]).as_scalar() + avg_bi_rate = select([alias( + select([func.avg(RouterStats.circ_bi_rate)], + stats_clause) + )]).as_scalar() + avg_ext = select([alias( + select([func.avg(RouterStats.avg_first_ext)], + stats_clause) + )]).as_scalar() + avg_sbw = select([alias( + select([func.avg(RouterStats.sbw)], + stats_clause) + )]).as_scalar() RouterStats.table.update(stats_clause, values= {RouterStats.table.c.circ_from_ratio:

1 0

[pytorctl/master] generalize db message, we now support postgres and mysql too
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit ffb68994a4d440db9deca2624ed33075f3fb7125 Author: aagbsn <aagbsn(a)extc.org> Date: Fri Jun 17 10:53:05 2011 -0700 generalize db message, we now support postgres and mysql too --- ScanSupport.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/ScanSupport.py b/ScanSupport.py index d102c78..f2a39d9 100644 --- a/ScanSupport.py +++ b/ScanSupport.py @@ -182,7 +182,7 @@ class SQLScanHandler(ScanHandler): ScanHandler.__init__(self, c, selmgr, RouterClass, strm_selector) def attach_sql_listener(self, db_uri): - plog("DEBUG", "Got sqlite: "+db_uri) + plog("DEBUG", "Got db: "+db_uri) SQLSupport.setup_db(db_uri, echo=False, drop=True) self.sql_consensus_listener = SQLSupport.ConsensusTrackerListener() self.add_event_listener(self.sql_consensus_listener)

1 0

[pytorctl/master] add case for long
by mikeperry＠torproject.org 24 Jun '11

24 Jun '11

commit ef0b770e9fa3ddb63ec52dfa4796c7dff27e8709 Author: aagbsn <aagbsn(a)extc.org> Date: Sat Jun 18 16:36:24 2011 -0700 add case for long --- SQLSupport.py | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/SQLSupport.py b/SQLSupport.py index e8feed9..8eefa20 100644 --- a/SQLSupport.py +++ b/SQLSupport.py @@ -568,6 +568,7 @@ class RouterStats(Entity): def cvt(a,b,c=1): if type(a) == float: return round(a/c,b) elif type(a) == int: return a + elif type(a) == long: return a elif type(a) == type(None): return "None" else: return type(a)

1 0