commit eb736ba5e24d1e4259db921bda52ab918c043553 Author: aagbsn <aagbsn@extc.org> Date: Thu Jun 23 15:40:05 2011 -0700 Added refresh_all() and warnings to reset_all() SQLSupport.refresh_all() is required to keep Elixir and tc_session in sync. Otherwise it is possible for routers added by the consensus update to not show up in queries using the Elixir model i.e. Router.query.all() Also, warnings have been added to SQLSupport.reset_all() because this does not work properly -- in some cases relation tables were not being reset properly (this resulted in old bw measurements being repeated in future output!). Finally, even when reset_all() works properly, bwauthority memory usage continues to grow. --- SQLSupport.py | 27 +++++++++++++++++++++++++++ ScanSupport.py | 1 + 2 files changed, 28 insertions(+), 0 deletions(-) diff --git a/SQLSupport.py b/SQLSupport.py index 8eefa20..cf2c753 100644 --- a/SQLSupport.py +++ b/SQLSupport.py @@ -657,7 +657,13 @@ class RouterStats(Entity): #################### Model Support ################ def reset_all(): + plog("WARN", "SQLSupport.reset_all() called. See SQLSupport.py for details") # Need to keep routers around.. + # WARNING! + # Must keep the routers around because circ_status_event may + # reference old Routers that are no longer in consensus + # and will raise an ObjectDeletedError. See function circ_status_event in + # class CircuitListener in SQLSupport.py for r in Router.query.all(): # This appears to be needed. the relation tables do not get dropped # automatically. @@ -671,6 +677,16 @@ def reset_all(): tc_session.commit() tc_session.expunge_all() + # WARNING! + # May not clear relation all tables! (SQLAlchemy or Elixir bug) + # Try: + # tc_session.execute('delete from router_streams__stream;') + + # WARNING! + # This will cause Postgres databases to hang + # on DROP TABLE. Possibly an issue with cascade. + # Sqlite works though. + BwHistory.table.drop() # Will drop subclasses Extension.table.drop() Stream.table.drop() @@ -691,6 +707,17 @@ def reset_all(): plog("NOTICE", "Reset all SQL stats") +def refresh_all(): + # necessary to keep all sessions synchronized + # This is probably a bug. See reset_all() above. + # Call this after update_consensus(), _update_rank_history() + # See: ScanSupport.reset_stats() + # Could be a cascade problem too, see: + # http://stackoverflow.com/questions/3481976/sqlalchemy-objectdeletederror-ins... + # Also see: + # http://groups.google.com/group/sqlalchemy/browse_thread/thread/c9099eaaffd7c... + [tc_session.refresh(r) for r in Router.query.all()] + ##################### End Model Support #################### class ConsensusTrackerListener(TorCtl.DualEventListener): diff --git a/ScanSupport.py b/ScanSupport.py index 97c43a8..8985fd7 100644 --- a/ScanSupport.py +++ b/ScanSupport.py @@ -268,6 +268,7 @@ class SQLScanHandler(ScanHandler): SQLSupport.reset_all() this.sql_consensus_listener.update_consensus() this.sql_consensus_listener._update_rank_history(this.sql_consensus_listener.consensus.ns_map.iterkeys()) + SQLSupport.refresh_all() cond.notify() cond.release() cond.acquire()