[tor-commits] [pytorctl/master] Added refresh_all() and warnings to reset_all()

[mikeperry at torproject.org]

commit eb736ba5e24d1e4259db921bda52ab918c043553
Author: aagbsn <aagbsn at extc.org>
Date:   Thu Jun 23 15:40:05 2011 -0700

    Added refresh_all() and warnings to reset_all()
    
    SQLSupport.refresh_all() is required to keep Elixir and
    tc_session in sync. Otherwise it is possible for
    routers added by the consensus update to not show up
    in queries using the Elixir model i.e. Router.query.all()
    
    Also, warnings have been added to SQLSupport.reset_all() because
    this does not work properly -- in some cases relation tables
    were not being reset properly (this resulted in old bw measurements
    being repeated in future output!).
    
    Finally, even when reset_all() works properly, bwauthority memory
    usage continues to grow.
---
 SQLSupport.py  |   27 +++++++++++++++++++++++++++
 ScanSupport.py |    1 +
 2 files changed, 28 insertions(+), 0 deletions(-)

diff --git a/SQLSupport.py b/SQLSupport.py
index 8eefa20..cf2c753 100644
--- a/SQLSupport.py
+++ b/SQLSupport.py
@@ -657,7 +657,13 @@ class RouterStats(Entity):
 
 #################### Model Support ################
 def reset_all():
+  plog("WARN", "SQLSupport.reset_all() called. See SQLSupport.py for details")
   # Need to keep routers around.. 
+  # WARNING!
+  # Must keep the routers around because circ_status_event may
+  # reference old Routers that are no longer in consensus
+  # and will raise an ObjectDeletedError. See function circ_status_event in
+  # class CircuitListener in SQLSupport.py
   for r in Router.query.all():
     # This appears to be needed. the relation tables do not get dropped 
     # automatically.
@@ -671,6 +677,16 @@ def reset_all():
   tc_session.commit()
   tc_session.expunge_all()
 
+  # WARNING!
+  # May not clear relation all tables! (SQLAlchemy or Elixir bug)
+  # Try: 
+  # tc_session.execute('delete from router_streams__stream;')
+
+  # WARNING!
+  # This will cause Postgres databases to hang
+  # on DROP TABLE. Possibly an issue with cascade.
+  # Sqlite works though.
+
   BwHistory.table.drop() # Will drop subclasses
   Extension.table.drop()
   Stream.table.drop() 
@@ -691,6 +707,17 @@ def reset_all():
 
   plog("NOTICE", "Reset all SQL stats")
 
+def refresh_all():
+  # necessary to keep all sessions synchronized
+  # This is probably a bug. See reset_all() above.
+  # Call this after update_consensus(), _update_rank_history()
+  # See: ScanSupport.reset_stats()
+  # Could be a cascade problem too, see:
+  # http://stackoverflow.com/questions/3481976/sqlalchemy-objectdeletederror-instance-class-at-has-been-deleted-help
+  # Also see:
+  # http://groups.google.com/group/sqlalchemy/browse_thread/thread/c9099eaaffd7c348
+  [tc_session.refresh(r) for r in Router.query.all()]
+
 ##################### End Model Support ####################
 
 class ConsensusTrackerListener(TorCtl.DualEventListener):
diff --git a/ScanSupport.py b/ScanSupport.py
index 97c43a8..8985fd7 100644
--- a/ScanSupport.py
+++ b/ScanSupport.py
@@ -268,6 +268,7 @@ class SQLScanHandler(ScanHandler):
       SQLSupport.reset_all()
       this.sql_consensus_listener.update_consensus()
       this.sql_consensus_listener._update_rank_history(this.sql_consensus_listener.consensus.ns_map.iterkeys())
+      SQLSupport.refresh_all()
       cond.notify()
       cond.release()
     cond.acquire()