[or-cvs] [pytorctl/master] Develop a better method to wait for consensus.

[mikeperry at torproject.org]

Author: Mike Perry <mikeperry-git at fscked.org>
Date: Tue, 10 Aug 2010 03:01:55 -0700
Subject: Develop a better method to wait for consensus.
Commit: cb91046748c407ddf477bb28adf14ab0b4a4242b

Do not return from wait_for_consensus unless we have at least 95% of all of
our router descriptors.

Also, sanity check that we aren't storing waay too many router descriptors.
---
 SQLSupport.py |    6 +++++-
 TorCtl.py     |   21 +++++++++++++++++++--
 2 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/SQLSupport.py b/SQLSupport.py
index e062d38..509097e 100644
--- a/SQLSupport.py
+++ b/SQLSupport.py
@@ -749,7 +749,11 @@ class ConsensusTrackerListener(TorCtl.DualEventListener):
       # on a condition used to signal we are doing this (and other) heavy 
       # lifting. We could have them possibly check self.last_desc_at..
       if not self.wait_for_signal and e.arrived_at - self.last_desc_at > 60.0:
-        if not PathSupport.PathBuilder.is_urgent_event(e):
+        if self.consensus.consensus_count  < 0.95*(len(self.consensus.ns_map)):
+          plog("INFO", "Not enough router descriptors: "
+                       +str(self.consensus.consensus_count)+"/"
+                       +str(len(self.consensus.ns_map)))
+        elif not PathSupport.PathBuilder.is_urgent_event(e):
           plog("INFO", "Newdesc timer is up. Assuming we have full consensus")
           self._update_rank_history(self.consensus.ns_map.iterkeys())
           self.last_desc_at = ConsensusTrackerListener.CONSENSUS_DONE
diff --git a/TorCtl.py b/TorCtl.py
index 3e3a034..1b08a19 100755
--- a/TorCtl.py
+++ b/TorCtl.py
@@ -1321,11 +1321,12 @@ class Consensus:
   of subsequent updates, use copy.deepcopy()
   """
 
-  def __init__(self, ns_map, sorted_r, router_map, nick_map):
+  def __init__(self, ns_map, sorted_r, router_map, nick_map, consensus_count):
     self.ns_map = ns_map
     self.sorted_r = sorted_r
     self.routers = router_map
     self.name_to_key = nick_map
+    self.consensus_count = consensus_count
 
 class ConsensusTracker(EventHandler):
   """
@@ -1340,8 +1341,11 @@ class ConsensusTracker(EventHandler):
     self.sorted_r = []
     self.name_to_key = {}
     self.RouterClass = RouterClass
+    self.consensus_count = 0
     self.update_consensus()
 
+  # XXX: If there were a potential memory leak through perpetually referenced
+  # objects, this function would be the #1 suspect.
   def _read_routers(self, nslist):
     # Routers can fall out of our consensus five different ways:
     # 1. Their descriptors disappear
@@ -1350,6 +1354,7 @@ class ConsensusTracker(EventHandler):
     # 4. They list a bandwidth of 0
     # 5. They have 'opt hibernating' set
     routers = self.c.read_routers(nslist) # Sets .down if 3,4,5
+    self.consensus_count = len(routers)
     old_idhexes = set(self.routers.keys())
     new_idhexes = set(map(lambda r: r.idhex, routers)) 
     for r in routers:
@@ -1392,6 +1397,12 @@ class ConsensusTracker(EventHandler):
     self._sanity_check(self.sorted_r)
 
   def _sanity_check(self, list):
+    if len(self.routers) > 1.5*self.consensus_count:
+      plog("WARN", "Router count of "+str(len(self.routers))+" exceeds consensus count "+str(len(self.consensus_count))+" by more than 50%")
+
+    if len(self.ns_map) > self.consensus_count:
+      plog("WARN", "NS map count of "+str(len(self.ns_map))+" exceeds consensus count "+str(len(self.consensus_count)))
+
     downed =  filter(lambda r: r.down, list)
     for d in downed:
       plog("WARN", "Router "+d.idhex+" still present but is down. Del: "+str(d.deleted)+", flags: "+str(d.flags)+", bw: "+str(d.bw))
@@ -1435,8 +1446,14 @@ class ConsensusTracker(EventHandler):
         continue
       elif len(r) != 1:
         plog("WARN", "Multiple descs for "+i+" after NEWDESC")
+
       r = r[0]
       ns = ns[0]
+      if ns.idhex in self.routers and self.routers[ns.idhex].orhash == r.orhash:
+        plog("NOTICE",
+             "Got extra NEWDESC event for router "+ns.nickname+"="+ns.idhex)
+      else:
+        self.consensus_count += 1
       self.name_to_key[ns.nickname] = "$"+ns.idhex
       if r and r.idhex in self.ns_map:
         if ns.orhash != self.ns_map[r.idhex].orhash:
@@ -1459,7 +1476,7 @@ class ConsensusTracker(EventHandler):
 
   def current_consensus(self):
     return Consensus(self.ns_map, self.sorted_r, self.routers, 
-                     self.name_to_key)
+                     self.name_to_key, self.consensus_count)
 
 class DebugEventHandler(EventHandler):
   """Trivial debug event handler: reassembles all parsed events to stdout."""
-- 
1.7.1