commit ba9799e20e8046ef1316616a3c665f8e4558665a Merge: d1218e0 1920baf Author: Damian Johnson atagar@torproject.org Date: Tue Oct 8 00:42:46 2013 -0700
Reducing memory usage of descriptor content
Handful of improvements to reduce the memory usage of stem's descriptor content (with a special focus on server descriptors). This drops memory usage by roughly 20% without having a sizable impact on runtime.
Script used for testing:
import resource import time
from stem.descriptor import reader
print "memory (initial): %s" % resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
start_time = time.time()
with reader.DescriptorReader(['/home/atagar/.tor/cached-descriptors']) as descriptor_reader: server_descriptors = list(descriptor_reader)
print "memory (after read): %s" % resource.getrusage(resource.RUSAGE_SELF).ru_maxrss print "read time: %0.2f seconds" % (time.time() - start_time)
Before:
% python scratch.py memory (initial): 7468 memory (after read): 50572 read time: 6.40 seconds
After:
% python scratch.py memory (initial): 7532 memory (after read): 40816 read time: 6.31 seconds
stem/descriptor/networkstatus.py | 2 +- stem/descriptor/router_status_entry.py | 2 +- stem/descriptor/server_descriptor.py | 13 +- stem/exit_policy.py | 264 +++++++++++++++----------------- stem/util/lru_cache.py | 10 +- stem/version.py | 11 ++ test/unit/exit_policy/policy.py | 22 --- 7 files changed, 152 insertions(+), 172 deletions(-)