November 2011 - tor-commits - lists.torproject.org

[torflow/master] Actually remove stale files.
by mikeperry＠torproject.org 17 Nov '11

17 Nov '11

commit 7bf82544ff04c1755667d12e7c2f966a12c9228d Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Fri Nov 4 15:13:05 2011 -0700 Actually remove stale files. Also kill some dead code. --- NetworkScanners/BwAuthority/aggregate.py | 14 +++++++------- 1 files changed, 7 insertions(+), 7 deletions(-) diff --git a/NetworkScanners/BwAuthority/aggregate.py b/NetworkScanners/BwAuthority/aggregate.py index 39a2e85..521aee6 100755 --- a/NetworkScanners/BwAuthority/aggregate.py +++ b/NetworkScanners/BwAuthority/aggregate.py @@ -13,7 +13,6 @@ from TorCtl import TorCtl,TorUtil from TorCtl.PathSupport import VersionRangeRestriction, NodeRestrictionList, NotNodeRestriction bw_files = [] -timestamps = {} nodes = {} prev_consensus = {} @@ -280,16 +279,17 @@ def main(argv): # measure hibernating routers for days. # This filter is just to remove REALLY old files if time.time() - timestamp > MAX_AGE: - plog("DEBUG", "Skipping old file "+f) - # FIXME: Unlink this file + sql- + sqlf = f.replace("bws-", "sql-") + plog("INFO", "Removing old file "+f+" and "+sqlf) + os.remove(sr+"/"+f) + try: + os.remove(sr+"/"+sqlf) + except: + pass # In some cases the sql file may not exist continue if timestamp > newest_timestamp: newest_timestamp = timestamp bw_files.append((slicenum, timestamp, sr+"/"+f)) - # FIXME: Can we kill this? - if slicenum not in timestamps or \ - timestamps[slicenum] < timestamp: - timestamps[slicenum] = timestamp scanner_timestamps[ds] = newest_timestamp # Need to only use most recent slice-file for each node..

1 0

[torflow/master] Submodule.
by mikeperry＠torproject.org 17 Nov '11

17 Nov '11

commit ba4efc2ded84e0867d3f0ea2c81badd8334f2f42 Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Fri Nov 4 15:42:47 2011 -0700 Submodule. --- TorCtl | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/TorCtl b/TorCtl index 66ed91b..391b439 160000 --- a/TorCtl +++ b/TorCtl @@ -1 +1 @@ -Subproject commit 66ed91b5c1453b9c25b7070b702e2e548efb4f80 +Subproject commit 391b4399adda91e8b68ffd6e1fa294efe846debd

1 0

[torflow/master] Bugs 4431+2014: Randomized url support + variable file sizes.
by mikeperry＠torproject.org 17 Nov '11

17 Nov '11

commit 5fdb018fe760def96eca098bee723dea4b5e9049 Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Mon Nov 7 21:16:07 2011 -0800 Bugs 4431+2014: Randomized url support + variable file sizes. Scanner child piece. --- NetworkScanners/BwAuthority/bwauthority_child.py | 35 ++++++++++++--------- 1 files changed, 20 insertions(+), 15 deletions(-) diff --git a/NetworkScanners/BwAuthority/bwauthority_child.py b/NetworkScanners/BwAuthority/bwauthority_child.py index 27dc0ba..1527a11 100755 --- a/NetworkScanners/BwAuthority/bwauthority_child.py +++ b/NetworkScanners/BwAuthority/bwauthority_child.py @@ -23,6 +23,7 @@ import ConfigParser import sqlalchemy import sets import re +import random sys.path.append("../../") @@ -46,18 +47,7 @@ user_agent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.0.37 # Note these urls should be https due to caching considerations. # If you really must make them http, be sure to change exit_ports to [80] # below, or else the scan will not finish. -# TODO: As the network balances, these can become more uniform in size -# TODO: We'll also want to try to prefer pairing unmeasured nodes -# together then, and use a different url set for them. -# cutoff percent URL -urls = [(5, "https://38.229.70.2/8M"), # fbw 2000k..451k - (10, "https://38.229.70.2/4M"), # fbw 451k..275k - (20, "https://38.229.70.2/2M"), # fbw 275k..150k - (30, "https://38.229.70.2/1M"), # fbw 150k..100k - (40, "https://38.229.70.2/1M"), # fbw 100k..75k - (50, "https://38.229.70.2/512k"), # fbw 75k..50k - (80, "https://38.229.70.2/256k"), # fbw 50k..25k - (100, "https://38.229.70.2/128k")] # fbw 25k..10k +urls = ["https://38.229.70.2/"] #, "https://38.229.70.19/"] # Do NOT modify this object directly after it is handed to PathBuilder @@ -109,10 +99,25 @@ def read_config(filename): sleep_start,sleep_stop,min_streams,pid_file,db_url) def choose_url(percentile): - for (pct, url) in urls: + # TODO: Maybe we don't want to read the file *every* time? + # Maybe once per slice? + # Read in the bw auths file + f = file("./data/bwfiles", "r") + lines = [] + valid = False + for l in f.readlines(): + if l == ".\n": + valid = True + break + pair = l.split() + lines.append((int(pair[0]), pair[1])) + + if not valid: + plog("ERROR", "File size list is invalid!") + + for (pct, fname) in lines: if percentile < pct: - return url - #return "https://86.59.21.36/torbrowser/dist/tor-im-browser-1.2.0_ru_split/tor-im-br…" + return random.choice(urls) + fname raise PathSupport.NoNodesRemain("No nodes left for url choice!") def http_request(address):

1 0

[torflow/master] Bug #4431: Automatically determine size cutoffs.
by mikeperry＠torproject.org 17 Nov '11

17 Nov '11

commit c3a99b10e86cc1ef420a62498c6ade3837381ecf Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Sun Nov 6 19:01:30 2011 -0800 Bug #4431: Automatically determine size cutoffs. Also update the spec and the use of Wgd. --- NetworkScanners/BwAuthority/README.spec.txt | 12 ++++---- NetworkScanners/BwAuthority/aggregate.py | 41 +++++++++++++++++++++++++-- 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/NetworkScanners/BwAuthority/README.spec.txt b/NetworkScanners/BwAuthority/README.spec.txt index 377841c..96568c6 100644 --- a/NetworkScanners/BwAuthority/README.spec.txt +++ b/NetworkScanners/BwAuthority/README.spec.txt @@ -209,9 +209,9 @@ For each download, the bandwidth scanners process STREAM and STREAM_BW events with a StreamListener (in TorCtl/SQLSupport.py). The throughput for each stream is defined as the ratio of total read bytes over the time delta between - the STREAM NEW timestamp and the STREAM CLOSED event received timestamp: + the STREAM SUCCEEDED timestamp and the STREAM CLOSED event received timestamp: - bandwidth = (STREAM_BW bytes / (CLOSED timestamp - NEW timestamp) + bandwidth = (STREAM_BW bytes / (CLOSED timestamp - SUCCEEDED timestamp) We store both read and write bandwidths in the SQL tables, but only use the read bytes for results. @@ -440,11 +440,11 @@ Guard+Exit nodes are treated as normal nodes in terms of measurement frequency (measurements are reported as soon as their slice results - are ready), except they are given a K_p of 1.0-Wgd (Wgd is the consensus - bandwidth-weight for selecting Guard+Exits for the Guard position: See - dir-spec.txt Section 3.4.3). + are ready), except they are given a K_p of K_p*(1.0-Wgd) (Wgd is the + consensus bandwidth-weight for selecting Guard+Exits for the Guard + position: See dir-spec.txt Section 3.4.3). - K_p=1.0-Wgd isn't expected to be the optimal value, but we needed a + K_p*(1.0-Wgd) isn't expected to be the optimal value, but we needed a dampening factor to slow the feedback loop, and it seems as good of an initial guess as any. Note that convergence towards zero error should still happen eventually with this value, just at a slower rate. diff --git a/NetworkScanners/BwAuthority/aggregate.py b/NetworkScanners/BwAuthority/aggregate.py index 3b3dc14..f9c95cc 100755 --- a/NetworkScanners/BwAuthority/aggregate.py +++ b/NetworkScanners/BwAuthority/aggregate.py @@ -218,10 +218,42 @@ class ConsensusJunk: self.bw_weights["Wgd"] = 0 self.bw_weights["Wgg"] = 1.0 +def write_file_list(datadir): + files = {64*1024:"64M", 32*1024:"32M", 16*1024:"16M", 8*1024:"8M", + 4*1024:"4M", 2*1024:"2M", 1024:"1M", 512:"512k", + 256:"256k", 128:"128k", 64:"64k", 32:"32k", 16:"16k", 0:"16k"} + file_sizes = files.keys() + node_fbws = map(lambda x: 5*x.filt_bw, nodes.itervalues()) + file_pairs = [] + file_sizes.sort(reverse=True) + node_fbws.sort() + prev_size = file_sizes[-1] + i = 0 + + # The idea here is to grab the largest file size such + # that 5*bw < file, and do this for each file size. + for bw in node_fbws: + i += 1 + for f in xrange(len(file_sizes)): + if bw > file_sizes[f]*1024 and file_sizes[f] > prev_size: + next_f = max(f-1,0) + file_pairs.append((100-(100*i)/len(node_fbws),files[file_sizes[next_f]])) + prev_size = file_sizes[f] + break + + file_pairs.reverse() + + outfile = file(datadir+"/bwfiles.new", "w") + for f in file_pairs: + outfile.write(str(f[0])+" "+f[1]+"\n") + outfile.close() + # atomic on POSIX + os.rename(datadir+"/bwfiles.new", datadir+"/bwfiles") + def main(argv): TorUtil.read_config(argv[1]+"/scanner.1/bwauthority.cfg") TorUtil.loglevel = "NOTICE" - + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect((TorUtil.control_host,TorUtil.control_port)) c = TorCtl.Connection(s) @@ -399,8 +431,9 @@ def main(argv): # knows how to represent that and still KISS? if n.idhex in prev_consensus and \ ("Guard" in prev_consensus[n.idhex].flags \ - and "Exit" not in prev_consensus[n.idhex].flags): - n.new_bw = n.get_pid_bw(prev_votes.vote_map[n.idhex], 1.0-cs_junk.bw_weights["Wgd"]) + and "Exit" in prev_consensus[n.idhex].flags): + n.new_bw = n.get_pid_bw(prev_votes.vote_map[n.idhex], + K_p*(1.0-cs_junk.bw_weights["Wgd"])) else: n.new_bw = n.get_pid_bw(prev_votes.vote_map[n.idhex], K_p) else: @@ -493,6 +526,8 @@ def main(argv): out.write("node_id="+n.idhex+" bw="+str(base10_round(n.new_bw))+" nick="+n.nick+ " measured_at="+str(int(n.measured_at))+" pid_error="+str(n.pid_error)+" pid_error_sum="+str(n.pid_error_sum)+" pid_bw="+str(int(n.pid_bw))+" pid_delta="+str(n.derror_dt)+" circ_fail="+str(n.circ_fail_rate)+"\n") out.close() + write_file_list(argv[1]) + if __name__ == "__main__": try: main(sys.argv)

1 0

[torflow/master] Fix accidental reset of circ_fail_rate.
by mikeperry＠torproject.org 17 Nov '11

17 Nov '11

commit 249764351c8884975bf7702f4bce4e344dbd3074 Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Sat Nov 5 13:23:35 2011 -0700 Fix accidental reset of circ_fail_rate. --- NetworkScanners/BwAuthority/aggregate.py | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diff --git a/NetworkScanners/BwAuthority/aggregate.py b/NetworkScanners/BwAuthority/aggregate.py index 521aee6..3b3dc14 100755 --- a/NetworkScanners/BwAuthority/aggregate.py +++ b/NetworkScanners/BwAuthority/aggregate.py @@ -154,11 +154,14 @@ class Line: self.measured_at = timestamp try: self.circ_fail_rate = float(re.search("[\s]*circ_fail_rate=([\S]+)[\s]*", line).group(1)) - self.strm_fail_rate = float(re.search("[\s]*strm_fail_rate=([\S]+)[\s]*", line).group(1)) except: self.circ_fail_rate = 0 + try: + self.strm_fail_rate = float(re.search("[\s]*strm_fail_rate=([\S]+)[\s]*", line).group(1)) + except: self.strm_fail_rate = 0 + class Vote: def __init__(self, line): # node_id=$DB8C6D8E0D51A42BDDA81A9B8A735B41B2CF95D1 bw=231000 diff=209281 nick=rainbowwarrior measured_at=1319822504 @@ -187,7 +190,6 @@ class VoteSet: self.vote_map[vote.idhex] = vote except IOError: plog("NOTICE", "No previous vote data.") - pass # Misc items we need to get out of the consensus class ConsensusJunk:

1 0

[torflow/master] Prior to feedback, we used the greater of two ratios.
by mikeperry＠torproject.org 17 Nov '11

17 Nov '11

commit 12eef3b8afb61a0b84bd7709319a225c4deb5a9e Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Mon Nov 7 13:22:22 2011 -0800 Prior to feedback, we used the greater of two ratios. Let's keep this behavior in place when the bwauthpid consensus param is missing. --- NetworkScanners/BwAuthority/aggregate.py | 12 +++++++++++- 1 files changed, 11 insertions(+), 1 deletions(-) diff --git a/NetworkScanners/BwAuthority/aggregate.py b/NetworkScanners/BwAuthority/aggregate.py index f9c95cc..be9c595 100755 --- a/NetworkScanners/BwAuthority/aggregate.py +++ b/NetworkScanners/BwAuthority/aggregate.py @@ -246,6 +246,7 @@ def write_file_list(datadir): outfile = file(datadir+"/bwfiles.new", "w") for f in file_pairs: outfile.write(str(f[0])+" "+f[1]+"\n") + outfile.write(".\n") outfile.close() # atomic on POSIX os.rename(datadir+"/bwfiles.new", datadir+"/bwfiles") @@ -362,6 +363,8 @@ def main(argv): else: true_filt_avg = sum(map(lambda n: n.filt_bw, nodes.itervalues()))/float(len(nodes)) + true_strm_avg = sum(map(lambda n: n.strm_bw, + nodes.itervalues()))/float(len(nodes)) plog("DEBUG", "Network true_filt_avg: "+str(true_filt_avg)) @@ -394,7 +397,7 @@ def main(argv): for n in nodes.itervalues(): n.fbw_ratio = n.filt_bw/true_filt_avg - # Always use filtered bandwidths + # Always use filtered bandwidths for feedback n.ratio = n.fbw_ratio if cs_junk.bwauth_pid_control: @@ -445,6 +448,13 @@ def main(argv): n.pid_bw = n.new_bw plog("INFO", "No prev vote for node "+n.nick+": Consensus feedback") else: # No PID feedback + # Choose the larger between sbw and fbw + n.sbw_ratio = n.strm_bw/true_strm_avg + if n.sbw_ratio > n.fbw_ratio: + n.ratio = n.sbw_ratio + else: + n.ratio = n.fbw_ratio + n.pid_bw = 0 n.pid_error = 0 n.pid_error_sum = 0

1 0

[torflow/master] Bug #4273: Reduce max fetch time
by mikeperry＠torproject.org 17 Nov '11

17 Nov '11

commit dbeb72f5feff6e2b969396dc73e1f783090a496f Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Wed Nov 9 13:54:15 2011 -0800 Bug #4273: Reduce max fetch time 5 minutes seems better. Now that we automatically select slice file sizes to complete in about 5 seconds, it should be more than enough. --- .../BwAuthority/data/scanner.1/bwauthority.cfg | 2 +- .../BwAuthority/data/scanner.2/bwauthority.cfg | 2 +- .../BwAuthority/data/scanner.3/bwauthority.cfg | 2 +- .../BwAuthority/data/scanner.4/bwauthority.cfg | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg index 8f9394f..abd8e1e 100644 --- a/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg @@ -23,6 +23,6 @@ save_every = 0 nodes_per_slice = 50 circs_per_node = 5 min_streams = 1 -max_fetch_time = 1800 +max_fetch_time = 300 sleep_start = 01:30 sleep_stop = 04:30 diff --git a/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg index 04c7a9f..9b7b591 100644 --- a/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg @@ -23,6 +23,6 @@ save_every = 0 nodes_per_slice = 50 circs_per_node = 5 min_streams = 1 -max_fetch_time = 1800 +max_fetch_time = 300 sleep_start = 01:30 sleep_stop = 04:30 diff --git a/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg index a662c8c..51dfdc5 100644 --- a/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg @@ -23,6 +23,6 @@ save_every = 0 nodes_per_slice = 50 circs_per_node = 5 min_streams = 1 -max_fetch_time = 1800 +max_fetch_time = 300 sleep_start = 01:30 sleep_stop = 04:30 diff --git a/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg index fa3f50f..94342a2 100644 --- a/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg +++ b/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg @@ -23,6 +23,6 @@ save_every = 0 nodes_per_slice = 50 circs_per_node = 5 min_streams = 1 -max_fetch_time = 1800 +max_fetch_time = 300 sleep_start = 01:30 sleep_stop = 04:30

1 0

[torflow/master] Tweaks to try to make PID look more like other bw auths.
by mikeperry＠torproject.org 17 Nov '11

17 Nov '11

commit 335ce85c130c5ffef8525afb56f77ad423e12f9b Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Thu Nov 10 14:25:11 2011 -0800 Tweaks to try to make PID look more like other bw auths. Just to make sure something crazy is not going on, let's try eliminating the circ failure dampening, and re-instate the "larger ratio" idea for PID control. With these both re-enabled, PID "feedback" should look more similar to the other bandwidth auths, because with just one auth running the code, there is no feedback. --- NetworkScanners/BwAuthority/aggregate.py | 18 ++++++++++++------ 1 files changed, 12 insertions(+), 6 deletions(-) diff --git a/NetworkScanners/BwAuthority/aggregate.py b/NetworkScanners/BwAuthority/aggregate.py index be9c595..d1c8b45 100755 --- a/NetworkScanners/BwAuthority/aggregate.py +++ b/NetworkScanners/BwAuthority/aggregate.py @@ -139,7 +139,8 @@ class Node: self.filt_bw = line.filt_bw self.ns_bw = line.ns_bw self.desc_bw = line.desc_bw - self.circ_fail_rate = line.circ_fail_rate + # XXX: Temporary test + self.circ_fail_rate = 0.0 #line.circ_fail_rate self.strm_fail_rate = line.strm_fail_rate class Line: @@ -360,6 +361,8 @@ def main(argv): # socket exhaustion issues.. true_filt_avg = sum(map(lambda n: n.filt_bw*(1.0-n.circ_fail_rate), nodes.itervalues()))/float(len(nodes)) + true_strm_avg = sum(map(lambda n: n.strm_bw*(1.0-n.circ_fail_rate), + nodes.itervalues()))/float(len(nodes)) else: true_filt_avg = sum(map(lambda n: n.filt_bw, nodes.itervalues()))/float(len(nodes)) @@ -396,13 +399,17 @@ def main(argv): tot_net_bw = 0 for n in nodes.itervalues(): n.fbw_ratio = n.filt_bw/true_filt_avg - - # Always use filtered bandwidths for feedback - n.ratio = n.fbw_ratio + n.sbw_ratio = n.strm_bw/true_strm_avg if cs_junk.bwauth_pid_control: # Penalize nodes for circ failure rate - n.pid_error = (n.filt_bw*(1.0-n.circ_fail_rate) - true_filt_avg)/true_filt_avg + # n.pid_error = (n.filt_bw*(1.0-n.circ_fail_rate) - true_filt_avg)/true_filt_avg + + # FIXME: For now, let's try this larger-ratio thing.. + if n.sbw_ratio > n.fbw_ratio: + n.pid_error = (n.strm_bw*(1.0-n.circ_fail_rate) - true_strm_avg)/true_strm_avg + else: + n.pid_error = (n.filt_bw*(1.0-n.circ_fail_rate) - true_filt_avg)/true_filt_avg if n.idhex in prev_votes.vote_map: # If there is a new sample, let's use it for all but guards @@ -449,7 +456,6 @@ def main(argv): plog("INFO", "No prev vote for node "+n.nick+": Consensus feedback") else: # No PID feedback # Choose the larger between sbw and fbw - n.sbw_ratio = n.strm_bw/true_strm_avg if n.sbw_ratio > n.fbw_ratio: n.ratio = n.sbw_ratio else:

1 0

[torflow/master] Fix for 0-bandwidth Guard issue.
by mikeperry＠torproject.org 17 Nov '11

17 Nov '11

commit e6ddd7a2572bac9be7a29c3fe725dbed7628c770 Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Sat Nov 12 15:56:54 2011 -0800 Fix for 0-bandwidth Guard issue. We were accidentally voting 0 for guards that weren't measured this round. --- NetworkScanners/BwAuthority/aggregate.py | 18 ++++++++++++------ 1 files changed, 12 insertions(+), 6 deletions(-) diff --git a/NetworkScanners/BwAuthority/aggregate.py b/NetworkScanners/BwAuthority/aggregate.py index 53f8dd7..1746256 100755 --- a/NetworkScanners/BwAuthority/aggregate.py +++ b/NetworkScanners/BwAuthority/aggregate.py @@ -95,7 +95,7 @@ class Node: self.strm_fail_rate = 0 def revert_to_vote(self, vote): - self.new_bw = vote.bw*1000 # XXX: Could be 0? + self.new_bw = vote.bw*1000 self.pid_bw = vote.pid_bw self.pid_error = vote.pid_error self.measured_at = vote.measured_at @@ -176,7 +176,7 @@ class Vote: self.pid_bw = float(re.search("[\s]*pid_bw=([\S]+)[\s]*", line).group(1)) except: plog("NOTICE", "No previous PID data.") - self.pid_bw = 0 + self.pid_bw = self.bw self.pid_error = 0 self.pid_error_sum = 0 @@ -432,8 +432,15 @@ def main(argv): n.revert_to_vote(prev_votes.vote_map[n.idhex]) # Don't use feedback here, but we might as well use our # new measurement against the previous vote. - n.new_bw = prev_votes.vote_map[n.idhex].pid_bw + \ - K_p*prev_votes.vote_map[n.idhex].pid_bw*pid_error + if prev_votes.vote_map[n.idhex].pid_bw == 0: + # This should no longer happen + plog("NOTICE", "Zero bw for Guard node "+n.nick+"="+n.idhex) + n.new_bw = prev_votes.vote_map[n.idhex].bw + \ + K_p*prev_votes.vote_map[n.idhex].bw*pid_error + n.pid_bw = n.new_bw + else: + n.new_bw = prev_votes.vote_map[n.idhex].pid_bw + \ + K_p*prev_votes.vote_map[n.idhex].pid_bw*pid_error else: # Everyone else should be pretty instantenous to respond. # Full feedback should be fine for them (we hope), @@ -451,7 +458,6 @@ def main(argv): n.new_bw = n.get_pid_bw(prev_votes.vote_map[n.idhex], K_p) else: # Reset values. Don't vote/sample this measurement round. - # XXX: This possibly breaks guards n.revert_to_vote(prev_votes.vote_map[n.idhex]) else: # No prev vote, pure consensus feedback this round n.new_bw = n.ns_bw + K_p*n.ns_bw*n.pid_error @@ -465,10 +471,10 @@ def main(argv): else: n.ratio = n.fbw_ratio - n.pid_bw = 0 n.pid_error = 0 n.pid_error_sum = 0 n.new_bw = n.desc_bw*n.ratio + n.pid_bw = n.new_bw # for transition between pid/no-pid n.change = n.new_bw - n.desc_bw

1 0

[torflow/master] Tweak a regex and add some log lines.
by mikeperry＠torproject.org 17 Nov '11

17 Nov '11

commit e37aedf5c87296f05023f74e6ce7360bc56bb147 Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Sat Nov 12 13:13:13 2011 -0800 Tweak a regex and add some log lines. --- NetworkScanners/BwAuthority/aggregate.py | 19 +++++++++++++------ 1 files changed, 13 insertions(+), 6 deletions(-) diff --git a/NetworkScanners/BwAuthority/aggregate.py b/NetworkScanners/BwAuthority/aggregate.py index d1c8b45..53f8dd7 100755 --- a/NetworkScanners/BwAuthority/aggregate.py +++ b/NetworkScanners/BwAuthority/aggregate.py @@ -95,7 +95,7 @@ class Node: self.strm_fail_rate = 0 def revert_to_vote(self, vote): - self.new_bw = vote.bw*1000 + self.new_bw = vote.bw*1000 # XXX: Could be 0? self.pid_bw = vote.pid_bw self.pid_error = vote.pid_error self.measured_at = vote.measured_at @@ -168,7 +168,7 @@ class Vote: # node_id=$DB8C6D8E0D51A42BDDA81A9B8A735B41B2CF95D1 bw=231000 diff=209281 nick=rainbowwarrior measured_at=1319822504 self.idhex = re.search("[\s]*node_id=([\S]+)[\s]*", line).group(1) self.nick = re.search("[\s]*nick=([\S]+)[\s]*", line).group(1) - self.bw = int(re.search("[\s]*bw=([\S]+)[\s]*", line).group(1)) + self.bw = int(re.search("[\s]+bw=([\S]+)[\s]*", line).group(1)) self.measured_at = int(re.search("[\s]*measured_at=([\S]+)[\s]*", line).group(1)) try: self.pid_error = float(re.search("[\s]*pid_error=([\S]+)[\s]*", line).group(1)) @@ -359,11 +359,13 @@ def main(argv): # Penalize nodes for circuit failure: it indicates CPU pressure # TODO: Potentially penalize for stream failure, if we run into # socket exhaustion issues.. + plog("INFO", "PID control enabled") true_filt_avg = sum(map(lambda n: n.filt_bw*(1.0-n.circ_fail_rate), nodes.itervalues()))/float(len(nodes)) true_strm_avg = sum(map(lambda n: n.strm_bw*(1.0-n.circ_fail_rate), nodes.itervalues()))/float(len(nodes)) else: + plog("INFO", "PID control disabled") true_filt_avg = sum(map(lambda n: n.filt_bw, nodes.itervalues()))/float(len(nodes)) true_strm_avg = sum(map(lambda n: n.strm_bw, @@ -381,7 +383,8 @@ def main(argv): node_measure_time = 0 for n in nodes.itervalues(): if n.idhex in prev_votes.vote_map and n.idhex in prev_consensus: - if "Guard" in prev_consensus[n.idhex].flags: + if "Guard" in prev_consensus[n.idhex].flags and \ + "Exit" not in prev_consensus[n.idhex].flags: guard_cnt += 1 guard_measure_time += (n.measured_at - \ prev_votes.vote_map[n.idhex].measured_at) @@ -448,6 +451,7 @@ def main(argv): n.new_bw = n.get_pid_bw(prev_votes.vote_map[n.idhex], K_p) else: # Reset values. Don't vote/sample this measurement round. + # XXX: This possibly breaks guards n.revert_to_vote(prev_votes.vote_map[n.idhex]) else: # No prev vote, pure consensus feedback this round n.new_bw = n.ns_bw + K_p*n.ns_bw*n.pid_error @@ -492,9 +496,12 @@ def main(argv): +str(n.new_bw)+"->"+str(int(tot_net_bw*NODE_CAP))+")") n.new_bw = int(tot_net_bw*NODE_CAP) n.pid_error_sum = 0 # Don't let unused error accumulate... - if n.new_bw < 0: - plog("INFO", "Node "+n.idhex+"="+n.nick+" has bandwidth < 0: "+str(n.new_bw)) - n.new_bw = 0 + if n.new_bw <= 0: + if n.idhex in prev_consensus: + plog("INFO", str(prev_consensus[n.idhex].flags)+" node "+n.idhex+"="+n.nick+" has bandwidth <= 0: "+str(n.new_bw)) + else: + plog("INFO", "New node "+n.idhex+"="+n.nick+" has bandwidth < 0: "+str(n.new_bw)) + n.new_bw = 1 # WTF is going on here? oldest_timestamp = min(map(lambda n: n.measured_at,

1 0