- tor-commits - lists.torproject.org

[torflow/master] Fix a few feedback bugs.
by mikeperry＠torproject.org 17 Nov '11

17 Nov '11

commit 7abd885aa8b0a07fc62c3be8f847b21e823af1cd Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Mon Oct 31 22:54:15 2011 -0700 Fix a few feedback bugs. --- NetworkScanners/BwAuthority/aggregate.py | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diff --git a/NetworkScanners/BwAuthority/aggregate.py b/NetworkScanners/BwAuthority/aggregate.py index e168f2e..87a13ef 100755 --- a/NetworkScanners/BwAuthority/aggregate.py +++ b/NetworkScanners/BwAuthority/aggregate.py @@ -96,7 +96,7 @@ class Node: self.strm_fail_rate = 0 def revert_to_vote(self, vote): - self.new_bw = vote.bw + self.new_bw = vote.bw*1000 self.pid_error = vote.pid_error self.measured_at = vote.measured_at @@ -473,7 +473,7 @@ def main(argv): plog("INFO", "Measured "+str(measured_pct)+"% of all tor nodes.") n_print = nodes.values() - n_print.sort(lambda x,y: int(y.change) - int(x.change)) + n_print.sort(lambda x,y: int(y.pid_error*1000) - int(x.pid_error*1000)) for scanner in scanner_timestamps.iterkeys(): scan_age = int(round(scanner_timestamps[scanner],0)) @@ -486,7 +486,7 @@ def main(argv): # FIXME: Split out debugging data for n in n_print: if not n.ignore: - out.write("node_id="+n.idhex+" bw="+str(base10_round(n.new_bw))+" nick="+n.nick+ " measured_at="+str(int(n.measured_at))+" pid_error="+str(n.pid_error)+" pid_error_sum="+str(n.pid_error_sum)+" pid_bw="+str(n.pid_bw)+" pid_delta="+str(n.derror_dt)+"\n") + out.write("node_id="+n.idhex+" bw="+str(base10_round(n.new_bw))+" nick="+n.nick+ " measured_at="+str(int(n.measured_at))+" pid_error="+str(n.pid_error)+" pid_error_sum="+str(n.pid_error_sum)+" pid_bw="+str(int(n.pid_bw))+" pid_delta="+str(n.derror_dt)+"\n") out.close() if __name__ == "__main__":

1 0

[torflow/master] Properly store roger's unfeedback var.
by mikeperry＠torproject.org 17 Nov '11

17 Nov '11

commit 8c47bb02c77d8e5e8f4955fe2d25902be5cfd8cd Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Mon Oct 31 23:00:30 2011 -0700 Properly store roger's unfeedback var. --- NetworkScanners/BwAuthority/aggregate.py | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/NetworkScanners/BwAuthority/aggregate.py b/NetworkScanners/BwAuthority/aggregate.py index 87a13ef..e963413 100755 --- a/NetworkScanners/BwAuthority/aggregate.py +++ b/NetworkScanners/BwAuthority/aggregate.py @@ -97,6 +97,7 @@ class Node: def revert_to_vote(self, vote): self.new_bw = vote.bw*1000 + self.pid_bw = vote.pid_bw self.pid_error = vote.pid_error self.measured_at = vote.measured_at

1 0

[torflow/master] Update spec with circ failure info.
by mikeperry＠torproject.org 17 Nov '11

17 Nov '11

commit 24bf42c5938bf996c878ddd1ad25c442e6128e69 Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Mon Oct 31 23:48:02 2011 -0700 Update spec with circ failure info. --- NetworkScanners/BwAuthority/README.spec.txt | 12 +++++++++--- 1 files changed, 9 insertions(+), 3 deletions(-) diff --git a/NetworkScanners/BwAuthority/README.spec.txt b/NetworkScanners/BwAuthority/README.spec.txt index a4e793e..377841c 100644 --- a/NetworkScanners/BwAuthority/README.spec.txt +++ b/NetworkScanners/BwAuthority/README.spec.txt @@ -369,10 +369,16 @@ 3.1. Modeling Measurement as PID Control The bandwidth authorities measure F_node: the filtered stream - capacity through a given node (filtering is described in Section 1.6). + capacity through a given node (filtering is described in Section 1.6) + times the circuit success rate of circuit EXTENDs to the node + (1.0 - circ_fail_rate). - The PID Setpoint, or target for each node is F_avg: the average filtered - stream capacity observed in the network + In PID control, we add in this extra failure rate as a damper, to prevent + the PID control system from driving nodes to CPU overload. Once nodes + begin failing circuits, we want to stop devoting more capacity. + + The PID Setpoint, or target for each node is F_avg: the average F_node + value observed across the entire network. The normalized PID error e(t) for each node is then:

1 0

[torflow/master] Handle negative bandwidths.
by mikeperry＠torproject.org 17 Nov '11

17 Nov '11

commit b923a0a5c6f14b165bb4de7abedf46d58df46939 Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Fri Nov 4 14:57:08 2011 -0700 Handle negative bandwidths. Also record the circ fail rate. --- NetworkScanners/BwAuthority/aggregate.py | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diff --git a/NetworkScanners/BwAuthority/aggregate.py b/NetworkScanners/BwAuthority/aggregate.py index a30d436..39a2e85 100755 --- a/NetworkScanners/BwAuthority/aggregate.py +++ b/NetworkScanners/BwAuthority/aggregate.py @@ -441,6 +441,9 @@ def main(argv): +str(n.new_bw)+"->"+str(int(tot_net_bw*NODE_CAP))+")") n.new_bw = int(tot_net_bw*NODE_CAP) n.pid_error_sum = 0 # Don't let unused error accumulate... + if n.new_bw < 0: + plog("INFO", "Node "+n.idhex+"="+n.nick+" has bandwidth < 0: "+str(n.new_bw)) + n.new_bw = 0 # WTF is going on here? oldest_timestamp = min(map(lambda n: n.measured_at, @@ -485,7 +488,7 @@ def main(argv): # FIXME: Split out debugging data for n in n_print: if not n.ignore: - out.write("node_id="+n.idhex+" bw="+str(base10_round(n.new_bw))+" nick="+n.nick+ " measured_at="+str(int(n.measured_at))+" pid_error="+str(n.pid_error)+" pid_error_sum="+str(n.pid_error_sum)+" pid_bw="+str(int(n.pid_bw))+" pid_delta="+str(n.derror_dt)+"\n") + out.write("node_id="+n.idhex+" bw="+str(base10_round(n.new_bw))+" nick="+n.nick+ " measured_at="+str(int(n.measured_at))+" pid_error="+str(n.pid_error)+" pid_error_sum="+str(n.pid_error_sum)+" pid_bw="+str(int(n.pid_bw))+" pid_delta="+str(n.derror_dt)+" circ_fail="+str(n.circ_fail_rate)+"\n") out.close() if __name__ == "__main__":

1 0

[torflow/master] Update cron script to differentiate output vs archive.
by mikeperry＠torproject.org 17 Nov '11

17 Nov '11

commit f964c66e4eeae5c6d21c540d9439573a0cb0f968 Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Fri Nov 4 15:05:19 2011 -0700 Update cron script to differentiate output vs archive. --- NetworkScanners/BwAuthority/cron.sh | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-) diff --git a/NetworkScanners/BwAuthority/cron.sh b/NetworkScanners/BwAuthority/cron.sh index c3cd5f6..e424a8f 100755 --- a/NetworkScanners/BwAuthority/cron.sh +++ b/NetworkScanners/BwAuthority/cron.sh @@ -3,14 +3,14 @@ SCANNER_DIR=~/code/tor/torflow/NetworkScanners/BwAuthority TIMESTAMP=`date +%Y%m%d-%H%M` -TOR_DEST=$SCANNER_DIR/bwscan.V3BandwidthsFile -OUTPUT=$SCANNER_DIR/data/bwscan.${TIMESTAMP} +ARCHIVE=$SCANNER_DIR/data/bwscan.${TIMESTAMP} +OUTPUT=$SCANNER_DIR/bwscan.V3BandwidthsFile cd $SCANNER_DIR # Needed for import to work properly. $SCANNER_DIR/aggregate.py $SCANNER_DIR/data $OUTPUT if [ $? = 0 ] then - cp $OUTPUT $TOR_DEST - #scp $TOR_DEST bwscan@torauthority.org:/var/lib/tor.scans/bwscan > /dev/null + cp $OUTPUT $ARCHIVE + #scp $OUTPUT bwscan@torauthority.org:/var/lib/tor.scans/bwscan > /dev/null fi

1 0

[torflow/master] Actually remove stale files.
by mikeperry＠torproject.org 17 Nov '11

17 Nov '11

commit 7bf82544ff04c1755667d12e7c2f966a12c9228d Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Fri Nov 4 15:13:05 2011 -0700 Actually remove stale files. Also kill some dead code. --- NetworkScanners/BwAuthority/aggregate.py | 14 +++++++------- 1 files changed, 7 insertions(+), 7 deletions(-) diff --git a/NetworkScanners/BwAuthority/aggregate.py b/NetworkScanners/BwAuthority/aggregate.py index 39a2e85..521aee6 100755 --- a/NetworkScanners/BwAuthority/aggregate.py +++ b/NetworkScanners/BwAuthority/aggregate.py @@ -13,7 +13,6 @@ from TorCtl import TorCtl,TorUtil from TorCtl.PathSupport import VersionRangeRestriction, NodeRestrictionList, NotNodeRestriction bw_files = [] -timestamps = {} nodes = {} prev_consensus = {} @@ -280,16 +279,17 @@ def main(argv): # measure hibernating routers for days. # This filter is just to remove REALLY old files if time.time() - timestamp > MAX_AGE: - plog("DEBUG", "Skipping old file "+f) - # FIXME: Unlink this file + sql- + sqlf = f.replace("bws-", "sql-") + plog("INFO", "Removing old file "+f+" and "+sqlf) + os.remove(sr+"/"+f) + try: + os.remove(sr+"/"+sqlf) + except: + pass # In some cases the sql file may not exist continue if timestamp > newest_timestamp: newest_timestamp = timestamp bw_files.append((slicenum, timestamp, sr+"/"+f)) - # FIXME: Can we kill this? - if slicenum not in timestamps or \ - timestamps[slicenum] < timestamp: - timestamps[slicenum] = timestamp scanner_timestamps[ds] = newest_timestamp # Need to only use most recent slice-file for each node..

1 0

[torflow/master] Submodule.
by mikeperry＠torproject.org 17 Nov '11

17 Nov '11

commit ba4efc2ded84e0867d3f0ea2c81badd8334f2f42 Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Fri Nov 4 15:42:47 2011 -0700 Submodule. --- TorCtl | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/TorCtl b/TorCtl index 66ed91b..391b439 160000 --- a/TorCtl +++ b/TorCtl @@ -1 +1 @@ -Subproject commit 66ed91b5c1453b9c25b7070b702e2e548efb4f80 +Subproject commit 391b4399adda91e8b68ffd6e1fa294efe846debd

1 0

[torflow/master] Bugs 4431+2014: Randomized url support + variable file sizes.
by mikeperry＠torproject.org 17 Nov '11

17 Nov '11

commit 5fdb018fe760def96eca098bee723dea4b5e9049 Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Mon Nov 7 21:16:07 2011 -0800 Bugs 4431+2014: Randomized url support + variable file sizes. Scanner child piece. --- NetworkScanners/BwAuthority/bwauthority_child.py | 35 ++++++++++++--------- 1 files changed, 20 insertions(+), 15 deletions(-) diff --git a/NetworkScanners/BwAuthority/bwauthority_child.py b/NetworkScanners/BwAuthority/bwauthority_child.py index 27dc0ba..1527a11 100755 --- a/NetworkScanners/BwAuthority/bwauthority_child.py +++ b/NetworkScanners/BwAuthority/bwauthority_child.py @@ -23,6 +23,7 @@ import ConfigParser import sqlalchemy import sets import re +import random sys.path.append("../../") @@ -46,18 +47,7 @@ user_agent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.0.37 # Note these urls should be https due to caching considerations. # If you really must make them http, be sure to change exit_ports to [80] # below, or else the scan will not finish. -# TODO: As the network balances, these can become more uniform in size -# TODO: We'll also want to try to prefer pairing unmeasured nodes -# together then, and use a different url set for them. -# cutoff percent URL -urls = [(5, "https://38.229.70.2/8M"), # fbw 2000k..451k - (10, "https://38.229.70.2/4M"), # fbw 451k..275k - (20, "https://38.229.70.2/2M"), # fbw 275k..150k - (30, "https://38.229.70.2/1M"), # fbw 150k..100k - (40, "https://38.229.70.2/1M"), # fbw 100k..75k - (50, "https://38.229.70.2/512k"), # fbw 75k..50k - (80, "https://38.229.70.2/256k"), # fbw 50k..25k - (100, "https://38.229.70.2/128k")] # fbw 25k..10k +urls = ["https://38.229.70.2/"] #, "https://38.229.70.19/"] # Do NOT modify this object directly after it is handed to PathBuilder @@ -109,10 +99,25 @@ def read_config(filename): sleep_start,sleep_stop,min_streams,pid_file,db_url) def choose_url(percentile): - for (pct, url) in urls: + # TODO: Maybe we don't want to read the file *every* time? + # Maybe once per slice? + # Read in the bw auths file + f = file("./data/bwfiles", "r") + lines = [] + valid = False + for l in f.readlines(): + if l == ".\n": + valid = True + break + pair = l.split() + lines.append((int(pair[0]), pair[1])) + + if not valid: + plog("ERROR", "File size list is invalid!") + + for (pct, fname) in lines: if percentile < pct: - return url - #return "https://86.59.21.36/torbrowser/dist/tor-im-browser-1.2.0_ru_split/tor-im-br…" + return random.choice(urls) + fname raise PathSupport.NoNodesRemain("No nodes left for url choice!") def http_request(address):

1 0

[torflow/master] Bug #4431: Automatically determine size cutoffs.
by mikeperry＠torproject.org 17 Nov '11

17 Nov '11

commit c3a99b10e86cc1ef420a62498c6ade3837381ecf Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Sun Nov 6 19:01:30 2011 -0800 Bug #4431: Automatically determine size cutoffs. Also update the spec and the use of Wgd. --- NetworkScanners/BwAuthority/README.spec.txt | 12 ++++---- NetworkScanners/BwAuthority/aggregate.py | 41 +++++++++++++++++++++++++-- 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/NetworkScanners/BwAuthority/README.spec.txt b/NetworkScanners/BwAuthority/README.spec.txt index 377841c..96568c6 100644 --- a/NetworkScanners/BwAuthority/README.spec.txt +++ b/NetworkScanners/BwAuthority/README.spec.txt @@ -209,9 +209,9 @@ For each download, the bandwidth scanners process STREAM and STREAM_BW events with a StreamListener (in TorCtl/SQLSupport.py). The throughput for each stream is defined as the ratio of total read bytes over the time delta between - the STREAM NEW timestamp and the STREAM CLOSED event received timestamp: + the STREAM SUCCEEDED timestamp and the STREAM CLOSED event received timestamp: - bandwidth = (STREAM_BW bytes / (CLOSED timestamp - NEW timestamp) + bandwidth = (STREAM_BW bytes / (CLOSED timestamp - SUCCEEDED timestamp) We store both read and write bandwidths in the SQL tables, but only use the read bytes for results. @@ -440,11 +440,11 @@ Guard+Exit nodes are treated as normal nodes in terms of measurement frequency (measurements are reported as soon as their slice results - are ready), except they are given a K_p of 1.0-Wgd (Wgd is the consensus - bandwidth-weight for selecting Guard+Exits for the Guard position: See - dir-spec.txt Section 3.4.3). + are ready), except they are given a K_p of K_p*(1.0-Wgd) (Wgd is the + consensus bandwidth-weight for selecting Guard+Exits for the Guard + position: See dir-spec.txt Section 3.4.3). - K_p=1.0-Wgd isn't expected to be the optimal value, but we needed a + K_p*(1.0-Wgd) isn't expected to be the optimal value, but we needed a dampening factor to slow the feedback loop, and it seems as good of an initial guess as any. Note that convergence towards zero error should still happen eventually with this value, just at a slower rate. diff --git a/NetworkScanners/BwAuthority/aggregate.py b/NetworkScanners/BwAuthority/aggregate.py index 3b3dc14..f9c95cc 100755 --- a/NetworkScanners/BwAuthority/aggregate.py +++ b/NetworkScanners/BwAuthority/aggregate.py @@ -218,10 +218,42 @@ class ConsensusJunk: self.bw_weights["Wgd"] = 0 self.bw_weights["Wgg"] = 1.0 +def write_file_list(datadir): + files = {64*1024:"64M", 32*1024:"32M", 16*1024:"16M", 8*1024:"8M", + 4*1024:"4M", 2*1024:"2M", 1024:"1M", 512:"512k", + 256:"256k", 128:"128k", 64:"64k", 32:"32k", 16:"16k", 0:"16k"} + file_sizes = files.keys() + node_fbws = map(lambda x: 5*x.filt_bw, nodes.itervalues()) + file_pairs = [] + file_sizes.sort(reverse=True) + node_fbws.sort() + prev_size = file_sizes[-1] + i = 0 + + # The idea here is to grab the largest file size such + # that 5*bw < file, and do this for each file size. + for bw in node_fbws: + i += 1 + for f in xrange(len(file_sizes)): + if bw > file_sizes[f]*1024 and file_sizes[f] > prev_size: + next_f = max(f-1,0) + file_pairs.append((100-(100*i)/len(node_fbws),files[file_sizes[next_f]])) + prev_size = file_sizes[f] + break + + file_pairs.reverse() + + outfile = file(datadir+"/bwfiles.new", "w") + for f in file_pairs: + outfile.write(str(f[0])+" "+f[1]+"\n") + outfile.close() + # atomic on POSIX + os.rename(datadir+"/bwfiles.new", datadir+"/bwfiles") + def main(argv): TorUtil.read_config(argv[1]+"/scanner.1/bwauthority.cfg") TorUtil.loglevel = "NOTICE" - + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect((TorUtil.control_host,TorUtil.control_port)) c = TorCtl.Connection(s) @@ -399,8 +431,9 @@ def main(argv): # knows how to represent that and still KISS? if n.idhex in prev_consensus and \ ("Guard" in prev_consensus[n.idhex].flags \ - and "Exit" not in prev_consensus[n.idhex].flags): - n.new_bw = n.get_pid_bw(prev_votes.vote_map[n.idhex], 1.0-cs_junk.bw_weights["Wgd"]) + and "Exit" in prev_consensus[n.idhex].flags): + n.new_bw = n.get_pid_bw(prev_votes.vote_map[n.idhex], + K_p*(1.0-cs_junk.bw_weights["Wgd"])) else: n.new_bw = n.get_pid_bw(prev_votes.vote_map[n.idhex], K_p) else: @@ -493,6 +526,8 @@ def main(argv): out.write("node_id="+n.idhex+" bw="+str(base10_round(n.new_bw))+" nick="+n.nick+ " measured_at="+str(int(n.measured_at))+" pid_error="+str(n.pid_error)+" pid_error_sum="+str(n.pid_error_sum)+" pid_bw="+str(int(n.pid_bw))+" pid_delta="+str(n.derror_dt)+" circ_fail="+str(n.circ_fail_rate)+"\n") out.close() + write_file_list(argv[1]) + if __name__ == "__main__": try: main(sys.argv)

1 0

[torflow/master] Fix accidental reset of circ_fail_rate.
by mikeperry＠torproject.org 17 Nov '11

17 Nov '11

commit 249764351c8884975bf7702f4bce4e344dbd3074 Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Sat Nov 5 13:23:35 2011 -0700 Fix accidental reset of circ_fail_rate. --- NetworkScanners/BwAuthority/aggregate.py | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diff --git a/NetworkScanners/BwAuthority/aggregate.py b/NetworkScanners/BwAuthority/aggregate.py index 521aee6..3b3dc14 100755 --- a/NetworkScanners/BwAuthority/aggregate.py +++ b/NetworkScanners/BwAuthority/aggregate.py @@ -154,11 +154,14 @@ class Line: self.measured_at = timestamp try: self.circ_fail_rate = float(re.search("[\s]*circ_fail_rate=([\S]+)[\s]*", line).group(1)) - self.strm_fail_rate = float(re.search("[\s]*strm_fail_rate=([\S]+)[\s]*", line).group(1)) except: self.circ_fail_rate = 0 + try: + self.strm_fail_rate = float(re.search("[\s]*strm_fail_rate=([\S]+)[\s]*", line).group(1)) + except: self.strm_fail_rate = 0 + class Vote: def __init__(self, line): # node_id=$DB8C6D8E0D51A42BDDA81A9B8A735B41B2CF95D1 bw=231000 diff=209281 nick=rainbowwarrior measured_at=1319822504 @@ -187,7 +190,6 @@ class VoteSet: self.vote_map[vote.idhex] = vote except IOError: plog("NOTICE", "No previous vote data.") - pass # Misc items we need to get out of the consensus class ConsensusJunk:

1 0