- tor-commits - lists.torproject.org

[tor/master] Don't segfault when checking whether a not-yet-used intro point should expire
by nickm＠torproject.org 02 Dec '11

02 Dec '11

commit 5ffa7102c00385445d3855bbc481668edf62d139 Author: Robert Ransom <rransom.8774(a)gmail.com> Date: Thu Dec 1 15:26:45 2011 -0800 Don't segfault when checking whether a not-yet-used intro point should expire Found by katmagic. Bugfix on the #3460 branch, not yet in any release. --- src/or/rendservice.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diff --git a/src/or/rendservice.c b/src/or/rendservice.c index a360d5c..177f3bf 100644 --- a/src/or/rendservice.c +++ b/src/or/rendservice.c @@ -1957,7 +1957,8 @@ intro_point_should_expire_now(rend_intro_point_t *intro, return 1; } - if (digestmap_size(intro->accepted_intro_rsa_parts) >= + if (intro->accepted_intro_rsa_parts != NULL && + digestmap_size(intro->accepted_intro_rsa_parts) >= INTRO_POINT_LIFETIME_INTRODUCTIONS) { /* This intro point has been used too many times. Expire it now. */ return 1;

1 0

[torflow/master] Fix variable shadow, and add some logs.
by mikeperry＠torproject.org 02 Dec '11

02 Dec '11

commit c183c9fe3970c9bd022d1d7816860dab62f34d07 Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Thu Dec 1 21:03:06 2011 -0800 Fix variable shadow, and add some logs. --- NetworkScanners/BwAuthority/aggregate.py | 32 +++++++++++++++-------------- 1 files changed, 17 insertions(+), 15 deletions(-) diff --git a/NetworkScanners/BwAuthority/aggregate.py b/NetworkScanners/BwAuthority/aggregate.py index ccf6436..a2372af 100755 --- a/NetworkScanners/BwAuthority/aggregate.py +++ b/NetworkScanners/BwAuthority/aggregate.py @@ -464,32 +464,34 @@ def main(argv): plog("INFO", "PID control enabled") if cs_junk.group_by_class: - for c in ["Guard+Exit", "Guard", "Exit", "Middle"]: - c_nodes = filter(lambda n: n.node_class() == c, nodes.itervalues()) - true_filt_avg[c] = sum(map(lambda n: n.filt_bw, c_nodes))/float(len(c_nodes)) - true_strm_avg[c] = sum(map(lambda n: n.strm_bw, c_nodes))/float(len(c_nodes)) - true_circ_avg[c] = sum(map(lambda n: (1.0-n.circ_fail_rate), + for cl in ["Guard+Exit", "Guard", "Exit", "Middle"]: + c_nodes = filter(lambda n: n.node_class() == cl, nodes.itervalues()) + true_filt_avg[cl] = sum(map(lambda n: n.filt_bw, c_nodes))/float(len(c_nodes)) + true_strm_avg[cl] = sum(map(lambda n: n.strm_bw, c_nodes))/float(len(c_nodes)) + true_circ_avg[cl] = sum(map(lambda n: (1.0-n.circ_fail_rate), c_nodes))/float(len(c_nodes)) - plog("INFO", "Network true_filt_avg["+c+"]: "+str(true_filt_avg[c])) - plog("INFO", "Network true_circ_avg["+c+"]: "+str(true_circ_avg[c])) + plog("INFO", "Network true_filt_avg["+cl+"]: "+str(true_filt_avg[cl])) + plog("INFO", "Network true_circ_avg["+cl+"]: "+str(true_circ_avg[cl])) else: filt_avg = sum(map(lambda n: n.filt_bw, nodes.itervalues()))/float(len(nodes)) strm_avg = sum(map(lambda n: n.strm_bw, nodes.itervalues()))/float(len(nodes)) circ_avg = sum(map(lambda n: (1.0-n.circ_fail_rate), nodes.itervalues()))/float(len(nodes)) - for c in ["Guard+Exit", "Guard", "Exit", "Middle"]: - true_filt_avg[c] = filt_avg - true_strm_avg[c] = strm_avg - true_circ_avg[c] = circ_avg + for cl in ["Guard+Exit", "Guard", "Exit", "Middle"]: + true_filt_avg[cl] = filt_avg + true_strm_avg[cl] = strm_avg + true_circ_avg[cl] = circ_avg + plog("INFO", "Network true_filt_avg: "+str(true_filt_avg["Middle"])) + plog("INFO", "Network true_circ_avg: "+str(true_circ_avg["Middle"])) else: plog("INFO", "PID control disabled") filt_avg = sum(map(lambda n: n.filt_bw*(1.0-n.circ_fail_rate), nodes.itervalues()))/float(len(nodes)) strm_avg = sum(map(lambda n: n.strm_bw*(1.0-n.circ_fail_rate), nodes.itervalues()))/float(len(nodes)) - for c in ["Guard+Exit", "Guard", "Exit", "Middle"]: - true_filt_avg[c] = filt_avg - true_strm_avg[c] = strm_avg + for cl in ["Guard+Exit", "Guard", "Exit", "Middle"]: + true_filt_avg[cl] = filt_avg + true_strm_avg[cl] = strm_avg prev_votes = None @@ -525,7 +527,7 @@ def main(argv): tot_net_bw = 0 for n in nodes.itervalues(): n.fbw_ratio = n.filt_bw/true_filt_avg[n.node_class()] - n.sbw_ratio = n.strm_bw/true_strm_avg[n.node_clasS()] + n.sbw_ratio = n.strm_bw/true_strm_avg[n.node_class()] if cs_junk.bwauth_pid_control: if cs_junk.use_desc_bw:

1 0

[torflow/master] Fix an exception that can happen with very large bandwidths.
by mikeperry＠torproject.org 02 Dec '11

02 Dec '11

commit 325bfcc0f000808c5528f90d69996ab0a186aa10 Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Thu Dec 1 16:56:46 2011 -0800 Fix an exception that can happen with very large bandwidths. We have a cap+log warn below that we will hit in this case instead of crashing. --- NetworkScanners/BwAuthority/aggregate.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/NetworkScanners/BwAuthority/aggregate.py b/NetworkScanners/BwAuthority/aggregate.py index 46dce2e..19697eb 100755 --- a/NetworkScanners/BwAuthority/aggregate.py +++ b/NetworkScanners/BwAuthority/aggregate.py @@ -345,7 +345,7 @@ def main(argv): ns_list = c.get_network_status() for n in ns_list: if n.bandwidth == None: n.bandwidth = -1 - ns_list.sort(lambda x, y: y.bandwidth - x.bandwidth) + ns_list.sort(lambda x, y: int(y.bandwidth/10000.0 - x.bandwidth/10000.0)) for n in ns_list: if n.bandwidth == -1: n.bandwidth = None got_ns_bw = False

1 0

[torflow/master] Alter default param values.
by mikeperry＠torproject.org 02 Dec '11

02 Dec '11

commit 4a4b8a73185f763f0def3e0d30c052f3abeb6fa0 Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Thu Dec 1 17:38:45 2011 -0800 Alter default param values. These defaults should still be equivalent to Section 2 in the spec, but these values will keep the PID interim state from disappearing every time ides crashes and the consensus params disappear. --- NetworkScanners/BwAuthority/README.spec.txt | 73 ++++++++++++++++----------- NetworkScanners/BwAuthority/aggregate.py | 24 ++++---- 2 files changed, 56 insertions(+), 41 deletions(-) diff --git a/NetworkScanners/BwAuthority/README.spec.txt b/NetworkScanners/BwAuthority/README.spec.txt index b8f86d4..81468b5 100644 --- a/NetworkScanners/BwAuthority/README.spec.txt +++ b/NetworkScanners/BwAuthority/README.spec.txt @@ -483,23 +483,59 @@ 3.6. Consensus Parameters The bandwidth auths listen for several consensus parameters to tweak - behavior: + behavior. - "bwauthpid=1" - If present, enables the PID control features in Section 3. + In the absence of any consensus parameters, the default behavior is + to use the PID control code to produce values identical to Section 2, + by using default values of: + + K_p = 1.0, K_i = 0, K_d = 0, as well as bwauthcircs=0. + + This equivalence was proved in Section 3.1, and has been observed + in practice. + + The available consensus parameters are: + + "bwauthpid=0" + If present, entirely disables the PID control features in + Section 3 and computes bandwidths according to Section 2. + + Setting this value to temporarily disable PID feedback is not + recommended, because it causes the PID code to lose interim + recorded state. + + To temporarily disable PID feedback, simply remove all consensus + parameters, and the system will compute Section 2 values while + retaining PID state. "bwauthcircs=1" If present, F_node is multiplied by (1.0 - circ_fail_rate) as described in Section 3.1. - + + "bwauthbestratio=0" + If absent, the larger of stream bandwidth vs filtered bandwidth + is used to compute F_node. + + If present, only filtered stream bandwidth ratios are used. + + "bwauthnsbw=1" + If present, uses consensus bandwidth to determine new bandwidth + values. + + If absent, uses descriptor bandwidth instead of feeding back + PID control values. This may be functionally equivalent to NS + bandwidth so long as T_i is non-zero, because error will get + accumulated in pid_error_sum as opposed to the consensus value + itself. + "bwauthkp=N" - Sets K_p to N/10000.0 + Sets K_p to N/10000.0. If absent, K_p=1.0. "bwauthti=N" - Sets T_i to N/10000.0. For T_i=0, K_i is set to 0. + Sets T_i to N/10000.0. If T_i=0 or absent, K_i is set to 0. "bwauthtd=N" - Sets T_d to N/10000.0. + Sets T_d to N/10000.0. If absent, K_d=0. "bwauthtidecay=N" Sets T_i_decay to N/10000.0. T_i_decay is an parameter @@ -513,25 +549,4 @@ the T_i'th round has experienced a reduction by T_i_decay for the values of T_i that are relevant to us. - If T_i is 0, K_i_decay is set to 0. - - "bwauthbestratio=1" - If present, the larger of stream bandwidth vs filtered bandwidth - is used to compute F_node. - - "bwauthdescbw=1" - If present, uses descriptor bandwidth instead of feeding back - PID control values. This may be functionally equivalent to NS - bandwidth so long as T_i is non-zero, because error will get - accumulated in pid_error_sum as opposed to the consensus value - itself. - - It also has the advantage of allowing the PID control code to be - exercised, yet to still produce identical results to Section 2 - by using the following consensus parameters: - - bwauthpid=1 bwauthdescbw=1 bwauthbestratio=1 bwauthcircs=0 - bwauthkp=10000 bwauthti=0 bwauthtd=0 - - This equivalence was proved in Section 3.1, and has been observed - in practice. + If T_i is 0 or absent, K_i_decay is set to 0. diff --git a/NetworkScanners/BwAuthority/aggregate.py b/NetworkScanners/BwAuthority/aggregate.py index 19697eb..6283f3f 100755 --- a/NetworkScanners/BwAuthority/aggregate.py +++ b/NetworkScanners/BwAuthority/aggregate.py @@ -28,19 +28,19 @@ GUARD_SAMPLE_RATE = 2*7*24*60*60 # 2wks K_p = 1.0 # We expect to correct steady state error in 5 samples (guess) -T_i = 5.0 +T_i = 0 # T_i_decay is a weight factor to govern how fast integral sums # decay. For the values of T_i that we care about, T_i_decay represents # the fraction of integral sum that is eliminated after T_i sample rounds. # This decay is non-standard, but we do it to avoid overflow -T_i_decay = 0.5 +T_i_decay = 0 # We can only expect to predict less than one sample into the future, as # after 1 sample, clients will have migrated # FIXME: Our prediction ability is a function of the consensus uptake time # vs measurement rate -T_d = 0.5 +T_d = 0 NODE_CAP = 0.05 @@ -221,10 +221,10 @@ class VoteSet: class ConsensusJunk: def __init__(self, c): cs_bytes = c.sendAndRecv("GETINFO dir/status-vote/current/consensus\r\n")[0][2] - self.bwauth_pid_control = False + self.bwauth_pid_control = True self.use_circ_fails = False - self.use_best_ratio = False - self.use_desc_bw = False + self.use_best_ratio = True + self.use_desc_bw = True self.K_p = K_p self.T_i = T_i @@ -235,16 +235,16 @@ class ConsensusJunk: cs_params = re.search("^params ((?:[\S]+=[\d]+[\s]?)+)", cs_bytes, re.M).group(1).split() for p in cs_params: - if p == "bwauthpid=1": - self.bwauth_pid_control = True - elif p == "bwauthdescbw=1": - self.use_desc_bw = True + if p == "bwauthpid=0": + self.bwauth_pid_control = False + elif p == "bwauthnsbw=1": + self.use_desc_bw = False plog("INFO", "Using descriptor bandwidth") elif p == "bwauthcircs=1": self.use_circ_fails = True plog("INFO", "Counting circuit failures") - elif p == "bwauthbestratio=1": - self.use_best_ratio = True + elif p == "bwauthbestratio=0": + self.use_best_ratio = False plog("INFO", "Choosing larger of sbw vs fbw") elif p.startswith("bwauthkp="): self.K_p = int(p.split("=")[1])/10000.0

1 0

[torflow/master] Implement per-class setpoints.
by mikeperry＠torproject.org 02 Dec '11

02 Dec '11

commit 4f4bc10fba85adc877481e3fb4445847a208da12 Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Thu Dec 1 20:09:25 2011 -0800 Implement per-class setpoints. Governed by consensus param (off by default). --- NetworkScanners/BwAuthority/README.spec.txt | 5 ++ NetworkScanners/BwAuthority/aggregate.py | 65 ++++++++++++++++++++++----- 2 files changed, 58 insertions(+), 12 deletions(-) diff --git a/NetworkScanners/BwAuthority/README.spec.txt b/NetworkScanners/BwAuthority/README.spec.txt index 81468b5..acce106 100644 --- a/NetworkScanners/BwAuthority/README.spec.txt +++ b/NetworkScanners/BwAuthority/README.spec.txt @@ -528,6 +528,11 @@ accumulated in pid_error_sum as opposed to the consensus value itself. + "bwauthbyclass=1" + If present, computes F_avg (see Section 3.1) for each class + of Guard, Middle, Exit, and Guard+Exit nodes, and uses these + flag-specific averages to compute pid_error. + "bwauthkp=N" Sets K_p to N/10000.0. If absent, K_p=1.0. diff --git a/NetworkScanners/BwAuthority/aggregate.py b/NetworkScanners/BwAuthority/aggregate.py index 6283f3f..120de07 100755 --- a/NetworkScanners/BwAuthority/aggregate.py +++ b/NetworkScanners/BwAuthority/aggregate.py @@ -92,6 +92,7 @@ class Node: self.new_bw = None self.change = None self.use_bw = -1 + self.flags = "" # measurement vars from bwauth lines self.measured_at = 0 @@ -132,6 +133,16 @@ class Node: + kd*self.use_bw*self.d_error_dt() return self.pid_bw + def node_class(self): + if "Guard" in self.flags and "Exit" in self.flags: + return "Guard+Exit" + elif "Guard" in self.flags: + return "Guard" + elif "Exit" in self.flags: + return "Exit" + else: + return "Middle" + # Time-weighted sum of error per unit of time (measurement sample) def integral_error(self): if self.prev_error == 0: @@ -222,6 +233,7 @@ class ConsensusJunk: def __init__(self, c): cs_bytes = c.sendAndRecv("GETINFO dir/status-vote/current/consensus\r\n")[0][2] self.bwauth_pid_control = True + self.group_by_class = False self.use_circ_fails = False self.use_best_ratio = True self.use_desc_bw = True @@ -246,6 +258,9 @@ class ConsensusJunk: elif p == "bwauthbestratio=0": self.use_best_ratio = False plog("INFO", "Choosing larger of sbw vs fbw") + elif p == "bwauthbyclass=1": + self.group_by_class = True + plog("INFO", "Grouping nodes by flag-class") elif p.startswith("bwauthkp="): self.K_p = int(p.split("=")[1])/10000.0 plog("INFO", "Got K_p=%f from consensus." % self.K_p) @@ -439,23 +454,45 @@ def main(argv): for n in nodes.itervalues(): n.circ_fail_rate = 0.0 + for idhex in nodes.iterkeys(): + if idhex in prev_consensus: + nodes[idhex].flags = prev_consensus[idhex].flags + + true_filt_avg = {} + true_strm_avg = {} + if cs_junk.bwauth_pid_control: # Penalize nodes for circuit failure: it indicates CPU pressure # TODO: Potentially penalize for stream failure, if we run into # socket exhaustion issues.. plog("INFO", "PID control enabled") - true_filt_avg = sum(map(lambda n: n.filt_bw*(1.0-n.circ_fail_rate), - nodes.itervalues()))/float(len(nodes)) - true_strm_avg = sum(map(lambda n: n.strm_bw*(1.0-n.circ_fail_rate), - nodes.itervalues()))/float(len(nodes)) + + if cs_junk.group_by_class: + for c in ["Guard+Exit", "Guard", "Exit", "Middle"]: + c_nodes = filter(lambda n: n.node_class() == c, nodes.itervalues()) + true_filt_avg[c] = sum(map(lambda n: n.filt_bw*(1.0-n.circ_fail_rate), + c_nodes))/float(len(c_nodes)) + true_strm_avg[c] = sum(map(lambda n: n.strm_bw*(1.0-n.circ_fail_rate), + c_nodes))/float(len(c_nodes)) + plog("INFO", "Network true_filt_avg["+c+"]: "+str(true_filt_avg[c])) + else: + filt_avg = sum(map(lambda n: n.filt_bw*(1.0-n.circ_fail_rate), + nodes.itervalues()))/float(len(nodes)) + strm_avg = sum(map(lambda n: n.strm_bw*(1.0-n.circ_fail_rate), + nodes.itervalues()))/float(len(nodes)) + for c in ["Guard+Exit", "Guard", "Exit", "Middle"]: + true_filt_avg[c] = filt_avg + true_strm_avg[c] = strm_avg else: plog("INFO", "PID control disabled") - true_filt_avg = sum(map(lambda n: n.filt_bw, - nodes.itervalues()))/float(len(nodes)) - true_strm_avg = sum(map(lambda n: n.strm_bw, + filt_avg = sum(map(lambda n: n.filt_bw*(1.0-n.circ_fail_rate), + nodes.itervalues()))/float(len(nodes)) + strm_avg = sum(map(lambda n: n.strm_bw*(1.0-n.circ_fail_rate), nodes.itervalues()))/float(len(nodes)) + for c in ["Guard+Exit", "Guard", "Exit", "Middle"]: + true_filt_avg[c] = filt_avg + true_strm_avg[c] = strm_avg - plog("DEBUG", "Network true_filt_avg: "+str(true_filt_avg)) prev_votes = None if cs_junk.bwauth_pid_control: @@ -489,8 +526,8 @@ def main(argv): tot_net_bw = 0 for n in nodes.itervalues(): - n.fbw_ratio = n.filt_bw/true_filt_avg - n.sbw_ratio = n.strm_bw/true_strm_avg + n.fbw_ratio = n.filt_bw/true_filt_avg[n.node_class()] + n.sbw_ratio = n.strm_bw/true_strm_avg[n.node_clasS()] if cs_junk.bwauth_pid_control: if cs_junk.use_desc_bw: @@ -500,9 +537,13 @@ def main(argv): # Penalize nodes for circ failure rate if cs_junk.use_best_ratio and n.sbw_ratio > n.fbw_ratio: - n.pid_error = (n.strm_bw*(1.0-n.circ_fail_rate) - true_strm_avg)/true_strm_avg + n.pid_error = (n.strm_bw*(1.0-n.circ_fail_rate) - + true_strm_avg[n.node_class()]) \ + / true_strm_avg[n.node_class()] else: - n.pid_error = (n.filt_bw*(1.0-n.circ_fail_rate) - true_filt_avg)/true_filt_avg + n.pid_error = (n.filt_bw*(1.0-n.circ_fail_rate) - + true_filt_avg[n.node_class()]) \ + / true_filt_avg[n.node_class()] if n.idhex in prev_votes.vote_map: # If there is a new sample, let's use it for all but guards

1 0

[torflow/master] Change how circuit failure is used.
by mikeperry＠torproject.org 02 Dec '11

02 Dec '11

commit 83f29ce70d0846206daa76f7ad8ab2cf3f5efbd6 Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Thu Dec 1 20:54:46 2011 -0800 Change how circuit failure is used. Let's try computing a second error and take their sum. --- NetworkScanners/BwAuthority/README.spec.txt | 16 +++++++--- NetworkScanners/BwAuthority/aggregate.py | 40 +++++++++++++++------------ 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/NetworkScanners/BwAuthority/README.spec.txt b/NetworkScanners/BwAuthority/README.spec.txt index acce106..db5c125 100644 --- a/NetworkScanners/BwAuthority/README.spec.txt +++ b/NetworkScanners/BwAuthority/README.spec.txt @@ -369,9 +369,7 @@ 3.1. Modeling Measurement as PID Control The bandwidth authorities measure F_node: the filtered stream - capacity through a given node (filtering is described in Section 1.6) - times the circuit success rate of circuit EXTENDs to the node - (1.0 - circ_fail_rate). + capacity through a given node (filtering is described in Section 1.6). In PID control, we add in this extra failure rate as a damper, to prevent the PID control system from driving nodes to CPU overload. Once nodes @@ -509,8 +507,16 @@ retaining PID state. "bwauthcircs=1" - If present, F_node is multiplied by (1.0 - circ_fail_rate) - as described in Section 3.1. + If present, an additional circ_error value is computed for each + node similar to pid_error of Section 3.1. This value is: + + circ_error = (circ_rate - circ_avg_rate)/circ_avg_rate + + Where circ_rate and circ_avg_rate are the EXTEND success rates + to the node, and the average success rate for the entire node class, + respectively. + + This error value is then added to pid_error. "bwauthbestratio=0" If absent, the larger of stream bandwidth vs filtered bandwidth diff --git a/NetworkScanners/BwAuthority/aggregate.py b/NetworkScanners/BwAuthority/aggregate.py index 120de07..ccf6436 100755 --- a/NetworkScanners/BwAuthority/aggregate.py +++ b/NetworkScanners/BwAuthority/aggregate.py @@ -449,17 +449,13 @@ def main(argv): plog("NOTICE", "No scan results yet.") sys.exit(1) - if not cs_junk.use_circ_fails: - plog("INFO", "Ignoring circuit failures") - for n in nodes.itervalues(): - n.circ_fail_rate = 0.0 - for idhex in nodes.iterkeys(): if idhex in prev_consensus: nodes[idhex].flags = prev_consensus[idhex].flags true_filt_avg = {} true_strm_avg = {} + true_circ_avg = {} if cs_junk.bwauth_pid_control: # Penalize nodes for circuit failure: it indicates CPU pressure @@ -470,19 +466,21 @@ def main(argv): if cs_junk.group_by_class: for c in ["Guard+Exit", "Guard", "Exit", "Middle"]: c_nodes = filter(lambda n: n.node_class() == c, nodes.itervalues()) - true_filt_avg[c] = sum(map(lambda n: n.filt_bw*(1.0-n.circ_fail_rate), - c_nodes))/float(len(c_nodes)) - true_strm_avg[c] = sum(map(lambda n: n.strm_bw*(1.0-n.circ_fail_rate), - c_nodes))/float(len(c_nodes)) + true_filt_avg[c] = sum(map(lambda n: n.filt_bw, c_nodes))/float(len(c_nodes)) + true_strm_avg[c] = sum(map(lambda n: n.strm_bw, c_nodes))/float(len(c_nodes)) + true_circ_avg[c] = sum(map(lambda n: (1.0-n.circ_fail_rate), + c_nodes))/float(len(c_nodes)) plog("INFO", "Network true_filt_avg["+c+"]: "+str(true_filt_avg[c])) + plog("INFO", "Network true_circ_avg["+c+"]: "+str(true_circ_avg[c])) else: - filt_avg = sum(map(lambda n: n.filt_bw*(1.0-n.circ_fail_rate), - nodes.itervalues()))/float(len(nodes)) - strm_avg = sum(map(lambda n: n.strm_bw*(1.0-n.circ_fail_rate), - nodes.itervalues()))/float(len(nodes)) + filt_avg = sum(map(lambda n: n.filt_bw, nodes.itervalues()))/float(len(nodes)) + strm_avg = sum(map(lambda n: n.strm_bw, nodes.itervalues()))/float(len(nodes)) + circ_avg = sum(map(lambda n: (1.0-n.circ_fail_rate), + nodes.itervalues()))/float(len(nodes)) for c in ["Guard+Exit", "Guard", "Exit", "Middle"]: true_filt_avg[c] = filt_avg true_strm_avg[c] = strm_avg + true_circ_avg[c] = circ_avg else: plog("INFO", "PID control disabled") filt_avg = sum(map(lambda n: n.filt_bw*(1.0-n.circ_fail_rate), @@ -535,16 +533,22 @@ def main(argv): else: n.use_bw = n.ns_bw - # Penalize nodes for circ failure rate if cs_junk.use_best_ratio and n.sbw_ratio > n.fbw_ratio: - n.pid_error = (n.strm_bw*(1.0-n.circ_fail_rate) - - true_strm_avg[n.node_class()]) \ + n.pid_error = (n.strm_bw - true_strm_avg[n.node_class()]) \ / true_strm_avg[n.node_class()] else: - n.pid_error = (n.filt_bw*(1.0-n.circ_fail_rate) - - true_filt_avg[n.node_class()]) \ + n.pid_error = (n.filt_bw - true_filt_avg[n.node_class()]) \ / true_filt_avg[n.node_class()] + # Penalize nodes for circ failure rate + if cs_junk.use_circ_fails: + circ_error = ((1.0-n.circ_fail_rate) - true_circ_avg[n.node_class()]) \ + / true_circ_avg[n.node_class()] + # FIXME: Hrmm, should we only penalize for circ successes, or should + # we reward, too? Let's try both for now. + # if circ_error < 0: + n.pid_error += circ_error + if n.idhex in prev_votes.vote_map: # If there is a new sample, let's use it for all but guards if n.measured_at > prev_votes.vote_map[n.idhex].measured_at:

1 0

[stem/master] Function and testing for cookie authentication
by atagar＠torproject.org 01 Dec '11

01 Dec '11

commit 8fd556572a170d06359458a37848d947313984d2 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Dec 1 10:10:43 2011 -0800 Function and testing for cookie authentication Adding a function for password authentication. This includes checks for the file's existance and that the size is valid (for 4303). --- stem/connection.py | 56 ++++++++++++++++++ test/integ/connection/authentication.py | 94 ++++++++++++++++++++++-------- 2 files changed, 125 insertions(+), 25 deletions(-) diff --git a/stem/connection.py b/stem/connection.py index 3ea5747..0b654d8 100644 --- a/stem/connection.py +++ b/stem/connection.py @@ -14,7 +14,9 @@ ProtocolInfoResponse - Reply from a PROTOCOLINFO query. +- convert - parses a ControlMessage, turning it into a ProtocolInfoResponse """ +import os import logging +import binascii import stem.socket import stem.version @@ -38,6 +40,9 @@ LOGGER = logging.getLogger("stem") AuthMethod = stem.util.enum.Enum("NONE", "PASSWORD", "COOKIE", "UNKNOWN") +AUTH_COOKIE_MISSING = "Authentication failed: '%s' doesn't exist" +AUTH_COOKIE_WRONG_SIZE = "Authentication failed: authentication cookie '%s' is the wrong size (%i bytes instead of 32)" + def authenticate_none(control_socket): """ Authenticates to an open control socket. All control connections need to @@ -92,6 +97,57 @@ def authenticate_password(control_socket, password): if str(auth_response) != "OK": raise ValueError(str(auth_response)) +def authenticate_cookie(control_socket, cookie_path): + """ + Authenticates to a control socket that uses the contents of an authentication + cookie (generated via the CookieAuthentication torrc option). This does basic + validation that this is a cookie before presenting the contents to the + socket. + + If authentication fails then tor will close the control socket. + + Arguments: + control_socket (stem.socket.ControlSocket) - socket to be authenticated + cookie_path (str) - path of the authentication cookie to send to tor + + Raises: + ValueError if the authentication credentials aren't accepted + OSError if the cookie file doesn't exist or we're unable to read it + stem.socket.ProtocolError the content from the socket is malformed + stem.socket.SocketError if problems arise in using the socket + """ + + if not os.path.exists(cookie_path): + raise OSError(AUTH_COOKIE_MISSING % cookie_path) + + # Abort if the file isn't 32 bytes long. This is to avoid exposing arbitrary + # file content to the port. + # + # Without this a malicious socket could, for instance, claim that + # '~/.bash_history' or '~/.ssh/id_rsa' was its authentication cookie to trick + # us into reading it for them with our current permissions. + # + # https://trac.torproject.org/projects/tor/ticket/4303 + + auth_cookie_size = os.path.getsize(cookie_path) + + if auth_cookie_size != 32: + raise ValueError(AUTH_COOKIE_WRONG_SIZE % (cookie_path, auth_cookie_size)) + + try: + auth_cookie_file = open(cookie_path, "r") + auth_cookie_contents = auth_cookie_file.read() + auth_cookie_file.close() + + control_socket.send("AUTHENTICATE %s" % binascii.b2a_hex(auth_cookie_contents)) + auth_response = control_socket.recv() + + # if we got anything but an OK response then error + if str(auth_response) != "OK": + raise ValueError(str(auth_response)) + except IOError, exc: + raise OSError(exc) + def get_protocolinfo_by_port(control_addr = "127.0.0.1", control_port = 9051, get_socket = False): """ Issues a PROTOCOLINFO query to a control port, getting information about the diff --git a/test/integ/connection/authentication.py b/test/integ/connection/authentication.py index c3759e6..7883c8a 100644 --- a/test/integ/connection/authentication.py +++ b/test/integ/connection/authentication.py @@ -3,6 +3,7 @@ Integration tests for authenticating to the control socket via stem.connection.authenticate_* functions. """ +import os import unittest import functools @@ -25,17 +26,18 @@ class TestAuthenticate(unittest.TestCase): integ target to exercise the widest range of use cases. """ + def setUp(self): + connection_type = test.runner.get_runner().get_connection_type() + + # none of these tests apply if there's no control connection + if connection_type == test.runner.TorConnection.NONE: + self.skipTest("(no connection)") + def test_authenticate_none(self): """ Tests the authenticate_none function. """ - runner = test.runner.get_runner() - connection_type = runner.get_connection_type() - - if connection_type == test.runner.TorConnection.NONE: - self.skipTest("(no connection)") - expect_success = self._is_authenticateable(stem.connection.AuthMethod.NONE) self._check_auth(stem.connection.AuthMethod.NONE, None, expect_success) @@ -44,12 +46,6 @@ class TestAuthenticate(unittest.TestCase): Tests the authenticate_password function. """ - runner = test.runner.get_runner() - connection_type = runner.get_connection_type() - - if connection_type == test.runner.TorConnection.NONE: - self.skipTest("(no connection)") - expect_success = self._is_authenticateable(stem.connection.AuthMethod.PASSWORD) self._check_auth(stem.connection.AuthMethod.PASSWORD, test.runner.CONTROL_PASSWORD, expect_success) @@ -61,6 +57,42 @@ class TestAuthenticate(unittest.TestCase): self._check_auth(stem.connection.AuthMethod.PASSWORD, "blarg", expect_success) self._check_auth(stem.connection.AuthMethod.PASSWORD, "this has a \" in it", expect_success) + def test_authenticate_cookie(self): + """ + Tests the authenticate_cookie function. + """ + + test_path = test.runner.get_runner().get_auth_cookie_path() + expect_success = self._is_authenticateable(stem.connection.AuthMethod.COOKIE) + self._check_auth(stem.connection.AuthMethod.COOKIE, test_path, expect_success) + + def test_authenticate_cookie_missing(self): + """ + Tests the authenticate_cookie function with a path that really, really + shouldn't exist. + """ + + test_path = "/if/this/exists/then/they're/asking/for/a/failure" + expected_exc = OSError(stem.connection.AUTH_COOKIE_MISSING % test_path) + self._check_auth(stem.connection.AuthMethod.COOKIE, test_path, False, expected_exc) + + def test_authenticate_cookie_wrong_size(self): + """ + Tests the authenticate_cookie function with our torrc as an auth cookie. + This is to confirm that we won't read arbitrary files to the control + socket. + """ + + test_path = test.runner.get_runner().get_torrc_path() + auth_cookie_size = os.path.getsize(test_path) + + if auth_cookie_size == 32: + # Weird coincidence? Fail so we can pick another file to check against. + self.fail("Our torrc is 32 bytes, preventing the test_authenticate_cookie_wrong_size test from running.") + else: + expected_exc = ValueError(stem.connection.AUTH_COOKIE_WRONG_SIZE % (test_path, auth_cookie_size)) + self._check_auth(stem.connection.AuthMethod.COOKIE, test_path, False, expected_exc) + def _get_socket_auth(self): """ Provides the types of authentication that our current test socket accepts. @@ -99,7 +131,7 @@ class TestAuthenticate(unittest.TestCase): elif auth_type == stem.connection.AuthMethod.COOKIE: return cookie_auth else: return False - def _check_auth(self, auth_type, auth_value, expect_success): + def _check_auth(self, auth_type, auth_value, expect_success, failure_exc = None): """ Attempts to use the given authentication function against our connection. If this works then checks that we can use the connection. If not then we @@ -111,6 +143,8 @@ class TestAuthenticate(unittest.TestCase): auth_value (str) - value to be provided to the authentication function expect_success (bool) - true if the authentication should succeed, false otherwise + failure_exc (Exception) - exception that we want to assert is raised, if + None then we'll check for an auth mismatch error """ runner = test.runner.get_runner() @@ -124,7 +158,7 @@ class TestAuthenticate(unittest.TestCase): elif auth_type == stem.connection.AuthMethod.PASSWORD: auth_function = stem.connection.authenticate_password elif auth_type == stem.connection.AuthMethod.COOKIE: - auth_function = None # TODO: fill in + auth_function = stem.connection.authenticate_cookie else: raise ValueError("unexpected auth type: %s" % auth_type) @@ -143,20 +177,30 @@ class TestAuthenticate(unittest.TestCase): self.assertEquals("config-file=%s\nOK" % runner.get_torrc_path(), str(config_file_response)) control_socket.close() else: - if cookie_auth and password_auth: failure_msg = MULTIPLE_AUTH_FAIL - elif cookie_auth: failure_msg = COOKIE_AUTH_FAIL - else: - # if we're attempting to authenticate with a password then it's a - # truncated message - - if auth_type == stem.connection.AuthMethod.PASSWORD: - failure_msg = INCORRECT_PASSWORD_FAIL + # if unset then determine what the general authentication error should + # look like + + if not failure_exc: + if cookie_auth and password_auth: + failure_exc = ValueError(MULTIPLE_AUTH_FAIL) + elif cookie_auth: + failure_exc = ValueError(COOKIE_AUTH_FAIL) else: - failure_msg = PASSWORD_AUTH_FAIL + # if we're attempting to authenticate with a password then it's a + # truncated message + + if auth_type == stem.connection.AuthMethod.PASSWORD: + failure_exc = ValueError(INCORRECT_PASSWORD_FAIL) + else: + failure_exc = ValueError(PASSWORD_AUTH_FAIL) try: auth_function() self.fail() - except ValueError, exc: - self.assertEqual(failure_msg, str(exc)) + except Exception, exc: + # we can't check exception equality directly because it contains other + # attributes which will fail + + self.assertEqual(type(failure_exc), type(exc)) + self.assertEqual(str(failure_exc), str(exc))

1 0

[stem/master] Skipping get_pid_by_name test when doomed
by atagar＠torproject.org 01 Dec '11

01 Dec '11

commit 2c91e342a58dd389d5cb777cb1f4f59bfc198f71 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Nov 30 19:02:32 2011 -0800 Skipping get_pid_by_name test when doomed The get_pid_by_name function fails when there's multiple instances of a process with that name. As a result the integ test for the function fails if there's extra tor instances running on the system. Using pgrep to check for other instances and skip those tests if they'd be doomed to failure. --- test/integ/util/system.py | 41 ++++++++++++++++++++++++++++++++++------- 1 files changed, 34 insertions(+), 7 deletions(-) diff --git a/test/integ/util/system.py b/test/integ/util/system.py index dfce42d..d1e9f88 100644 --- a/test/integ/util/system.py +++ b/test/integ/util/system.py @@ -16,6 +16,18 @@ class TestSystem(unittest.TestCase): running. """ + is_extra_tor_running = None + + def setUp(self): + # Try to figure out if there's more than one tor instance running. This + # check will fail if pgrep is unavailable (for instance on bsd) but this + # isn't the end of the world. It's just used to skip tests if they should + # legitemately fail. + + if self.is_extra_tor_running == None: + pgrep_results = stem.util.system.call("pgrep -x tor") + self.is_extra_tor_running = len(pgrep_results) > 1 + def tearDown(self): # resets call mocking back to being disabled stem.util.system.CALL_MOCKING = None @@ -45,6 +57,9 @@ class TestSystem(unittest.TestCase): will fail if there's other tor instances running. """ + if self.is_extra_tor_running: + self.skipTest("(multiple tor instances)") + runner = test.runner.get_runner() self.assertEquals(runner.get_pid(), stem.util.system.get_pid_by_name("tor")) self.assertEquals(None, stem.util.system.get_pid_by_name("blarg_and_stuff")) @@ -54,7 +69,9 @@ class TestSystem(unittest.TestCase): Tests the get_pid_by_name function with a pgrep response. """ - if not stem.util.system.is_available("pgrep"): + if self.is_extra_tor_running: + self.skipTest("(multiple tor instances)") + elif not stem.util.system.is_available("pgrep"): self.skipTest("(pgrep unavailable)") pgrep_prefix = stem.util.system.GET_PID_BY_NAME_PGREP % "" @@ -65,7 +82,9 @@ class TestSystem(unittest.TestCase): Tests the get_pid_by_name function with a pidof response. """ - if not stem.util.system.is_available("pidof"): + if self.is_extra_tor_running: + self.skipTest("(multiple tor instances)") + elif not stem.util.system.is_available("pidof"): self.skipTest("(pidof unavailable)") pidof_prefix = stem.util.system.GET_PID_BY_NAME_PIDOF % "" @@ -76,9 +95,12 @@ class TestSystem(unittest.TestCase): Tests the get_pid_by_name function with the linux variant of ps. """ - if not stem.util.system.is_available("ps"): + if self.is_extra_tor_running: + self.skipTest("(multiple tor instances)") + elif not stem.util.system.is_available("ps"): self.skipTest("(ps unavailable)") - elif stem.util.system.is_bsd(): self.skipTest("(linux only)") + elif stem.util.system.is_bsd(): + self.skipTest("(linux only)") ps_prefix = stem.util.system.GET_PID_BY_NAME_PS_LINUX % "" self._run_pid_test(ps_prefix, stem.util.system.get_pid_by_name, "tor") @@ -88,9 +110,12 @@ class TestSystem(unittest.TestCase): Tests the get_pid_by_name function with the bsd variant of ps. """ - if not stem.util.system.is_available("ps"): + if self.is_extra_tor_running: + self.skipTest("(multiple tor instances)") + elif not stem.util.system.is_available("ps"): self.skipTest("(ps unavailable)") - elif not stem.util.system.is_bsd(): self.skipTest("(bsd only)") + elif not stem.util.system.is_bsd(): + self.skipTest("(bsd only)") ps_cmd = stem.util.system.GET_PID_BY_NAME_PS_BSD self._run_pid_test(ps_cmd, stem.util.system.get_pid_by_name, "tor") @@ -100,7 +125,9 @@ class TestSystem(unittest.TestCase): Tests the get_pid_by_name function with a lsof response. """ - if not stem.util.system.is_available("lsof"): + if self.is_extra_tor_running: + self.skipTest("(multiple tor instances)") + elif not stem.util.system.is_available("lsof"): self.skipTest("(lsof unavailable)") lsof_prefix = stem.util.system.GET_PID_BY_NAME_LSOF % ""

1 0

[arm/master] Fix a help entry on connection panel
by atagar＠torproject.org 01 Dec '11

01 Dec '11

commit 1945566a7e57e45bb9f60c050859ac82fdc44d43 Author: Tim Wilde <github(a)krellis.org> Date: Thu Dec 1 01:37:58 2011 +0000 Fix a help entry on connection panel --- src/cli/connections/connPanel.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/src/cli/connections/connPanel.py b/src/cli/connections/connPanel.py index 9745176..3a8a07c 100644 --- a/src/cli/connections/connPanel.py +++ b/src/cli/connections/connPanel.py @@ -308,7 +308,7 @@ class ConnectionPanel(panel.Panel, threading.Thread): options.append(("down arrow", "scroll down a line", None)) options.append(("page up", "scroll up a page", None)) options.append(("page down", "scroll down a page", None)) - options.append(("enter", "edit configuration option", None)) + options.append(("enter", "toggle connection details", None)) options.append(("d", "raw consensus descriptor", None)) if self.isClientsAllowed():

1 0

[arm/master] Replacing 'toggle' with 'show'
by atagar＠torproject.org 01 Dec '11

01 Dec '11

commit fbf672b17899f185e799b57f24ddfafcb5e89dd5 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Dec 1 07:32:22 2011 -0800 Replacing 'toggle' with 'show' Minor change to the previous fix's wording. --- src/cli/connections/connPanel.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/src/cli/connections/connPanel.py b/src/cli/connections/connPanel.py index 3a8a07c..d108252 100644 --- a/src/cli/connections/connPanel.py +++ b/src/cli/connections/connPanel.py @@ -308,7 +308,7 @@ class ConnectionPanel(panel.Panel, threading.Thread): options.append(("down arrow", "scroll down a line", None)) options.append(("page up", "scroll up a page", None)) options.append(("page down", "scroll down a page", None)) - options.append(("enter", "toggle connection details", None)) + options.append(("enter", "show connection details", None)) options.append(("d", "raw consensus descriptor", None)) if self.isClientsAllowed():

1 0