August 2012 - tor-commits - lists.torproject.org

[tor/master] Merge remote-tracking branch 'origin/maint-0.2.3'
by nickm＠torproject.org 24 Aug '12

24 Aug '12

commit 6864a44a4a6372c909ed8ea308881e4e8420046e Merge: ca09ea0 ce4add4 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Aug 24 12:51:50 2012 -0400 Merge remote-tracking branch 'origin/maint-0.2.3' changes/bug6472 | 4 ++++ src/or/dns.c | 16 ++++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-)

1 0

[tor/master] When iterating over connections pending DNS, skip marked ones
by nickm＠torproject.org 24 Aug '12

24 Aug '12

commit 223e7cfabec5a75a7acd919210f838511e39cb16 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Aug 17 16:46:11 2012 -0400 When iterating over connections pending DNS, skip marked ones Failure to do this would lead to double-free cases and similar, especially when the exit's DNS was broken. See bug 6472 for full details; this is a fix for 6472. Anonymous patch from "cypherpunks" on trac. --- changes/bug6472 | 4 ++++ src/or/dns.c | 16 ++++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/changes/bug6472 b/changes/bug6472 new file mode 100644 index 0000000..dcd42eb --- /dev/null +++ b/changes/bug6472 @@ -0,0 +1,4 @@ + o Minor bugfixes: + - Avoid a pair of double-free and use-after-mark bugs that can + occur with certain timings in canceled and re-received DNS + requests. Fix for bug 6472; bugfix on 0.0.7rc1. diff --git a/src/or/dns.c b/src/or/dns.c index 3e88fad..78893bf 100644 --- a/src/or/dns.c +++ b/src/or/dns.c @@ -450,16 +450,17 @@ purge_expired_resolves(time_t now) if (resolve->pending_connections) { log_debug(LD_EXIT, "Closing pending connections on timed-out DNS resolve!"); - tor_fragile_assert(); while (resolve->pending_connections) { pend = resolve->pending_connections; resolve->pending_connections = pend->next; /* Connections should only be pending if they have no socket. */ tor_assert(!SOCKET_OK(pend->conn->_base.s)); pendconn = pend->conn; - connection_edge_end(pendconn, END_STREAM_REASON_TIMEOUT); - circuit_detach_stream(circuit_get_by_edge_conn(pendconn), pendconn); - connection_free(TO_CONN(pendconn)); + if (!pendconn->_base.marked_for_close) { + connection_edge_end(pendconn, END_STREAM_REASON_TIMEOUT); + circuit_detach_stream(circuit_get_by_edge_conn(pendconn), pendconn); + connection_free(TO_CONN(pendconn)); + } tor_free(pend); } } @@ -1091,6 +1092,13 @@ dns_found_answer(const char *address, uint8_t is_reverse, uint32_t addr, pendconn = pend->conn; /* don't pass complex things to the connection_mark_for_close macro */ assert_connection_ok(TO_CONN(pendconn),time(NULL)); + if (pendconn->_base.marked_for_close) { + /* prevent double-remove. */ + pendconn->_base.state = EXIT_CONN_STATE_RESOLVEFAILED; + resolve->pending_connections = pend->next; + tor_free(pend); + continue; + } tor_addr_from_ipv4h(&pendconn->_base.addr, addr); pendconn->address_ttl = ttl;

1 0

[tor/master] Merge remote-tracking branch 'public/bug6472' into maint-0.2.3
by nickm＠torproject.org 24 Aug '12

24 Aug '12

commit ce4add498f6af197a0e856d262825d547f898305 Merge: 991a8ac 223e7cf Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Aug 24 12:51:02 2012 -0400 Merge remote-tracking branch 'public/bug6472' into maint-0.2.3 changes/bug6472 | 4 ++++ src/or/dns.c | 16 ++++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-)

1 0

[metrics-web/master] Change almost-fast-exit graphs to match Compass' results.
by karsten＠torproject.org 24 Aug '12

24 Aug '12

commit 69eeeeaa8991378fb58e195ebbfa8356d8837d65 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Aug 24 20:21:56 2012 +0200 Change almost-fast-exit graphs to match Compass' results. --- rserve/graphs.R | 29 +++++++++++++++++++---------- 1 files changed, 19 insertions(+), 10 deletions(-) diff --git a/rserve/graphs.R b/rserve/graphs.R index 1379691..470cdce 100644 --- a/rserve/graphs.R +++ b/rserve/graphs.R @@ -957,21 +957,30 @@ plot_almost_fast_exits <- function(start, end, path, dpi) { t$min_advbw == 5000 & t$same_network == TRUE, ] t2 <- t[t$min_rate == 10000 & t$ports == "80-443" & t$min_advbw == 2000 & t$same_network == FALSE, ] - t <- rbind(data.frame(t1, var = paste("95+ Mbit/s, 5000+ KB/s,", - "80/443/554/1755, 2- per /24")), - data.frame(t2, var = "80+ Mbit/s, 2000+ KB/s, 80/443")) - t <- aggregate(list(relays = t$relays, P_exit = 100 * t$exit_prob), + t <- rbind(data.frame(t1, var = "fast"), + data.frame(t2, var = "almost_fast")) + r <- cast(t, valid_after ~ var, value = "relays") + r <- data.frame(valid_after = r$valid_after, fast = r$fast, + almost = r$almost_fast - r$fast, var = "relays") + e <- cast(t, valid_after ~ var, value = "exit_prob") + e <- data.frame(valid_after = e$valid_after, fast = 100 * e$fast, + almost = 100 * (e$almost_fast - e$fast), var = "exit_prob") + t <- rbind(r, e) + t <- aggregate(list(fast = t$fast, almost = t$almost), by = list(date = as.Date(cut.Date(as.Date(t$valid_after), "day")), - var = t$var), FUN = median) + var = ifelse(t$var == "exit_prob", "Total exit probability (in %)", + "Number of relays")), FUN = median) t <- melt(t, id.vars = c("date", "var")) - t <- data.frame(t, type = ifelse(t$variable == "P_exit", - "Total exit probability (in %)", "Number of relays")) - ggplot(t, aes(x = date, y = value, colour = var)) + + t <- data.frame(t, type = ifelse(t$variable == "fast", + "fast exits (95+ Mbit/s, 5000+ KB/s, 80/443/554/1755, 2- per /24", + paste("almost fast exits (80+ Mbit/s, 2000+ KB/s, 80/443,", + "not in set of fast exits)"))) + ggplot(t, aes(x = date, y = value, colour = type)) + geom_line(size = 0.75) + - facet_grid(type ~ ., scales = "free_y") + + facet_grid(var ~ ., scales = "free_y") + scale_x_date(name = "") + scale_y_continuous(name = "") + - scale_colour_manual(name = "", values = c("purple", "orange")) + + scale_colour_manual(name = "", values = c("orange", "purple")) + opts(title = "Relays almost meeting the fast-exit requirements", legend.position = "top") ggsave(filename = path, width = 8, height = 6, dpi = as.numeric(dpi))

1 0

[compass/master] Refactor FastExitFilter
by karsten＠torproject.org 24 Aug '12

24 Aug '12

commit e59c6d108ef2089b788698ba3a0b5cbf2537eb80 Author: Sathyanarayanan Gunasekaran <gsathya.ceg(a)gmail.com> Date: Mon Aug 20 12:56:38 2012 +0530 Refactor FastExitFilter FastExitFilter now takes bandwidth_rate, advertised_bandwidth, list of ports and an "inverse" boolean as constructor args. The "inverse" switch makes the filter /retain/ all relays which it would otherwise /remove/ --- compass.py | 26 ++++++++++++++++---------- 1 files changed, 16 insertions(+), 10 deletions(-) diff --git a/compass.py b/compass.py index 4d350b4..971b2e0 100755 --- a/compass.py +++ b/compass.py @@ -78,19 +78,25 @@ class GuardFilter(BaseFilter): return relay.get('guard_probability', -1) > 0.0 class FastExitFilter(BaseFilter): + def __init__(self, bandwidth_rate, advertised_bandwidth, ports, inverse=False): + self.bandwidth_rate = bandwidth_rate + self.advertised_bandwidth = advertised_bandwidth + self.ports = ports + self.inverse = inverse + def accept(self, relay): - if relay.get('bandwidth_rate', -1) < 12500 * 1024: - return False - if relay.get('advertised_bandwidth', -1) < 5000 * 1024: - return False - relevant_ports = set([80, 443, 554, 1755]) + if relay.get('bandwidth_rate', -1) < self.bandwidth_rate: + return self.inverse + if relay.get('advertised_bandwidth', -1) < self.advertised_bandwidth: + return self.inverse + relevant_ports = set(self.ports) summary = relay.get('exit_policy_summary', {}) if 'accept' in summary: portlist = summary['accept'] elif 'reject' in summary: portlist = summary['reject'] else: - return False + return self.inverse ports = [] for p in portlist: if '-' in p: @@ -100,10 +106,10 @@ class FastExitFilter(BaseFilter): ports.append(int(p)) policy_ports = set(ports) if 'accept' in summary and not relevant_ports.issubset(policy_ports): - return False + return self.inverse if 'reject' in summary and not relevant_ports.isdisjoint(policy_ports): - return False - return True + return self.inverse + return not self.inverse class RelayStats(object): def __init__(self, options): @@ -149,7 +155,7 @@ class RelayStats(object): if options.guards_only: filters.append(GuardFilter()) if options.fast_exits_only: - filters.append(FastExitFilter()) + filters.append(FastExitFilter(95 * 125 * 1024, 5000 * 1024, [80, 443, 554, 1755], False)) return filters def _get_group_function(self, options):

1 0

[compass/master] Iterate all relays over each filter
by karsten＠torproject.org 24 Aug '12

24 Aug '12

commit 9bb37cc3e7f541474f349d5f5f30fddb88b856d5 Author: Sathyanarayanan Gunasekaran <gsathya.ceg(a)gmail.com> Date: Mon Aug 20 13:53:55 2012 +0530 Iterate all relays over each filter Previously all the filters were iterated over each relay. The order of the filter applied couldn't be controlled with this approach. Now, we can specify the order in which the relays are to be filtered in. --- compass.py | 14 ++++++-------- 1 files changed, 6 insertions(+), 8 deletions(-) diff --git a/compass.py b/compass.py index 971b2e0..544306e 100755 --- a/compass.py +++ b/compass.py @@ -130,14 +130,12 @@ class RelayStats(object): return self._relays self._relays = {} - for relay in self.data['relays']: - accepted = True - for f in self._filters: - if not f.accept(relay): - accepted = False - break - if accepted: - self.add_relay(relay) + relays = self.data['relays'] + for f in self._filters: + relays = filter(f.accept, relays) + + for relay in relays: + self.add_relay(relay) return self._relays def _create_filters(self, options):

1 0

[compass/master] Add load() method to BaseFilter
by karsten＠torproject.org 24 Aug '12

24 Aug '12

commit cba240accc983d885737daaef15774ed515fb6ca Author: Sathyanarayanan Gunasekaran <gsathya.ceg(a)gmail.com> Date: Mon Aug 20 17:05:40 2012 +0530 Add load() method to BaseFilter This can be used as a basic init method for all the filters that inherit BaseFilter to do some initial preprocessing. --- compass.py | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diff --git a/compass.py b/compass.py index 544306e..9d727c8 100755 --- a/compass.py +++ b/compass.py @@ -20,6 +20,9 @@ class BaseFilter(object): def accept(self, relay): pass + def load(self, relays): + pass + class RunningFilter(BaseFilter): def accept(self, relay): return relay['running'] @@ -132,6 +135,7 @@ class RelayStats(object): self._relays = {} relays = self.data['relays'] for f in self._filters: + f.load(relays) relays = filter(f.accept, relays) for relay in relays:

1 0

[compass/master] Use load() instead of accept()
by karsten＠torproject.org 24 Aug '12

24 Aug '12

commit bb72ff540c105523202f1c254392f8d3832076b8 Author: Sathyanarayanan Gunasekaran <gsathya.ceg(a)gmail.com> Date: Mon Aug 20 22:09:51 2012 +0530 Use load() instead of accept() The RelayStats.relays() method uses load() which internally calls accept() --- compass.py | 20 ++++++++++---------- 1 files changed, 10 insertions(+), 10 deletions(-) diff --git a/compass.py b/compass.py index 2d8ebc0..d25a3a4 100755 --- a/compass.py +++ b/compass.py @@ -21,7 +21,7 @@ class BaseFilter(object): pass def load(self, relays): - pass + return filter(self.accept, relays) class RunningFilter(BaseFilter): def accept(self, relay): @@ -81,17 +81,18 @@ class SameNetworkFilter(BaseFilter): def __init__(self, relay): self.exit = relay.get('exit_probability') self.fp = relay.get('fingerprint') + self.relay = relay def __init__(self): self.network_data = {} - self.relays_fp = [] + self.relays = [] def load(self, all_relays): for relay in all_relays: or_addresses = relay.get("or_addresses") if len(or_addresses) > 1: print "[WARNING] - %s has more than two OR Addresses - %s" % relay.get("fingerprint"), or_addresses - for ip in relay.get("or_addresses", []): + for ip in or_addresses: ip, port = ip.rsplit(':', 1) # skip if ipv6 if ':' in ip: @@ -116,10 +117,10 @@ class SameNetworkFilter(BaseFilter): self.network_data[network] = [relay_info] for relay_list in self.network_data.values(): - for relay in relay_list: - self.relays_fp.append(relay.fp) - - def accept(self, relay): + self.relays.extend([relay.relay for relay in relay_list]) + return self.relays + + def accept(self, relay): return relay.get('fingerprint') in self.relays_fp class GuardFilter(BaseFilter): @@ -181,9 +182,8 @@ class RelayStats(object): self._relays = {} relays = self.data['relays'] for f in self._filters: - f.load(relays) - relays = filter(f.accept, relays) - + relays = f.load(relays) + for relay in relays: self.add_relay(relay) return self._relays

1 0

[compass/master] Add SameNetworkFilter
by karsten＠torproject.org 24 Aug '12

24 Aug '12

commit 6cd58b7eec4ac73ee00cffc2e598653be1399359 Author: Sathyanarayanan Gunasekaran <gsathya.ceg(a)gmail.com> Date: Mon Aug 20 18:38:30 2012 +0530 Add SameNetworkFilter SameNetworkFilter returns the fastest two relays per /24 --- compass.py | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 47 insertions(+), 0 deletions(-) diff --git a/compass.py b/compass.py index 9d727c8..2d8ebc0 100755 --- a/compass.py +++ b/compass.py @@ -76,6 +76,52 @@ class ExitFilter(BaseFilter): def accept(self, relay): return relay.get('exit_probability', -1) > 0.0 +class SameNetworkFilter(BaseFilter): + class Relay(object): + def __init__(self, relay): + self.exit = relay.get('exit_probability') + self.fp = relay.get('fingerprint') + + def __init__(self): + self.network_data = {} + self.relays_fp = [] + + def load(self, all_relays): + for relay in all_relays: + or_addresses = relay.get("or_addresses") + if len(or_addresses) > 1: + print "[WARNING] - %s has more than two OR Addresses - %s" % relay.get("fingerprint"), or_addresses + for ip in relay.get("or_addresses", []): + ip, port = ip.rsplit(':', 1) + # skip if ipv6 + if ':' in ip: + continue + network = ip.rsplit('.', 1)[0] + relay_info = self.Relay(relay) + if self.network_data.has_key(network): + if len(self.network_data[network]) > 1: + # assume current relay to have smallest exit_probability + min_exit = relay.get('exit_probability') + min_id = -1 + for id, value in enumerate(self.network_data[network]): + if value.exit < min_exit: + min_exit = value.exit + min_id = id + if min_id != -1: + del self.network_data[network][min_id] + self.network_data[network].append(relay_info) + else: + self.network_data[network].append(relay_info) + else: + self.network_data[network] = [relay_info] + + for relay_list in self.network_data.values(): + for relay in relay_list: + self.relays_fp.append(relay.fp) + + def accept(self, relay): + return relay.get('fingerprint') in self.relays_fp + class GuardFilter(BaseFilter): def accept(self, relay): return relay.get('guard_probability', -1) > 0.0 @@ -158,6 +204,7 @@ class RelayStats(object): filters.append(GuardFilter()) if options.fast_exits_only: filters.append(FastExitFilter(95 * 125 * 1024, 5000 * 1024, [80, 443, 554, 1755], False)) + filters.append(SameNetworkFilter()) return filters def _get_group_function(self, options):

1 0

[compass/master] Add --almost-fast-exits-only
by karsten＠torproject.org 24 Aug '12

24 Aug '12

commit 71a216b46a02f4cee37f91f56b8f44b7d05284eb Author: Sathyanarayanan Gunasekaran <gsathya.ceg(a)gmail.com> Date: Tue Aug 21 20:02:30 2012 +0530 Add --almost-fast-exits-only --- compass.py | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diff --git a/compass.py b/compass.py index d25a3a4..3e20c96 100755 --- a/compass.py +++ b/compass.py @@ -205,6 +205,10 @@ class RelayStats(object): if options.fast_exits_only: filters.append(FastExitFilter(95 * 125 * 1024, 5000 * 1024, [80, 443, 554, 1755], False)) filters.append(SameNetworkFilter()) + if options.almost_fast_exits_only: + filters.append(FastExitFilter(80 * 125 * 1024, 2000 * 1024, [80, 443], False)) + filters.append(SameNetworkFilter()) + filters.append(FastExitFilter(95 * 125 * 1024, 5000 * 1024, [80, 443, 554, 1755], True)) return filters def _get_group_function(self, options): @@ -325,6 +329,8 @@ def create_option_parser(): help="select only relays suitable for guard position") group.add_option("-x", "--fast-exits-only", action="store_true", help="select only 100+ Mbit/s exits allowing ports 80, 443, 554, and 1755") + group.add_option("-w", "--almost-fast-exits-only", action="store_true", + help="select only 80+ & 95- Mbit/s exits allowing ports 80, 443, 554, and 1755") parser.add_option_group(group) group = OptionGroup(parser, "Grouping options") group.add_option("-A", "--by-as", action="store_true", default=False,

1 0