- tor-commits - lists.torproject.org

[flashproxy/master] Remove any existing old signature before signing.
by dcf＠torproject.org 23 Jul '12

23 Jul '12

commit 9bbc3351033ea22a0824d60de136b100c99524e5 Author: David Fifield <david(a)bamsoftware.com> Date: Mon Jul 23 08:49:58 2012 -0700 Remove any existing old signature before signing. This avoids the interactive error message File `flashproxy-client-0.3.zip.asc' exists. Overwrite? (y/N) --- Makefile | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/Makefile b/Makefile index 584094c..8c6cbfe 100644 --- a/Makefile +++ b/Makefile @@ -32,6 +32,7 @@ dist/$(DISTNAME).zip: $(CLIENT_DIST_FILES) $(MAKE) dist sign: dist/$(DISTNAME).zip + rm -f dist/$(DISTNAME).zip.asc cd dist && gpg --sign --detach-sign --armor $(DISTNAME).zip cd dist && gpg --verify $(DISTNAME).zip.asc $(DISTNAME).zip

1 0

[metrics-tasks/master] Add graphing code for #6443.
by karsten＠torproject.org 23 Jul '12

23 Jul '12

commit 4e568b12507c575bd3c75f93e7c4aa97df38b4b1 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Jul 23 14:17:42 2012 +0200 Add graphing code for #6443. --- task-6443/README | 7 +++++++ task-6443/cumulated-weights.R | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 0 deletions(-) diff --git a/task-6443/README b/task-6443/README new file mode 100644 index 0000000..5f10bdf --- /dev/null +++ b/task-6443/README @@ -0,0 +1,7 @@ +Graph cdf of probability of selecting among the biggest k exits (#6443) + +- Run #5755 code to produce cumulated-weights.csv and + inverse-cumulated-weights.csv and copy those files to this directory. + +- Plot CDFs using: R --slave -f cumulated-weights.R + diff --git a/task-6443/cumulated-weights.R b/task-6443/cumulated-weights.R new file mode 100755 index 0000000..50d0f4c --- /dev/null +++ b/task-6443/cumulated-weights.R @@ -0,0 +1,35 @@ +require(ggplot2) +require(scales) +require(reshape) + +c <- read.csv("cumulated-weights.csv") +c <- c[c$top_relays %in% c(1, 2, 3, 5, 10, 15, 20, 30, 50), ] +c <- aggregate(list(total_exit_probability = c$total_exit_probability), + by = list(date = as.Date(c$validafter, origin = "1970-01-01 00:00:00"), + top_relays = c$top_relays), FUN = mean) +ggplot(c, aes(x = date, y = total_exit_probability, + colour = as.factor(top_relays))) + +geom_line() + +scale_x_date(name = "") + +scale_y_continuous(name = "Total exit probability of top-x relays\n", + limits = c(0, 1), labels = percent) + +scale_colour_hue(name = "Top-x relays by exit probability") + +opts(title = paste("Probability of selecting one of the top-x relays for", + "the exit position\n"), legend.position = "bottom") +ggsave("exit-probability-cdf.png", width = 8, height = 5, dpi = 100) + +i <- read.csv("inverse-cumulated-weights.csv") +i <- i[i$total_exit_probability > 0.1 & i$total_exit_probability < 0.9, ] +i <- aggregate(list(top_relays = i$top_relays), + by = list(date = as.Date(i$validafter, origin = "1970-01-01 00:00:00"), + total_exit_probability = i$total_exit_probability), FUN = mean) +ggplot(i, aes(x = date, y = top_relays, + colour = paste(100 * total_exit_probability, "%", sep = ""))) + +geom_line() + +scale_x_date(name = "") + +scale_y_continuous(name = "Top-x relays by exit probability\n") + +scale_colour_hue(name = "Total exit probability of top-x relays") + +opts(title = paste("Number of relays making up the top-x for a given", + "total exit probability\n"), legend.position = "bottom") +ggsave("exit-probability-icdf.png", width = 8, height = 5, dpi = 100) +

1 0

[metrics-tasks/master] Output exit probabilities for CDFs (#5755, #6443).
by karsten＠torproject.org 23 Jul '12

23 Jul '12

commit 6450c754687fc1437d406a49a77e69a1443b2e23 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Jul 23 14:00:21 2012 +0200 Output exit probabilities for CDFs (#5755, #6443). We need sorted, cumulated exit probabilities for graphing them in #6443. Instead of writing a new script to extract them, extend #5755 code to output these files, too. --- .../src/CalculatePathSelectionProbabilities.java | 44 ++++++++++++++++++-- 1 files changed, 40 insertions(+), 4 deletions(-) diff --git a/task-5755/src/CalculatePathSelectionProbabilities.java b/task-5755/src/CalculatePathSelectionProbabilities.java index 369a994..de6a889 100644 --- a/task-5755/src/CalculatePathSelectionProbabilities.java +++ b/task-5755/src/CalculatePathSelectionProbabilities.java @@ -2,9 +2,12 @@ import java.io.BufferedWriter; import java.io.File; import java.io.FileWriter; import java.text.SimpleDateFormat; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.SortedMap; import java.util.SortedSet; @@ -88,10 +91,16 @@ public class CalculatePathSelectionProbabilities { descriptorReader = DescriptorSourceFactory.createDescriptorReader(); descriptorReader.addDirectory(new File("in/consensuses")); descriptorFiles = descriptorReader.readDescriptors(); - BufferedWriter bw = new BufferedWriter(new FileWriter("out.csv")); + BufferedWriter bw = new BufferedWriter(new FileWriter("weights.csv")); bw.write("validafter,fingerprint,advertised_bandwidth_fraction," + "consensus_weight_fraction,guard_probability," + "middle_probability,exit_probability\n"); + BufferedWriter bw2 = new BufferedWriter(new FileWriter( + "cumulated-weights.csv")); + bw2.write("validafter,top_relays,total_exit_probability\n"); + BufferedWriter bw3 = new BufferedWriter(new FileWriter( + "inverse-cumulated-weights.csv")); + bw3.write("validafter,total_exit_probability,top_relays\n"); while (descriptorFiles.hasNext()) { DescriptorFile descriptorFile = descriptorFiles.next(); for (Descriptor descriptor : descriptorFile.getDescriptors()) { @@ -158,9 +167,6 @@ public class CalculatePathSelectionProbabilities { toUpperCase(); double advertisedBandwidth; if (!serverDescriptors.containsKey(serverDescriptorDigest)) { - System.err.println("Missing server descriptor " - + serverDescriptorDigest + " while processing consensus " - + validAfter + ". Setting advertised bandwidth to 0."); advertisedBandwidth = 0.0; } else { advertisedBandwidth = (double) serverDescriptors.get( @@ -238,9 +244,39 @@ public class CalculatePathSelectionProbabilities { middleWeights.get(fingerprint) / totalMiddleWeight, exitWeights.get(fingerprint) / totalExitWeight)); } + + /* Write exit probabilities for top-x relays to the second and + * third output files. */ + List<Double> sortedExitWeights = new ArrayList<Double>( + exitWeights.values()); + Collections.sort(sortedExitWeights); + Collections.reverse(sortedExitWeights); + int topRelays = 0; + double totalExitProbability = 0.0; + List<Double> inverseProbabilities = new ArrayList<Double>( + Arrays.asList(new Double[] { 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, + 0.7, 0.8, 0.9 })); + for (double exitWeight : sortedExitWeights) { + topRelays++; + totalExitProbability += exitWeight / totalExitWeight; + if (topRelays <= 50) { + bw2.write(String.format("%s,%d,%.9f%n", validAfter, topRelays, + totalExitProbability)); + } + while (!inverseProbabilities.isEmpty() && + totalExitProbability > inverseProbabilities.get(0)) { + bw3.write(String.format("%s,%.1f,%d%n", validAfter, + inverseProbabilities.remove(0), topRelays)); + } + if (inverseProbabilities.isEmpty() && topRelays > 50) { + break; + } + } } } bw.close(); + bw2.close(); + bw3.close(); } }

1 0

[metrics-tasks/master] Take out --family-for-fp option which is not entirely correct (#6329).
by karsten＠torproject.org 23 Jul '12

23 Jul '12

commit f3e79b74a1af9dbd4adc3fdc4ae5963fe0495c76 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Jul 23 08:52:56 2012 +0200 Take out --family-for-fp option which is not entirely correct (#6329). This option looks for relays having the given fingerprint in their family line, but it doesn't cross-check if the given relay has those relays in its family line, too. The option also doesn't look at nicknames (of relays with the Named flag) in family lines, which means it might miss some family members. --- task-6329/tor-relays-stats.py | 8 +------- 1 files changed, 1 insertions(+), 7 deletions(-) diff --git a/task-6329/tor-relays-stats.py b/task-6329/tor-relays-stats.py index ee2a3dc..245a241 100755 --- a/task-6329/tor-relays-stats.py +++ b/task-6329/tor-relays-stats.py @@ -23,7 +23,7 @@ class RelayStats(object): self._data = json.load(file('details.json')) return self._data - def get_relays(self, countries=[], as_sets=[], exits_only=False, guards_only=False, fprint="", inactive=False): + def get_relays(self, countries=[], as_sets=[], exits_only=False, guards_only=False, inactive=False): relays = [] if countries: countries = [x.lower() for x in countries] @@ -40,8 +40,6 @@ class RelayStats(object): continue if guards_only and not relay.get('guard_probability', -1) > 0.0: continue - if fprint and fprint != relay.get('fingerprint') and fprint not in [ffp.strip('$') for ffp in relay.get('family', [])]: - continue relays.append(relay) return relays @@ -91,7 +89,6 @@ class RelayStats(object): as_name = "*" if by_as_number and not by_country: country = "*" - formatted_group = "%8.4f%% %8.4f%% %8.4f%% %8.4f%% %8.4f%% %-19s %-40s %-4s %-5s %-2s %-9s %s" % ( group_weights[0] * 100.0, group_weights[1] * 100.0, @@ -158,8 +155,6 @@ if '__main__' == __name__: help="select only relays suitable for exit position") group.add_option("-g", "--guards-only", action="store_true", help="select only relays suitable for guard position") - group.add_option("-f", "--family-for-fp", action="store", type="string", metavar="FP", - help="select only relays of a family filtered by fingerprint") parser.add_option_group(group) group = OptionGroup(parser, "Grouping options") group.add_option("-A", "--by-as", action="store_true", default=False, @@ -188,7 +183,6 @@ if '__main__' == __name__: as_sets=options.ases, exits_only=options.exits_only, guards_only=options.guards_only, - fprint=options.family_for_fp, inactive=options.inactive) grouped_relays = stats.group_relays(relays, by_country=options.by_country,

1 0

[metrics-tasks/master] Include inactive relays, filter by fingerprint (in family) and shorter output
by karsten＠torproject.org 23 Jul '12

23 Jul '12

commit 9227ebd4e249fb77d00c02afdaab9eab4288336d Author: Michael Zeltner <m(a)niij.org> Date: Sat Jul 21 15:56:22 2012 +0200 Include inactive relays, filter by fingerprint (in family) and shorter output --- task-6329/tor-relays-stats.py | 28 ++++++++++++++++++++-------- 1 files changed, 20 insertions(+), 8 deletions(-) diff --git a/task-6329/tor-relays-stats.py b/task-6329/tor-relays-stats.py index 8fe142e..ee2a3dc 100755 --- a/task-6329/tor-relays-stats.py +++ b/task-6329/tor-relays-stats.py @@ -23,14 +23,14 @@ class RelayStats(object): self._data = json.load(file('details.json')) return self._data - def get_relays(self, countries=[], as_sets=[], exits_only=False, guards_only=False): + def get_relays(self, countries=[], as_sets=[], exits_only=False, guards_only=False, fprint="", inactive=False): relays = [] if countries: countries = [x.lower() for x in countries] if as_sets: as_sets = [x if not x.isdigit() else "AS" + x for x in as_sets] for relay in self.data['relays']: - if not relay['running']: + if not inactive and inactive == relay['running']: continue if countries and not relay.get('country', ' ') in countries: continue @@ -40,6 +40,8 @@ class RelayStats(object): continue if guards_only and not relay.get('guard_probability', -1) > 0.0: continue + if fprint and fprint != relay.get('fingerprint') and fprint not in [ffp.strip('$') for ffp in relay.get('family', [])]: + continue relays.append(relay) return relays @@ -89,6 +91,7 @@ class RelayStats(object): as_name = "*" if by_as_number and not by_country: country = "*" + formatted_group = "%8.4f%% %8.4f%% %8.4f%% %8.4f%% %8.4f%% %-19s %-40s %-4s %-5s %-2s %-9s %s" % ( group_weights[0] * 100.0, group_weights[1] * 100.0, @@ -102,11 +105,11 @@ class RelayStats(object): sorted_groups.reverse() return sorted_groups - def print_groups(self, sorted_groups, count=10, by_country=False, by_as_number=False): - print " CW adv_bw P_guard P_middle P_exit Nickname Fingerprint Exit Guard CC AS_num AS_name" + def print_groups(self, sorted_groups, count=10, by_country=False, by_as_number=False, short=None): + print " CW adv_bw P_guard P_middle P_exit Nickname Fingerprint Exit Guard CC AS_num AS_name"[:short] for formatted_group, weight in sorted_groups[:count]: - print formatted_group + print formatted_group[:short] if len(sorted_groups) > count: if by_country and by_as_number: type = "countries and ASes" @@ -133,7 +136,7 @@ class RelayStats(object): selection_weights[4] * 100.0) def download_details_file(): - url = urllib.urlopen('https://onionoo.torproject.org/details?type=relay&running=true') + url = urllib.urlopen('https://onionoo.torproject.org/details?type=relay') details_file = open("details.json", 'w') details_file.write(url.read()) url.close() @@ -144,6 +147,8 @@ if '__main__' == __name__: parser.add_option("-d", "--download", action="store_true", help="download details.json from Onionoo service") group = OptionGroup(parser, "Filtering options") + group.add_option("-i", "--inactive", action="store_true", default=False, + help="include relays in selection that aren't currently running") group.add_option("-a", "--as", dest="ases", action="append", help="select only relays from autonomous system number AS", metavar="AS") @@ -153,6 +158,8 @@ if '__main__' == __name__: help="select only relays suitable for exit position") group.add_option("-g", "--guards-only", action="store_true", help="select only relays suitable for guard position") + group.add_option("-f", "--family-for-fp", action="store", type="string", metavar="FP", + help="select only relays of a family filtered by fingerprint") parser.add_option_group(group) group = OptionGroup(parser, "Grouping options") group.add_option("-A", "--by-as", action="store_true", default=False, @@ -163,6 +170,8 @@ if '__main__' == __name__: group = OptionGroup(parser, "Display options") group.add_option("-t", "--top", type="int", default=10, metavar="NUM", help="display only the top results (default: %default)") + group.add_option("-s", "--short", action="store_true", + help="cut the length of the line output at 70 chars") parser.add_option_group(group) (options, args) = parser.parse_args() if len(args) > 0: @@ -178,7 +187,9 @@ if '__main__' == __name__: relays = stats.get_relays(countries=options.country, as_sets=options.ases, exits_only=options.exits_only, - guards_only=options.guards_only) + guards_only=options.guards_only, + fprint=options.family_for_fp, + inactive=options.inactive) grouped_relays = stats.group_relays(relays, by_country=options.by_country, by_as_number=options.by_as) @@ -187,4 +198,5 @@ if '__main__' == __name__: by_as_number=options.by_as) stats.print_groups(sorted_groups, options.top, by_country=options.by_country, - by_as_number=options.by_as) + by_as_number=options.by_as, + short=70 if options.short else None)

1 0

[metrics-tasks/master] Python 2.6.6 does not have dict.viewvalues()
by karsten＠torproject.org 23 Jul '12

23 Jul '12

commit fc5076ad8332c595d063d3159d4d4d0fa68ac9b6 Author: Michael Zeltner <m(a)niij.org> Date: Sat Jul 21 16:04:42 2012 +0200 Python 2.6.6 does not have dict.viewvalues() --- task-6329/tor-relays-stats.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/task-6329/tor-relays-stats.py b/task-6329/tor-relays-stats.py index 09bf98a..8fe142e 100755 --- a/task-6329/tor-relays-stats.py +++ b/task-6329/tor-relays-stats.py @@ -61,7 +61,7 @@ class RelayStats(object): def format_and_sort_groups(self, grouped_relays, by_country=False, by_as_number=False): formatted_groups = {} - for group in grouped_relays.viewvalues(): + for group in grouped_relays.values(): group_weights = (0, 0, 0, 0, 0) relays_in_group = 0 for relay in group:

1 0

[flashproxy/master] Comment that the relay needs to support the websocket transport.
by dcf＠torproject.org 23 Jul '12

23 Jul '12

commit b24b214da5d9ec59269ac40e5a48335e6414e1ed Author: David Fifield <david(a)bamsoftware.com> Date: Sun Jul 22 19:09:48 2012 -0700 Comment that the relay needs to support the websocket transport. --- init.d/facilitator | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/init.d/facilitator b/init.d/facilitator index 4e8ab1d..e7f919b 100755 --- a/init.d/facilitator +++ b/init.d/facilitator @@ -16,8 +16,8 @@ # Source function library. . /etc/rc.d/init.d/functions -# Replace this with the address of a Tor relay. You can use host:port -# syntax. +# Replace this with the address of a Tor relay with a websocket pluggable +# transport. You can use host:port syntax. RELAY=... BINDIR=/usr/local/bin

1 0

[flashproxy/master] Don't let websockify send "Sec-WebSocket-Protocol: binary".
by dcf＠torproject.org 23 Jul '12

23 Jul '12

commit 1dd0da7ef968f914cc99137cf98852d77351a174 Author: David Fifield <david(a)bamsoftware.com> Date: Sun Jul 22 20:53:53 2012 -0700 Don't let websockify send "Sec-WebSocket-Protocol: binary". We patch websockify so that it assumes binary transport when no subprotocol is given. Previously we continued to send a "Sec-WebSocket-Protocol: binary" in our response, which is a violation of section 4.2.2 of RFC 6455: "The value chosen MUST be derived from the client's handshake, specifically by selecting one of the values from the |Sec-WebSocket-Protocol| field that the server is willing to use for this connection (if any). If the client's handshake did not contain such a header field or if the server does not agree to any of the client's requested subprotocols, the only acceptable value is null." Chromium (version 20, I believe) started being more strict about enforcing this, which caused the relay connection to fail. https://bugs.webkit.org/show_bug.cgi?id=82307 http://trac.webkit.org/changeset/112499 --- patches/websockify-binary-default.patch | 32 ++++++++++++++++++++++-------- 1 files changed, 23 insertions(+), 9 deletions(-) diff --git a/patches/websockify-binary-default.patch b/patches/websockify-binary-default.patch index 9ba7f06..bb7a469 100644 --- a/patches/websockify-binary-default.patch +++ b/patches/websockify-binary-default.patch @@ -1,26 +1,40 @@ -From 066c665c2e437a399e2fd0b55394b0ec82c7aeab Mon Sep 17 00:00:00 2001 +From dc3a5f38d9921aa3cd128744ab3be5e6cfb7ff48 Mon Sep 17 00:00:00 2001 From: David Fifield <david(a)bamsoftware.com> -Date: Thu, 19 Apr 2012 14:16:16 -0700 +Date: Sun, 22 Jul 2012 20:52:47 -0700 Subject: [PATCH] Assume binary transfer when no subprotocol is given. --- - websocket.py | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) + websocket.py | 8 ++------ + 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/websocket.py b/websocket.py -index 646160c..d6cea57 100644 +index 96156b9..7a42ff6 100644 --- a/websocket.py +++ b/websocket.py -@@ -660,7 +660,8 @@ Sec-WebSocket-Accept: %s\r - elif 'base64' in protocols: +@@ -585,12 +585,10 @@ Sec-WebSocket-Accept: %s\r + key = h['Sec-WebSocket-Key'] + + # Choose binary if client supports it +- if 'binary' in protocols: +- self.base64 = False +- elif 'base64' in protocols: ++ if 'base64' in protocols: self.base64 = True else: - raise self.EClose("Client must support 'binary' or 'base64' protocol") + self.base64 = False -+ # raise self.EClose("Client must support 'binary' or 'base64' protocol") # Generate the hash value for the accept header accept = b64encode(sha1(s2b(key + self.GUID)).digest()) +@@ -598,8 +596,6 @@ Sec-WebSocket-Accept: %s\r + response = self.server_handshake_hybi % b2s(accept) + if self.base64: + response += "Sec-WebSocket-Protocol: base64\r\n" +- else: +- response += "Sec-WebSocket-Protocol: binary\r\n" + response += "\r\n" + + else: -- -1.7.9.5 +1.7.10.4

1 0

[flashproxy/master] Re-register after a failed WebSocket handshake.
by dcf＠torproject.org 23 Jul '12

23 Jul '12

commit 0b2adc04b24f89ec5a6c572a3f8832fc7d7004aa Author: David Fifield <david(a)bamsoftware.com> Date: Sun Jul 22 19:31:17 2012 -0700 Re-register after a failed WebSocket handshake. --- flashproxy-client | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/flashproxy-client b/flashproxy-client index 1f85f87..ad60057 100755 --- a/flashproxy-client +++ b/flashproxy-client @@ -811,6 +811,7 @@ def main(): unlinked_remotes.append(wrapped) else: fd.close() + register() websocket_pending.remove(fd) report_pending() elif fd in socks_pending:

1 0

[flashproxy/master] Might as well fill in a real RELAY so this works out of the box.
by dcf＠torproject.org 23 Jul '12

23 Jul '12

commit 1d822effb02f8b312e6083453d8e0ba08c2afed7 Author: David Fifield <david(a)bamsoftware.com> Date: Sun Jul 22 19:10:16 2012 -0700 Might as well fill in a real RELAY so this works out of the box. --- init.d/facilitator | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/init.d/facilitator b/init.d/facilitator index e7f919b..5510d11 100755 --- a/init.d/facilitator +++ b/init.d/facilitator @@ -18,7 +18,7 @@ # Replace this with the address of a Tor relay with a websocket pluggable # transport. You can use host:port syntax. -RELAY=... +RELAY=tor1.bamsoftware.com:9901 BINDIR=/usr/local/bin VARDIR=/var/flashproxy

1 0