October 2013 - tor-commits - lists.torproject.org

[flashproxy/master] I think this comment is what we expect.
by infinity0＠torproject.org 28 Oct '13

28 Oct '13

commit be2c0e00cfdf63076e4b1edf90abcbd2ee2f0e5c Author: David Fifield <david(a)bamsoftware.com> Date: Fri Sep 20 22:58:19 2013 -0700 I think this comment is what we expect. --- facilitator/facilitator | 3 --- 1 file changed, 3 deletions(-) diff --git a/facilitator/facilitator b/facilitator/facilitator index 9610993..8e0a032 100755 --- a/facilitator/facilitator +++ b/facilitator/facilitator @@ -79,9 +79,6 @@ def parse_transport_chain(spec): ('obfs3', 'websocket') """ assert(spec) - # XXX We explicitly cast to tuple so that the return value can be - # used as a dictionary key. Transport chains are not supposed to - # be manipulated so it should be cool. return tuple(spec.split("|")) def get_outermost_transport(transports):

1 0

[flashproxy/master] Fix what the global RELAYS looks like.
by infinity0＠torproject.org 28 Oct '13

28 Oct '13

commit df76e0527c45414d84ccb475d96731e90ba14f11 Author: David Fifield <david(a)bamsoftware.com> Date: Fri Sep 20 22:57:01 2013 -0700 Fix what the global RELAYS looks like. --- facilitator/facilitator | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/facilitator/facilitator b/facilitator/facilitator index 986b166..481ccdd 100755 --- a/facilitator/facilitator +++ b/facilitator/facilitator @@ -29,7 +29,7 @@ LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S" # Dictionary containing Tor relay addresses that we can pass to # proxies. It looks like this: -# {"websocket" : ["1.1.1.1:123"], "obfs3|websocket" : ["3.1.2.3:5555", "1.2.3.4:6666"], "obfs2|websocket" : ["1.2.4.4:5123"]} +# {("websocket"): [("1.1.1.1", 123)], ("obfs3", "websocket"): [("3.1.2.3", 5555), ("1.2.3.4", 6666)], ("obfs2", "websocket"): [("1.2.4.4", 5123)]} RELAYS = {} class UnknownTransport(Exception): pass

1 0

[flashproxy/master] Have relay file parser not modify global state.
by infinity0＠torproject.org 28 Oct '13

28 Oct '13

commit 5ebc99f1e00b37aa70975336287dc79de598bb85 Author: David Fifield <david(a)bamsoftware.com> Date: Fri Sep 20 22:59:56 2013 -0700 Have relay file parser not modify global state. --- facilitator/facilitator | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/facilitator/facilitator b/facilitator/facilitator index 8e0a032..0d8e8bc 100755 --- a/facilitator/facilitator +++ b/facilitator/facilitator @@ -44,11 +44,6 @@ class options(object): privdrop_username = None safe_logging = True - @staticmethod - def add_relay(addr, transport): - RELAYS.setdefault(transport, []) - RELAYS[transport].append(addr) - def usage(f = sys.stdout): print >> f, """\ Usage: %(progname)s -r RELAY <OPTIONS> @@ -443,13 +438,14 @@ def put_reg(reg): REGS = get_regs(af, get_outermost_transport(reg.transports)) return REGS.add(reg) -def handle_relay_file(fname): +def parse_relay_file(fname): """Parse a file containing Tor relays that we can point proxies to. Throws ValueError if the file is not properly formatted.""" # File format is: # <transport> <addrport> # Example: # obfs2|websocket 1.4.6.1:4123 + relays = {} with open(fname) as input: for line in input: words_list = line.split() @@ -458,7 +454,9 @@ def handle_relay_file(fname): addr = fac.parse_addr_spec(words_list[1], defport = DEFAULT_RELAY_PORT, resolve = True) transports = parse_transport_chain(words_list[0]) - options.add_relay(addr, transports) + relays.setdefault(transports, []) + relays[transports].append(addr) + return relays def main(): opts, args = getopt.gnu_getopt(sys.argv[1:], "dhl:p:r:", @@ -493,7 +491,7 @@ obfs2|websocket 1.4.6.1:4123\ """ sys.exit(1) - handle_relay_file(options.relay_filename) + RELAYS.update(parse_relay_file(options.relay_filename)) if not RELAYS: print >> sys.stderr, u"Warning: no relays configured."

1 0

[flashproxy/master] May as well add the plain websocket relay too.
by infinity0＠torproject.org 28 Oct '13

28 Oct '13

commit 12ee89e53632ac9e4b87d70fd34cc0ae656cf968 Author: David Fifield <david(a)bamsoftware.com> Date: Fri Sep 20 23:11:36 2013 -0700 May as well add the plain websocket relay too. --- facilitator/relays | 1 + 1 file changed, 1 insertion(+) diff --git a/facilitator/relays b/facilitator/relays index 8fa0320..82a8f71 100644 --- a/facilitator/relays +++ b/facilitator/relays @@ -1 +1,2 @@ obfs3|websocket 173.255.221.44:9500 +websocket 173.255.221.44:9901

1 0

[flashproxy/master] Style in parse_relay_file.
by infinity0＠torproject.org 28 Oct '13

28 Oct '13

commit b1fad263ebb4db322237124cc5e834cc138eb9c4 Author: David Fifield <david(a)bamsoftware.com> Date: Fri Sep 20 23:05:31 2013 -0700 Style in parse_relay_file. --- facilitator/facilitator | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/facilitator/facilitator b/facilitator/facilitator index 0d8e8bc..cf99141 100755 --- a/facilitator/facilitator +++ b/facilitator/facilitator @@ -438,22 +438,21 @@ def put_reg(reg): REGS = get_regs(af, get_outermost_transport(reg.transports)) return REGS.add(reg) -def parse_relay_file(fname): +def parse_relay_file(filename): """Parse a file containing Tor relays that we can point proxies to. - Throws ValueError if the file is not properly formatted.""" - # File format is: - # <transport> <addrport> - # Example: - # obfs2|websocket 1.4.6.1:4123 + Throws ValueError on a parsing error. Each line contains a transport chain + and an address, for example + obfs2|websocket 1.4.6.1:4123 + """ relays = {} - with open(fname) as input: - for line in input: - words_list = line.split() - if len(words_list) != 2: - raise ValueError("Wrong line format: %s" % line) - - addr = fac.parse_addr_spec(words_list[1], defport = DEFAULT_RELAY_PORT, resolve = True) - transports = parse_transport_chain(words_list[0]) + with open(filename) as f: + for line in f: + try: + transport_spec, addr_spec = line.strip().split() + except ValueError, e: + raise ValueError("Wrong line format: %s." % repr(line)) + addr = fac.parse_addr_spec(addr_spec, defport=DEFAULT_RELAY_PORT, resolve=True) + transports = parse_transport_chain(transport_spec) relays.setdefault(transports, []) relays[transports].append(addr) return relays

1 0

[flashproxy/master] make tests work against the new interfaces
by infinity0＠torproject.org 28 Oct '13

28 Oct '13

commit a3a6b832c9d3db2b24251f09741ac34b92eafcd6 Author: Ximin Luo <infinity0(a)gmx.com> Date: Mon Oct 7 12:14:26 2013 +0100 make tests work against the new interfaces --- facilitator/facilitator-test | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/facilitator/facilitator-test b/facilitator/facilitator-test index cbd45f8..bd4894e 100755 --- a/facilitator/facilitator-test +++ b/facilitator/facilitator-test @@ -1,15 +1,19 @@ #!/usr/bin/env python +from cStringIO import StringIO import socket import subprocess +import tempfile import time import unittest +from facilitator import parse_transport_chain import fac FACILITATOR_HOST = "127.0.0.1" -FACILITATOR_PORT = 9002 +FACILITATOR_PORT = 39002 # diff port to not conflict with production service FACILITATOR_ADDR = (FACILITATOR_HOST, FACILITATOR_PORT) +WS_TRANSPORT = ("websocket",) def gimme_socket(host, port): addrinfo = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM, socket.IPPROTO_TCP)[0] @@ -19,6 +23,12 @@ def gimme_socket(host, port): return s class FacilitatorTest(unittest.TestCase): + + def test_parse_transport_chain(self): + self.assertEquals(parse_transport_chain("a"), ("a",)) + self.assertEquals(parse_transport_chain("a|b|c"), ("a","b","c")) + +class FacilitatorProcTest(unittest.TestCase): IPV4_CLIENT_ADDR = ("1.1.1.1", 9000) IPV6_CLIENT_ADDR = ("[11::11]", 9000) IPV4_PROXY_ADDR = ("2.2.2.2", 13000) @@ -28,7 +38,12 @@ class FacilitatorTest(unittest.TestCase): return gimme_socket(FACILITATOR_HOST, FACILITATOR_PORT) def setUp(self): - self.process = subprocess.Popen(["./facilitator", "-d", "-p", str(FACILITATOR_PORT), "-r", "0.0.1.0:1", "-l", "/dev/null"]) + self.relay_file = tempfile.NamedTemporaryFile() + self.relay_file.write("websocket 0.0.1.0:1\n") + self.relay_file.write("websocket [0:0::1:0]:1\n") + self.relay_file.flush() + self.relay_file.seek(0) + self.process = subprocess.Popen(["./facilitator", "-p", str(FACILITATOR_PORT), "-r", self.relay_file.name, "-d", "--unsafe-logging"]) time.sleep(0.1) def tearDown(self): @@ -66,34 +81,34 @@ class FacilitatorTest(unittest.TestCase): def test_af_v4_v4(self): """Test that IPv4 proxies can get IPv4 clients.""" - fac.put_reg(FACILITATOR_ADDR, self.IPV4_CLIENT_ADDR) - fac.put_reg(FACILITATOR_ADDR, self.IPV6_CLIENT_ADDR) - reg = fac.get_reg(FACILITATOR_ADDR, self.IPV4_PROXY_ADDR) + fac.put_reg(FACILITATOR_ADDR, self.IPV4_CLIENT_ADDR, WS_TRANSPORT) + fac.put_reg(FACILITATOR_ADDR, self.IPV6_CLIENT_ADDR, WS_TRANSPORT) + reg = fac.get_reg(FACILITATOR_ADDR, self.IPV4_PROXY_ADDR, WS_TRANSPORT) self.assertEqual(reg["client"], fac.format_addr(self.IPV4_CLIENT_ADDR)) def test_af_v4_v6(self): """Test that IPv4 proxies do not get IPv6 clients.""" - fac.put_reg(FACILITATOR_ADDR, self.IPV6_CLIENT_ADDR) - reg = fac.get_reg(FACILITATOR_ADDR, self.IPV4_PROXY_ADDR) + fac.put_reg(FACILITATOR_ADDR, self.IPV6_CLIENT_ADDR, WS_TRANSPORT) + reg = fac.get_reg(FACILITATOR_ADDR, self.IPV4_PROXY_ADDR, WS_TRANSPORT) self.assertEqual(reg["client"], "") def test_af_v6_v4(self): """Test that IPv6 proxies do not get IPv4 clients.""" - fac.put_reg(FACILITATOR_ADDR, self.IPV4_CLIENT_ADDR) - reg = fac.get_reg(FACILITATOR_ADDR, self.IPV6_PROXY_ADDR) + fac.put_reg(FACILITATOR_ADDR, self.IPV4_CLIENT_ADDR, WS_TRANSPORT) + reg = fac.get_reg(FACILITATOR_ADDR, self.IPV6_PROXY_ADDR, WS_TRANSPORT) self.assertEqual(reg["client"], "") def test_af_v6_v6(self): """Test that IPv6 proxies can get IPv6 clients.""" - fac.put_reg(FACILITATOR_ADDR, self.IPV4_CLIENT_ADDR) - fac.put_reg(FACILITATOR_ADDR, self.IPV6_CLIENT_ADDR) - reg = fac.get_reg(FACILITATOR_ADDR, self.IPV6_PROXY_ADDR) + fac.put_reg(FACILITATOR_ADDR, self.IPV4_CLIENT_ADDR, WS_TRANSPORT) + fac.put_reg(FACILITATOR_ADDR, self.IPV6_CLIENT_ADDR, WS_TRANSPORT) + reg = fac.get_reg(FACILITATOR_ADDR, self.IPV6_PROXY_ADDR, WS_TRANSPORT) self.assertEqual(reg["client"], fac.format_addr(self.IPV6_CLIENT_ADDR)) def test_check_back_in(self): """Test that facilitator responses contain a CHECK-BACK-IN key with a numeric value.""" - reg = fac.get_reg(FACILITATOR_ADDR, self.IPV6_PROXY_ADDR) + reg = fac.get_reg(FACILITATOR_ADDR, self.IPV6_PROXY_ADDR, WS_TRANSPORT) self.assertGreater(int(reg["check-back-in"]), 0) # def test_same_proxy(self):

1 0

[flashproxy/master] use with-syntax for locks
by infinity0＠torproject.org 28 Oct '13

28 Oct '13

commit 7719078521bacddc4c33ccc7e7b0dcd92492ca27 Author: Ximin Luo <infinity0(a)gmx.com> Date: Mon Oct 7 12:08:45 2013 +0100 use with-syntax for locks --- facilitator/facilitator | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/facilitator/facilitator b/facilitator/facilitator index cf99141..647e1ed 100755 --- a/facilitator/facilitator +++ b/facilitator/facilitator @@ -93,12 +93,9 @@ def safe_str(s): log_lock = threading.Lock() def log(msg): - log_lock.acquire() - try: + with log_lock: print >> options.log_file, (u"%s %s" % (time.strftime(LOG_DATE_FORMAT), msg)).encode("UTF-8") options.log_file.flush() - finally: - log_lock.release() class TCPReg(object): def __init__(self, host, port, transports):

1 0

[metrics-web/master] Retire old user number estimates.
by karsten＠torproject.org 28 Oct '13

28 Oct '13

commit b9ce7127ccb722bfe2a368450ba4569d68cd11e3 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Oct 28 15:44:37 2013 +0100 Retire old user number estimates. --- web/WEB-INF/users.jsp | 407 +++++++++++++++++-------------------------------- 1 file changed, 143 insertions(+), 264 deletions(-) diff --git a/web/WEB-INF/users.jsp b/web/WEB-INF/users.jsp index 06d7f4a..788b3a8 100644 --- a/web/WEB-INF/users.jsp +++ b/web/WEB-INF/users.jsp @@ -16,238 +16,11 @@ <h2>Tor Metrics Portal: Users</h2> <br> -<a name="direct-users"></a> -<h3><a href="#direct-users" class="anchor">Directly connecting Tor -users</a></h3> -<br> -<p>After being connected to the Tor network, users need to refresh their -list of running relays on a regular basis. They send their requests to one -out of a few hundred directory mirrors to save bandwidth of the directory -authorities. The following graphs show an estimate of recurring Tor users -based on the requests seen by a few dozen directory mirrors.</p> -<p><b>Daily directly connecting users:</b></p> -<img src="direct-users.png${direct_users_url}" - width="576" height="360" alt="Direct users graph"> -<form action="users.html#direct-users"> - <div class="formrow"> - <input type="hidden" name="graph" value="direct-users"> - <p> - <label>Start date (yyyy-mm-dd):</label> - <input type="text" name="start" size="10" - value="<c:choose><c:when test="${fn:length(direct_users_start) == 0}">${default_start_date}</c:when><c:otherwise>${direct_users_start[0]}</c:otherwise></c:choose>"> - <label>End date (yyyy-mm-dd):</label> - <input type="text" name="end" size="10" - value="<c:choose><c:when test="${fn:length(direct_users_end) == 0}">${default_end_date}</c:when><c:otherwise>${direct_users_end[0]}</c:otherwise></c:choose>"> - </p><p> - Source: <select name="country"> - <option value="all"<c:if test="${direct_users_country[0] eq 'all'}"> selected</c:if>>All users</option> - <c:forEach var="country" items="${countries}" > - <option value="${country[0]}"<c:if test="${direct_users_country[0] eq country[0]}"> selected</c:if>>${country[1]}</option> - </c:forEach> - </select> - </p><p> - Show possible censorship events if available (<a - href="http://research.torproject.org/techreports/detector-2011-09-09.pdf">BETA</a>) - <select name="events"> - <option value="off">Off</option> - <option value="on"<c:if test="${direct_users_events[0] eq 'on'}"> selected</c:if>>On: both points and expected range</option> - <option value="points"<c:if test="${direct_users_events[0] eq 'points'}"> selected</c:if>>On: points only, no expected range</option> - </select> - </p><p> - <input class="submit" type="submit" value="Update graph"> - </p> - </div> -</form> -<p>Download graph as -<a href="direct-users.pdf${direct_users_url}">PDF</a> or -<a href="direct-users.svg${direct_users_url}">SVG</a>.</p> -<hr> -<a name="direct-users-table"></a> -<p><b>Top-10 countries by directly connecting users:</b></p> -<form action="users.html#direct-users-table"> - <div class="formrow"> - <input type="hidden" name="table" value="direct-users"> - <p> - <label>Start date (yyyy-mm-dd):</label> - <input type="text" name="start" size="10" - value="<c:choose><c:when test="${fn:length(direct_users_start) == 0}">${default_start_date}</c:when><c:otherwise>${direct_users_start[0]}</c:otherwise></c:choose>"> - <label>End date (yyyy-mm-dd):</label> - <input type="text" name="end" size="10" - value="<c:choose><c:when test="${fn:length(direct_users_end) == 0}">${default_end_date}</c:when><c:otherwise>${direct_users_end[0]}</c:otherwise></c:choose>"> - </p><p> - <input class="submit" type="submit" value="Update table"> - </p> - </div> -</form> -<br> -<table> - <tr> - <th>Country</th> - <th>Mean daily users</th> - </tr> - <c:forEach var="row" items="${direct_users_tabledata}"> - <tr> - <td><a href="users.html?graph=direct-users&country=${row['cc']}#direct-users">${row['country']}</a>&emsp;</td> - <td>${row['abs']} (<fmt:formatNumber type="number" minFractionDigits="2" value="${row['rel']}" /> %)</td> - </tr> - </c:forEach> -</table> -<hr> -<a name="censorship-events"></a> -<p><b>Top-10 countries by possible censorship events (<a - href="http://research.torproject.org/techreports/detector-2011-09-09.pdf">BETA</a>):</b></p> -<form action="users.html#censorship-events"> - <div class="formrow"> - <input type="hidden" name="table" value="censorship-events"> - <p> - <label>Start date (yyyy-mm-dd):</label> - <input type="text" name="start" size="10" - value="<c:choose><c:when test="${fn:length(censorship_events_start) == 0}">${default_start_date}</c:when><c:otherwise>${censorship_events_start[0]}</c:otherwise></c:choose>"> - <label>End date (yyyy-mm-dd):</label> - <input type="text" name="end" size="10" - value="<c:choose><c:when test="${fn:length(censorship_events_end) == 0}">${default_end_date}</c:when><c:otherwise>${censorship_events_end[0]}</c:otherwise></c:choose>"> - </p><p> - <input class="submit" type="submit" value="Update table"> - </p> - </div> -</form> -<br> -<table> - <tr> - <th>Country</th> - <th>Downturns</th> - <th>Upturns</th> - </tr> - <c:forEach var="row" items="${censorship_events_tabledata}"> - <tr> - <td><a href="users.html?graph=direct-users&country=${row['cc']}&events=on#direct-users">${row['country']}</a>&emsp;</td> - <td>${row['downturns']}</td> - <td>${row['upturns']}</td> - </tr> - </c:forEach> -</table> -<hr> -<p><a href="csv/direct-users.csv">CSV</a> file containing daily directly -connecting users by country.</p> -<p><a href="csv/monthly-users-peak.csv">CSV</a> file containing peak daily -Tor users (direct and bridge) per month by country.</p> -<p><a href="csv/monthly-users-average.csv">CSV</a> file containing average -daily Tor users (direct and bridge) per month by country.</p> -<br> - -<a name="bridge-users"></a> -<h3><a href="#bridge-users" class="anchor">Tor users via bridges</a></h3> -<br> -<p>Users who cannot connect directly to the Tor network instead connect -via bridges, which are non-public relays. The following graphs display an -estimate of Tor users via bridges based on the unique IP addresses as seen -by a few hundred bridges.</p> -<img src="bridge-users.png${bridge_users_url}" - width="576" height="360" alt="Bridge users graph"> -<form action="users.html#bridge-users"> - <div class="formrow"> - <input type="hidden" name="graph" value="bridge-users"> - <p> - <label>Start date (yyyy-mm-dd):</label> - <input type="text" name="start" size="10" - value="<c:choose><c:when test="${fn:length(bridge_users_start) == 0}">${default_start_date}</c:when><c:otherwise>${bridge_users_start[0]}</c:otherwise></c:choose>"> - <label>End date (yyyy-mm-dd):</label> - <input type="text" name="end" size="10" - value="<c:choose><c:when test="${fn:length(bridge_users_end) == 0}">${default_end_date}</c:when><c:otherwise>${bridge_users_end[0]}</c:otherwise></c:choose>"> - </p><p> - Source: <select name="country"> - <option value="all"<c:if test="${bridge_users_country[0] eq 'all'}"> selected</c:if>>All users</option> - <c:forEach var="country" items="${countries}" > - <option value="${country[0]}"<c:if test="${bridge_users_country[0] eq country[0]}"> selected</c:if>>${country[1]}</option> - </c:forEach> - </select> - </p><p> - <input class="submit" type="submit" value="Update graph"> - </p> - </div> -</form> -<p>Download graph as -<a href="bridge-users.pdf${bridge_users_url}">PDF</a> or -<a href="bridge-users.svg${bridge_users_url}">SVG</a>.</p> -<hr> -<a name="bridge-users-table"></a> -<p><b>Top-10 countries by bridge users:</b></p> -<form action="users.html#bridge-users-table"> - <div class="formrow"> - <input type="hidden" name="table" value="bridge-users"> - <p> - <label>Start date (yyyy-mm-dd):</label> - <input type="text" name="start" size="10" - value="<c:choose><c:when test="${fn:length(bridge_users_start) == 0}">${default_start_date}</c:when><c:otherwise>${bridge_users_start[0]}</c:otherwise></c:choose>"> - <label>End date (yyyy-mm-dd):</label> - <input type="text" name="end" size="10" - value="<c:choose><c:when test="${fn:length(bridge_users_end) == 0}">${default_end_date}</c:when><c:otherwise>${bridge_users_end[0]}</c:otherwise></c:choose>"> - </p><p> - <input class="submit" type="submit" value="Update table"> - </p> - </div> -</form> -<br> -<table> - <tr> - <th>Country</th> - <th>Mean daily users</th> - </tr> - <c:forEach var="row" items="${bridge_users_tabledata}"> - <tr> - <td><a href="users.html?graph=bridge-users&country=${row['cc']}#bridge-users">${row['country']}</a>&emsp;</td> - <td>${row['abs']} (<fmt:formatNumber type="number" minFractionDigits="2" value="${row['rel']}" /> %)</td> - </tr> - </c:forEach> -</table> -<hr> -<p><a href="csv/bridge-users.csv">CSV</a> file containing all data.</p> -<p><a href="csv/monthly-users-peak.csv">CSV</a> file containing peak daily -Tor users (direct and bridge) per month by country.</p> -<p><a href="csv/monthly-users-average.csv">CSV</a> file containing average -daily Tor users (direct and bridge) per month by country.</p> -<br> - -<hr> -<hr> - -<a name="userstats"></a> -<h3><a href="#userstats" class="anchor">New approach to estimating daily -Tor users (BETA)</a></h3> -<br> -<p>As of April 2013, we are experimenting with a new approach to estimating -daily Tor users. -The new approach works very similar to the existing approach to estimate -directly connecting users, but can also be applied to bridge users. -This new approach can break down user numbers by country, pluggable -transport, and IP version. -See the tech report on -<a href="https://research.torproject.org/techreports/counting-daily-bridge-users-201…">counting daily bridge users</a> -and the -<a href="https://gitweb.torproject.org/metrics-tasks.git/tree/HEAD:/task-8462">source code</a> -for details. - <a name="userstats-relay-country"></a> -<p><b>Direct users by country (BETA):</b></p> - -<font color="red"> -<p>This graph is quite similar to the graphs above, -except for the following differences:</p> -<ul> -<li>In contrast to the graphs above, this graph is based on -requests to directory mirrors <i>and</i> directory authorities. -The idea is that we want to estimate both new and recurring users. -That is why the numbers here are higher.</li> -<li>This graph uses byte histories for written <i>directory bytes</i> -rather than general byte history to weight what fraction of directory -requests a relay has answered in the network.</li> -<li>The implementation behind this graph is much more efficient, which -reduces time to graph from about 3 days to about 1 day.</li> -</ul> -</font> +<p><b>Direct users by country:</b></p> <img src="userstats-relay-country.png${userstats_relay_country_url}" - width="576" height="360" alt="Direct users by country graph (BETA)"> + width="576" height="360" alt="Direct users by country graph"> <form action="users.html#userstats-relay-country"> <div class="formrow"> <input type="hidden" name="graph" value="userstats-relay-country"> @@ -283,7 +56,7 @@ reduces time to graph from about 3 days to about 1 day.</li> <a href="userstats-relay-country.svg${userstats_relay_country_url}">SVG</a>.</p> <hr> <a name="userstats-relay-table"></a> -<p><b>Top-10 countries by directly connecting users (BETA):</b></p> +<p><b>Top-10 countries by directly connecting users:</b></p> <form action="users.html#userstats-relay-table"> <div class="formrow"> <input type="hidden" name="table" value="userstats-relay"> @@ -349,16 +122,10 @@ reduces time to graph from about 3 days to about 1 day.</li> <hr> <a name="userstats-bridge-country"></a> -<p><b>Bridge users by country (BETA):</b></p> - -<p> -<font color="red">In contrast to the bridge-user graph above, this graph -uses directory requests to estimate user numbers, not unique IP address sets. -It's yet to be decided which approach is more correct.</font> -</p> +<p><b>Bridge users by country:</b></p> <img src="userstats-bridge-country.png${userstats_bridge_country_url}" - width="576" height="360" alt="Bridge users by country graph (BETA)"> + width="576" height="360" alt="Bridge users by country graph"> <form action="users.html#userstats-bridge-country"> <div class="formrow"> <input type="hidden" name="graph" value="userstats-bridge-country"> @@ -386,7 +153,7 @@ It's yet to be decided which approach is more correct.</font> <a href="userstats-bridge-country.svg${userstats_bridge_country_url}">SVG</a>.</p> <hr> <a name="userstats-bridge-table"></a> -<p><b>Top-10 countries by bridge users (BETA):</b></p> +<p><b>Top-10 countries by bridge users:</b></p> <form action="users.html#userstats-bridge-table"> <div class="formrow"> <input type="hidden" name="table" value="userstats-bridge"> @@ -418,19 +185,10 @@ It's yet to be decided which approach is more correct.</font> <hr> <a name="userstats-bridge-transport"></a> -<p><b>Bridge users by transport (BETA):</b></p> - -<p> -<font color="red">Almost none of the currently running bridges report the -transport name of connecting users, which is why non-OR transport usage is -so low. -By default, we consider all users of a bridge OR transport users, unless told -otherwise. -Non-OR transport numbers will become more accurate over time.</font> -</p> +<p><b>Bridge users by transport:</b></p> <img src="userstats-bridge-transport.png${userstats_bridge_transport_url}" - width="576" height="360" alt="Bridge users by transport graph (BETA)"> + width="576" height="360" alt="Bridge users by transport graph"> <form action="users.html#userstats-bridge-transport"> <div class="formrow"> <input type="hidden" name="graph" value="userstats-bridge-transport"> @@ -460,18 +218,10 @@ Non-OR transport numbers will become more accurate over time.</font> <hr> <a name="userstats-bridge-version"></a> -<p><b>Bridge users by IP version (BETA):</b></p> - -<p> -<font color="red">Not all of the currently running bridges report the -IP version of connecting users. -By default, we consider all users of a bridge IPv4 users, unless told -otherwise. -IPv6 numbers will become more accurate over time.</font> -</p> +<p><b>Bridge users by IP version:</b></p> <img src="userstats-bridge-version.png${userstats_bridge_version_url}" - width="576" height="360" alt="Bridge users by IP version graph (BETA)"> + width="576" height="360" alt="Bridge users by IP version graph"> <form action="users.html#userstats-bridge-version"> <div class="formrow"> <input type="hidden" name="graph" value="userstats-bridge-version"> @@ -498,14 +248,143 @@ IPv6 numbers will become more accurate over time.</font> <hr> <p><a href="csv/userstats.csv">CSV</a> file containing new user -estimates (BETA).</p> +estimates.</p> <p><a href="csv/monthly-userstats-peak.csv">CSV</a> file containing peak -daily Tor users (direct and bridge) per month by country (BETA).</p> +daily Tor users (direct and bridge) per month by country.</p> <p><a href="csv/monthly-userstats-average.csv">CSV</a> file containing -average daily Tor users (direct and bridge) per month by country -(BETA).</p> +average daily Tor users (direct and bridge) per month by country.</p> <br> +<hr> +<a name="questions-and-answers"></a> +<p><b>Questions and answers</b></p> +<p> +Q: How is it even possible to count users in an anonymity network?<br/> +A: We actually don't count users, but we count requests to the directories +that clients make periodically to update their list of relays and estimate +user numbers indirectly from there. +</p> +<p> +Q: Do all directories report these directory request numbers?<br/> +A: No, but we can see what fraction of directories reported them, and then +we can extrapolate the total number in the network. +</p> + +<p> +Q: How do you get from these directory requests to user numbers?<br/> +A: We put in the assumption that the average client makes 10 such requests +per day. A tor client that is connected 24/7 makes about 15 requests per +day, but not all clients are connected 24/7, so we picked the number 10 +for the average client. We simply divide directory requests by 10 and +consider the result as the number of users. +</p> + +<p> +Q: So, are these distinct users per day, average number of users connected +over the day, or what?<br/> +A: Average number of users connected over the day. We can't say how many +distinct users there are. +</p> + +<p> +Q: Are these tor clients or users? What if there's more than one user +behind a tor client?<br/> +A: Then we count those users as one. We really count clients, but it's +more intuitive for most people to think of users, that's why we say users +and not clients. +</p> + +<p> +Q: What if a user runs tor on a laptop and changes their IP address a few +times per day? Don't you overcount that user?<br/> +A: No, because that user updates their list of relays as often as a user +that doesn't change IP address over the day. +</p> + +<p> +Q: How do you know which countries users come from?<br/> +A: The directories resolve IP addresses to country codes and report these +numbers in aggregate form. This is one of the reasons why tor ships with +a GeoIP database. +</p> + +<p> +Q: Why are there so few bridge users that are not using the default OR +protocol or that are using IPv6?<br/> +A: Very few bridges report data on transports or IP versions yet, and by +default we consider requests to use the default OR protocol and IPv4. +Once more bridges report these data, the numbers will become more +accurate. +</p> + +<p> +Q: Why do the graphs end 2 days in the past and not today?<br/> +A: Relays and bridges report some of the data in 24-hour intervals which +may end at any time of the day. And after such an interval is over relays +and bridges might take another 18 hours to report the data. We cut off +the last two days from the graphs, because we want to avoid that the last +data point in a graph indicates a recent trend change which is in fact +just an artifact of the algorithm. +</p> + +<p> +Q: But I noticed that the last data point went up/down a bit since I last +looked a few hours ago. Why is that?<br/> +A: You're an excellent observer! The reason is that we publish user +numbers once we're confident enough that they won't change significantly +anymore. But it's always possible that a directory reports data a few +hours after we were confident enough, but which then slightly changed the +graph. +</p> + +<p> +Q: Why are no numbers available before September 2011?<br/> +A: We do have descriptor archives from before that time, but those +descriptors didn't contain all the data we use to estimate user numbers. +We do have older user numbers from an earlier estimation approach here +(add link), but we believe the current approach is more accurate. +</p> + +<p> +Q: Why do you believe the current approach to estimate user numbers is +more accurate?<br/> +A: For direct users, we include all directories which we didn't do in the +old approach. We also use histories that only contain bytes written to +answer directory requests, which is more precise than using general byte +histories. +</p> + +<p> +Q: And what about the advantage of the current approach over the old one +when it comes to bridge users?<br/> +A: Oh, that's a whole different story. We wrote a 13 page long +<a href="https://research.torproject.org/techreports/counting-daily-bridge-users-201…">technical +report</a> explaining the reasons for retiring the old approach. But the +old data is still <a href="/data/old-user-number-estimates.tar.gz">available</a>. +tl;dr: in the old approach we measured the wrong thing, and now we measure +the right thing. +</p> + +<p> +Q: Are the data and the source code for estimating these user numbers +available?<br/> +A: Sure, <a href="/data.html">data</a> and +<a href="https://gitweb.torproject.org/metrics-tasks.git/tree/HEAD:/task-8462">source +code</a> are publicly available. +</p> + +<p> +Q: What are these red and blue dots indicating possible censorship +events?<br/> +A: We run an anomaly-based censorship-detection system that looks at +estimated user numbers over a series of days and predicts the user number +in the next days. If the actual number is higher or lower, this might +indicate a possible censorship event or release of censorship. For more +details, see our +<a href="https://research.torproject.org/techreports/detector-2011-09-09.pdf">technical +report</a>. +</p> + </div> </div> <div class="bottom" id="bottom">

1 0

[metrics-tasks/master] Add graphing code for old user numbers (#10041).
by karsten＠torproject.org 28 Oct '13

28 Oct '13

commit 1fa116045b42db527729c2e254b3d2508cff4822 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Oct 28 15:10:39 2013 +0100 Add graphing code for old user numbers (#10041). --- task-10041/.gitignore | 3 + task-10041/LICENSE | 37 +++++ task-10041/README | 20 +++ task-10041/plot-users.R | 368 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 428 insertions(+) diff --git a/task-10041/.gitignore b/task-10041/.gitignore new file mode 100644 index 0000000..1f86c56 --- /dev/null +++ b/task-10041/.gitignore @@ -0,0 +1,3 @@ +Rplots.pdf +*.csv.gz + diff --git a/task-10041/LICENSE b/task-10041/LICENSE new file mode 100644 index 0000000..d5362a0 --- /dev/null +++ b/task-10041/LICENSE @@ -0,0 +1,37 @@ +Copyright 2013 The Tor Project + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + +* Neither the names of the copyright owners nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Included data is freely available under a CC0 no copyright declaration: To +the extent possible under law, the Tor Project has waived all copyright +and related or neighboring rights in the data. + +Graphs produced with this code are licensed under a Creative Commons +Attribution 3.0 United States License. + diff --git a/task-10041/README b/task-10041/README new file mode 100644 index 0000000..007aa3d --- /dev/null +++ b/task-10041/README @@ -0,0 +1,20 @@ +Usage: + + Rscript plot-users.R direct [start] [end] [country] [events] [file] + +or + + Rscript plot-users.R bridge [start] [end] [country] [file] + + start: start date YYYY-MM-DD + end: end date YYYY-MM-DD + country: two-letter country code, or "all" + events: censorship events ("on" or "points"), or "off" + file: output filename, typically .png + +Examples: + + Rscript plot-users.R direct 2013-07-01 2013-09-30 us on ex1.png + + Rscript plot-users.R bridge 2013-01-01 2013-03-31 all ex2.png + diff --git a/task-10041/plot-users.R b/task-10041/plot-users.R new file mode 100644 index 0000000..f07985f --- /dev/null +++ b/task-10041/plot-users.R @@ -0,0 +1,368 @@ +# Copyright 2013 The Tor Project +# See LICENSE for licensing information + +args <- commandArgs(TRUE) +if (length(args) < 5) { + print("Not enough arguments. See README for usage instructions and examples.") + print(" Rscript plot-users.R direct 2013-07-01 2013-09-30 us on 1.png") + print(" Rscript plot-users.R bridge 2013-01-01 2013-03-31 all 2.png") + quit() +} + +require(ggplot2, quietly = TRUE) +require(reshape, quietly = TRUE, warn.conflicts = FALSE) +options(scipen = 15) + +countrylist <- list( + "ad" = "Andorra", + "ae" = "the United Arab Emirates", + "af" = "Afghanistan", + "ag" = "Antigua and Barbuda", + "ai" = "Anguilla", + "al" = "Albania", + "am" = "Armenia", + "an" = "the Netherlands Antilles", + "ao" = "Angola", + "aq" = "Antarctica", + "ar" = "Argentina", + "as" = "American Samoa", + "at" = "Austria", + "au" = "Australia", + "aw" = "Aruba", + "ax" = "the Aland Islands", + "az" = "Azerbaijan", + "ba" = "Bosnia and Herzegovina", + "bb" = "Barbados", + "bd" = "Bangladesh", + "be" = "Belgium", + "bf" = "Burkina Faso", + "bg" = "Bulgaria", + "bh" = "Bahrain", + "bi" = "Burundi", + "bj" = "Benin", + "bl" = "Saint Bartelemey", + "bm" = "Bermuda", + "bn" = "Brunei", + "bo" = "Bolivia", + "br" = "Brazil", + "bs" = "the Bahamas", + "bt" = "Bhutan", + "bv" = "the Bouvet Island", + "bw" = "Botswana", + "by" = "Belarus", + "bz" = "Belize", + "ca" = "Canada", + "cc" = "the Cocos (Keeling) Islands", + "cd" = "the Democratic Republic of the Congo", + "cf" = "Central African Republic", + "cg" = "Congo", + "ch" = "Switzerland", + "ci" = "Côte d'Ivoire", + "ck" = "the Cook Islands", + "cl" = "Chile", + "cm" = "Cameroon", + "cn" = "China", + "co" = "Colombia", + "cr" = "Costa Rica", + "cu" = "Cuba", + "cv" = "Cape Verde", + "cx" = "the Christmas Island", + "cy" = "Cyprus", + "cz" = "the Czech Republic", + "de" = "Germany", + "dj" = "Djibouti", + "dk" = "Denmark", + "dm" = "Dominica", + "do" = "the Dominican Republic", + "dz" = "Algeria", + "ec" = "Ecuador", + "ee" = "Estonia", + "eg" = "Egypt", + "eh" = "the Western Sahara", + "er" = "Eritrea", + "es" = "Spain", + "et" = "Ethiopia", + "fi" = "Finland", + "fj" = "Fiji", + "fk" = "the Falkland Islands (Malvinas)", + "fm" = "the Federated States of Micronesia", + "fo" = "the Faroe Islands", + "fr" = "France", + "fx" = "Metropolitan France", + "ga" = "Gabon", + "gb" = "the United Kingdom", + "gd" = "Grenada", + "ge" = "Georgia", + "gf" = "French Guiana", + "gg" = "Guernsey", + "gh" = "Ghana", + "gi" = "Gibraltar", + "gl" = "Greenland", + "gm" = "Gambia", + "gn" = "Guinea", + "gp" = "Guadeloupe", + "gq" = "Equatorial Guinea", + "gr" = "Greece", + "gs" = "South Georgia and the South Sandwich Islands", + "gt" = "Guatemala", + "gu" = "Guam", + "gw" = "Guinea-Bissau", + "gy" = "Guyana", + "hk" = "Hong Kong", + "hm" = "Heard Island and McDonald Islands", + "hn" = "Honduras", + "hr" = "Croatia", + "ht" = "Haiti", + "hu" = "Hungary", + "id" = "Indonesia", + "ie" = "Ireland", + "il" = "Israel", + "im" = "the Isle of Man", + "in" = "India", + "io" = "the British Indian Ocean Territory", + "iq" = "Iraq", + "ir" = "Iran", + "is" = "Iceland", + "it" = "Italy", + "je" = "Jersey", + "jm" = "Jamaica", + "jo" = "Jordan", + "jp" = "Japan", + "ke" = "Kenya", + "kg" = "Kyrgyzstan", + "kh" = "Cambodia", + "ki" = "Kiribati", + "km" = "Comoros", + "kn" = "Saint Kitts and Nevis", + "kp" = "North Korea", + "kr" = "the Republic of Korea", + "kw" = "Kuwait", + "ky" = "the Cayman Islands", + "kz" = "Kazakhstan", + "la" = "Laos", + "lb" = "Lebanon", + "lc" = "Saint Lucia", + "li" = "Liechtenstein", + "lk" = "Sri Lanka", + "lr" = "Liberia", + "ls" = "Lesotho", + "lt" = "Lithuania", + "lu" = "Luxembourg", + "lv" = "Latvia", + "ly" = "Libya", + "ma" = "Morocco", + "mc" = "Monaco", + "md" = "the Republic of Moldova", + "me" = "Montenegro", + "mf" = "Saint Martin", + "mg" = "Madagascar", + "mh" = "the Marshall Islands", + "mk" = "Macedonia", + "ml" = "Mali", + "mm" = "Burma", + "mn" = "Mongolia", + "mo" = "Macau", + "mp" = "the Northern Mariana Islands", + "mq" = "Martinique", + "mr" = "Mauritania", + "ms" = "Montserrat", + "mt" = "Malta", + "mu" = "Mauritius", + "mv" = "the Maldives", + "mw" = "Malawi", + "mx" = "Mexico", + "my" = "Malaysia", + "mz" = "Mozambique", + "na" = "Namibia", + "nc" = "New Caledonia", + "ne" = "Niger", + "nf" = "Norfolk Island", + "ng" = "Nigeria", + "ni" = "Nicaragua", + "nl" = "the Netherlands", + "no" = "Norway", + "np" = "Nepal", + "nr" = "Nauru", + "nu" = "Niue", + "nz" = "New Zealand", + "om" = "Oman", + "pa" = "Panama", + "pe" = "Peru", + "pf" = "French Polynesia", + "pg" = "Papua New Guinea", + "ph" = "the Philippines", + "pk" = "Pakistan", + "pl" = "Poland", + "pm" = "Saint Pierre and Miquelon", + "pn" = "the Pitcairn Islands", + "pr" = "Puerto Rico", + "ps" = "the Palestinian Territory", + "pt" = "Portugal", + "pw" = "Palau", + "py" = "Paraguay", + "qa" = "Qatar", + "re" = "Reunion", + "ro" = "Romania", + "rs" = "Serbia", + "ru" = "Russia", + "rw" = "Rwanda", + "sa" = "Saudi Arabia", + "sb" = "the Solomon Islands", + "sc" = "the Seychelles", + "sd" = "Sudan", + "se" = "Sweden", + "sg" = "Singapore", + "sh" = "Saint Helena", + "si" = "Slovenia", + "sj" = "Svalbard and Jan Mayen", + "sk" = "Slovakia", + "sl" = "Sierra Leone", + "sm" = "San Marino", + "sn" = "Senegal", + "so" = "Somalia", + "sr" = "Suriname", + "st" = "São Tomé and Príncipe", + "sv" = "El Salvador", + "sy" = "the Syrian Arab Republic", + "sz" = "Swaziland", + "tc" = "Turks and Caicos Islands", + "td" = "Chad", + "tf" = "the French Southern Territories", + "tg" = "Togo", + "th" = "Thailand", + "tj" = "Tajikistan", + "tk" = "Tokelau", + "tl" = "East Timor", + "tm" = "Turkmenistan", + "tn" = "Tunisia", + "to" = "Tonga", + "tr" = "Turkey", + "tt" = "Trinidad and Tobago", + "tv" = "Tuvalu", + "tw" = "Taiwan", + "tz" = "the United Republic of Tanzania", + "ua" = "Ukraine", + "ug" = "Uganda", + "um" = "the United States Minor Outlying Islands", + "us" = "the United States", + "uy" = "Uruguay", + "uz" = "Uzbekistan", + "va" = "Vatican City", + "vc" = "Saint Vincent and the Grenadines", + "ve" = "Venezuela", + "vg" = "the British Virgin Islands", + "vi" = "the United States Virgin Islands", + "vn" = "Vietnam", + "vu" = "Vanuatu", + "wf" = "Wallis and Futuna", + "ws" = "Samoa", + "ye" = "Yemen", + "yt" = "Mayotte", + "za" = "South Africa", + "zm" = "Zambia", + "zw" = "Zimbabwe") + +countryname <- function(country) { + res <- countrylist[[country]] + if (is.null(res)) + res <- "no-man's-land" + res +} + +plot_direct_users <- function(start, end, country, events, path) { + u <- read.csv("direct-users.csv.gz", stringsAsFactors = FALSE) + u <- u[u$date >= start & u$date <= end, ] + u <- melt(u, id.vars = "date") + u <- u[u$variable %in% c(country, "all"), ] + a <- u[u$variable == "all", ] + if (country != "all") + u <- u[u$variable == country, ] + u <- data.frame(date = u$date, users = u$value) + dates <- seq(from = as.Date(start, "%Y-%m-%d"), + to = as.Date(end, "%Y-%m-%d"), by="1 day") + missing <- setdiff(dates, as.Date(a$date)) + if (length(missing) > 0) + u <- rbind(u, + data.frame(date = as.Date(missing, origin = "1970-01-01"), + users = NA)) + missing <- setdiff(dates, as.Date(u$date)) + if (length(missing) > 0) + u <- rbind(u, + data.frame(date = as.Date(missing, origin = "1970-01-01"), + users = 0)) + title <- ifelse(country == "all", + "Directly connecting users from all countries\n", + paste("Directly connecting users from ", countryname(country), "\n", + sep = "")) + max_y <- ifelse(length(na.omit(u$users)) == 0, 0, + max(u$users, na.rm = TRUE)) + plot <- ggplot(u, aes(x = as.Date(date, "%Y-%m-%d"), y = users)) + if (length(na.omit(u$users)) > 0 & events != "off" & country != "all") { + r <- read.csv("direct-users-ranges.csv.gz", stringsAsFactors = FALSE) + r <- r[r$date >= start & r$date <= end & r$country == country, + c("date", "minusers", "maxusers")] + r[r$minusers < 0, "minusers"] <- 0 + r <- cast(rbind(melt(u, id.vars = "date"), melt(r, id.vars = "date"))) + upturns <- r[r$users > r$maxusers, 1:2] + downturns <- r[r$users < r$minusers, 1:2] + if (events == "on") { + if (length(r$maxusers) > 0) + max_y <- max(max_y, max(r$maxusers, na.rm = TRUE)) + plot <- plot + + geom_ribbon(data = r, aes(ymin = minusers, + ymax = maxusers), fill = "gray") + } + if (length(upturns$date) > 0) + plot <- plot + + geom_point(data = upturns, aes(x = as.Date(date), y = users), size = 5, + colour = "dodgerblue2") + if (length(downturns$date) > 0) + plot <- plot + + geom_point(data = downturns, aes(x = as.Date(date), y = users), size = 5, + colour = "firebrick2") + } + plot <- plot + + geom_line(size = 1) + + scale_x_date(name = paste("\nThe Tor Project - ", + "https://metrics.torproject.org/", sep = "")) + + scale_y_continuous(name = "", limits = c(0, max_y)) + + ggtitle(title) + print(plot) + ggsave(filename = path, width = 8, height = 5, dpi = 72) +} + +plot_bridge_users <- function(start, end, country, path) { + b <- read.csv("bridge-users.csv.gz", stringsAsFactors = FALSE) + b <- b[b$date >= start & b$date <= end, ] + b <- melt(b, id.vars = "date") + b <- b[b$variable %in% c(country, "all"), ] + bridgeusers <- data.frame(date = as.Date(b$date), users = b$value) + dates <- seq(from = as.Date(start, "%Y-%m-%d"), + to = as.Date(end, "%Y-%m-%d"), by="1 day") + missing <- setdiff(dates, bridgeusers$date) + if (length(missing) > 0) + bridgeusers <- rbind(bridgeusers, + data.frame(date = as.Date(missing, origin = "1970-01-01"), + users = NA)) + title <- ifelse(country == "all", + "Bridge users from all countries\n", + paste("Bridge users from ", countryname(country), "\n", sep = "")) + ggplot(bridgeusers, aes(x = as.Date(date, "%Y-%m-%d"), y = users)) + + geom_line(size = 1) + + scale_x_date(name = paste("\nThe Tor Project - ", + "https://metrics.torproject.org/", sep = "")) + + scale_y_continuous(name = "", limits = c(0, + ifelse(length(na.omit(bridgeusers$users)) == 0, 0, + max(bridgeusers$users, na.rm = TRUE)))) + + ggtitle(title) + ggsave(filename = path, width = 8, height = 5, dpi = 72) +} + +if (args[1] == "direct") { + plot_direct_users(start = args[2], end = args[3], country = args[4], + events = args[5], path = args[6]) +} else if (args[1] == "bridge") { + plot_bridge_users(start = args[2], end = args[3], country = args[4], + path = args[5]) +} +

1 0

[metrics-web/master] Fix visualization bug in user graphs.
by karsten＠torproject.org 28 Oct '13

28 Oct '13

commit 1575912022e65ff7710e8879e50e64b5584180fc Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Oct 28 14:39:22 2013 +0100 Fix visualization bug in user graphs. --- rserve/graphs.R | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rserve/graphs.R b/rserve/graphs.R index 9070308..8a62bf4 100644 --- a/rserve/graphs.R +++ b/rserve/graphs.R @@ -689,8 +689,9 @@ plot_direct_users <- function(start, end, country, events, path) { if (events == "on") { if (length(r$maxusers) > 0) max_y <- max(max_y, max(r$maxusers, na.rm = TRUE)) + r[r$minusers < 0, "minusers"] <- 0 plot <- plot + - geom_ribbon(data = r, aes(ymin = max(0, minusers), + geom_ribbon(data = r, aes(ymin = minusers, ymax = maxusers), fill = "gray") } if (length(upturns$date) > 0) @@ -1085,8 +1086,9 @@ plot_userstats <- function(start, end, node, variable, value, events, if (events == "on") { if (length(r$maxusers) > 0) max_y <- max(max_y, max(r$maxusers, na.rm = TRUE)) + r[r$minusers < 0, "minusers"] <- 0 plot <- plot + - geom_ribbon(data = r, aes(ymin = max(0, minusers), + geom_ribbon(data = r, aes(ymin = minusers, ymax = maxusers), fill = "gray") } if (length(upturns$date) > 0)

1 0