March 2015 - tor-commits - lists.torproject.org

[ooni-probe/master] clean up ClientTransportPlugin logic
by art＠torproject.org 27 Mar '15

27 Mar '15

commit bc4e167f6a6a46aca79e84c39da867f6859fa082 Author: Joe Landers <joe(a)joelanders.net> Date: Tue Feb 24 02:36:24 2015 +0100 clean up ClientTransportPlugin logic --- ooni/nettests/blocking/bridge_reachability.py | 107 +++++++++---------------- ooni/tests/test_onion.py | 23 ++++++ ooni/utils/onion.py | 42 ++++++++++ 3 files changed, 103 insertions(+), 69 deletions(-) diff --git a/ooni/nettests/blocking/bridge_reachability.py b/ooni/nettests/blocking/bridge_reachability.py index 864ae2a..aa1140a 100644 --- a/ooni/nettests/blocking/bridge_reachability.py +++ b/ooni/nettests/blocking/bridge_reachability.py @@ -2,7 +2,6 @@ import os import random import tempfile -from distutils.spawn import find_executable from twisted.python import usage from twisted.internet import reactor, error @@ -71,9 +70,6 @@ class BridgeReachability(nettest.NetTestCase): self.bridge = self.input if self.input.startswith('Bridge'): self.bridge = self.input.replace('Bridge ', '') - self.pyobfsproxy_bin = onion.obfsproxy_details['binary'] - self.fteproxy_bin = find_executable('fteproxy') - self.obfs4proxy_bin = find_executable('obfs4proxy') def postProcessor(self, measurements): if 'successes' not in self.summary: @@ -132,74 +128,47 @@ class BridgeReachability(nettest.NetTestCase): (self.bridge, onion.tor_details['version'])) transport_name = onion.transport_name(self.bridge) - - if transport_name: + if transport_name == None: + self.report['bridge_address'] = self.bridge.split(' ')[0] + else: + self.report['bridge_address'] = self.bridge.split(' ')[1] self.report['transport_name'] = transport_name - if transport_name == 'fte': - if self.fteproxy_bin: - config.ClientTransportPlugin = "%s exec %s --managed" % ( - transport_name, self.fteproxy_bin) - log.debug("Using fte from %s" % self.fteproxy_bin) - self.report['bridge_address'] = self.bridge.split(' ')[1] - else: - log.err("Unable to test bridge because fteproxy is not " - "installed") - self.report['error'] = 'missing-fteproxy' - return - elif ['scramblesuit', 'obfs2', 'obfs3'].count(transport_name) > 0: - if self.pyobfsproxy_bin: - config.ClientTransportPlugin = ("%s exec %s " - "--log-min-severity info --log-file %s managed") % \ - (transport_name, self.pyobfsproxy_bin, - self.obfsproxy_logfile) - if onion.OBFSProxyVersion('0.2') > \ - onion.obfsproxy_details['version']: - log.err( - "The obfsproxy version you are using appears to " - "be outdated." - ) - self.report['error'] = 'old-obfsproxy' - return - log.debug("Using pyobfsproxy from %s" % \ - self.pyobfsproxy_bin) - self.report['bridge_address'] = self.bridge.split(' ')[1] - else: - log.err( - "Unable to test bridge because pyobfsproxy is not " - "installed") - self.report['error'] = 'missing-pyobfsproxy' - return - elif transport_name == 'obfs4': - if self.obfs4proxy_bin: - config.ClientTransportPlugin = ("%s exec %s") % \ - (transport_name, self.obfs4proxy_bin) - log.debug("Using obfs4proxy from %s" % \ - self.obfs4proxy_bin) - self.report['bridge_address'] = self.bridge.split(' ')[1] - else: - log.err( - "Unable to test bridge because obfs4proxy is not " - "installed") - self.report['error'] = 'missing-obfs4proxy' - return - else: - log.err("Unable to test bridge because we don't handle %s" % \ - transport_name) - self.report['error'] = 'missing-transport' + + try: + config.ClientTransportPlugin = \ + onion.bridge_line(transport_name, self.obfsproxy_logfile) + except onion.UnrecognizedTransport: + log.err("Unable to test bridge because we don't recognize " + "the %s transport" % transport_name) + self.report['error'] = "unrecognized-transport" + return + except onion.UninstalledTransport: + bin_name = onion.transport_bin_name.get(transport_name) + log.err("Unable to test bridge because %s is not installed" % + bin_name) + self.report['error'] = "missing-%s" % bin_name return - else: - self.report['bridge_address'] = self.bridge.split(' ')[0] - if transport_name and transport_name == 'scramblesuit' and \ - onion.TorVersion('0.2.5.1') > onion.tor_details['version']: - self.report['error'] = 'unsupported-tor-version' - log.err("Unsupported Tor version.") - return - elif transport_name and \ - onion.TorVersion('0.2.4.1') > onion.tor_details['version']: - self.report['error'] = 'unsupported-tor-version' - log.err("Unsupported Tor version.") - return + if onion.OBFSProxyVersion('0.2') > \ + onion.obfsproxy_details['version']: + log.err("The obfsproxy version you are using " \ + "appears to be outdated.") + self.report['error'] = 'old-obfsproxy' + return + + if transport_name == 'scramblesuit' and \ + onion.TorVersion('0.2.5.1') > onion.tor_details['version']: + log.err("Unsupported Tor version.") + self.report['error'] = 'unsupported-tor-version' + return + + if onion.TorVersion('0.2.4.1') > onion.tor_details['version']: + log.err("Unsupported Tor version.") + self.report['error'] = 'unsupported-tor-version' + return + + log.debug("Using ClientTransportPlugin '%s'" % \ + config.ClientTransportPlugin) config.Bridge = self.bridge config.UseBridges = 1 diff --git a/ooni/tests/test_onion.py b/ooni/tests/test_onion.py index 94d9dd9..fa608d6 100644 --- a/ooni/tests/test_onion.py +++ b/ooni/tests/test_onion.py @@ -1,5 +1,13 @@ from twisted.trial import unittest from ooni.utils import onion +from mock import Mock + +sample_transport_lines = { + 'fte': 'fte exec /fakebin --managed', + 'scramblesuit': 'scramblesuit exec /fakebin --log-min-severity info --log-file /log.txt managed', + 'obfs2': 'obfs2 exec /fakebin --log-min-severity info --log-file /log.txt managed', + 'obfs3': 'obfs3 exec /fakebin --log-min-severity info --log-file /log.txt managed', + 'obfs4': 'obfs4 exec /fakebin' } class TestOnion(unittest.TestCase): @@ -7,3 +15,18 @@ class TestOnion(unittest.TestCase): assert isinstance(onion.tor_details, dict) assert onion.tor_details['version'] assert onion.tor_details['binary'] + def test_transport_dicts(self): + self.assertEqual( set(onion.transport_bin_name.keys()), + set(onion._transport_line_templates.keys()) ) + def test_bridge_line(self): + self.assertRaises(onion.UnrecognizedTransport, + onion.bridge_line, 'rot13', '/log.txt') + + onion.find_executable = Mock(return_value=False) + self.assertRaises(onion.UninstalledTransport, + onion.bridge_line, 'fte', '/log.txt') + + onion.find_executable = Mock(return_value="/fakebin") + for transport, exp_line in sample_transport_lines.iteritems(): + self.assertEqual(onion.bridge_line(transport, '/log.txt'), + exp_line) diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py index 09e104c..b0638f8 100644 --- a/ooni/utils/onion.py +++ b/ooni/utils/onion.py @@ -74,3 +74,45 @@ obfsproxy_details = { 'binary': find_executable('obfsproxy'), 'version': obfsproxy_version() } + +transport_bin_name = { 'fte': 'fteproxy', + 'scramblesuit': 'obfsproxy', + 'obfs2': 'obfsproxy', + 'obfs3': 'obfsproxy', + 'obfs4': 'obfs4proxy' } + +_pyobfsproxy_line = lambda transport, bin_loc, log_file: \ + "%s exec %s --log-min-severity info --log-file %s managed" % \ + (transport, bin_loc, log_file) + +_transport_line_templates = { + 'fte': lambda bin_loc, log_file : \ + "fte exec %s --managed" % bin_loc, + + 'scramblesuit': lambda bin_loc, log_file: \ + _pyobfsproxy_line('scramblesuit', bin_loc, log_file), + + 'obfs2': lambda bin_loc, log_file: \ + _pyobfsproxy_line('obfs2', bin_loc, log_file), + + 'obfs3': lambda bin_loc, log_file: \ + _pyobfsproxy_line('obfs3', bin_loc, log_file), + + 'obfs4': lambda bin_loc, log_file: \ + "obfs4 exec %s" % bin_loc } + +class UnrecognizedTransport(Exception): + pass +class UninstalledTransport(Exception): + pass + +def bridge_line(transport, log_file): + bin_name = transport_bin_name.get(transport) + if not bin_name: + raise UnrecognizedTransport + + bin_loc = find_executable(bin_name) + if not bin_loc: + raise UninstalledTransport + + return _transport_line_templates[transport](bin_loc, log_file)

1 0

[ooni-probe/master] Point to the new citizenlab test lists directory
by art＠torproject.org 27 Mar '15

27 Mar '15

commit 8ec21f642126874a3ffbb7306ebef7939e3a8394 Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Mar 17 21:47:34 2015 +0100 Point to the new citizenlab test lists directory --- ooni/deckgen/processors/citizenlab_test_lists.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/deckgen/processors/citizenlab_test_lists.py b/ooni/deckgen/processors/citizenlab_test_lists.py index b455008..0a56ec0 100644 --- a/ooni/deckgen/processors/citizenlab_test_lists.py +++ b/ooni/deckgen/processors/citizenlab_test_lists.py @@ -28,7 +28,7 @@ def generate_country_input(country_code, dst): input_list = config.get_data_file_path("resources/" "citizenlab-test-lists/" - "test-lists-master/csv/" + "test-lists-master/lists/" + country_code + ".csv") if not os.path.exists(input_list): @@ -44,7 +44,7 @@ def generate_global_input(dst): input_list = config.get_data_file_path("resources/" "citizenlab-test-lists/" - "test-lists-master/csv/" + "test-lists-master/lists/" "global.csv") load_input(input_list, filename)

1 0

[ooni-probe/master] Merge branch 'joe/13597'
by art＠torproject.org 27 Mar '15

27 Mar '15

commit 2c069e1379a07b2384a8201c2de0f19e30da49bb Merge: 8ec21f6 9ffbbc3 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Mar 18 09:54:54 2015 +0100 Merge branch 'joe/13597' * joe/13597: clean up checking for outdated versions set our own datadir so we can grab obfs4 logs clean up ClientTransportPlugin logic fixes from comments add obfs4 bridge reachability support factor out another level of if/elses factor out a level of if/elses ooni/nettests/blocking/bridge_reachability.py | 81 ++++++++++++------------- ooni/tests/test_onion.py | 41 +++++++++++++ ooni/utils/onion.py | 57 +++++++++++++++++ 3 files changed, 136 insertions(+), 43 deletions(-)

1 0

[ooni-probe/master] clean up checking for outdated versions
by art＠torproject.org 27 Mar '15

27 Mar '15

commit 9ffbbc32975ed5b60ddcbdb5928b7af1813a24d7 Author: Joe Landers <joe(a)joelanders.net> Date: Fri Feb 27 19:52:32 2015 +0100 clean up checking for outdated versions --- ooni/nettests/blocking/bridge_reachability.py | 13 ++----------- ooni/tests/test_onion.py | 20 +++++++++++++++++++- ooni/utils/onion.py | 15 +++++++++++++++ 3 files changed, 36 insertions(+), 12 deletions(-) diff --git a/ooni/nettests/blocking/bridge_reachability.py b/ooni/nettests/blocking/bridge_reachability.py index 8ec1bf9..5d9dc96 100644 --- a/ooni/nettests/blocking/bridge_reachability.py +++ b/ooni/nettests/blocking/bridge_reachability.py @@ -151,21 +151,12 @@ class BridgeReachability(nettest.NetTestCase): bin_name) self.report['error'] = "missing-%s" % bin_name return - - if onion.OBFSProxyVersion('0.2') > \ - onion.obfsproxy_details['version']: + except onion.OutdatedObfsproxy: log.err("The obfsproxy version you are using " \ "appears to be outdated.") self.report['error'] = 'old-obfsproxy' return - - if transport_name == 'scramblesuit' and \ - onion.TorVersion('0.2.5.1') > onion.tor_details['version']: - log.err("Unsupported Tor version.") - self.report['error'] = 'unsupported-tor-version' - return - - if onion.TorVersion('0.2.4.1') > onion.tor_details['version']: + except onion.OutdatedTor: log.err("Unsupported Tor version.") self.report['error'] = 'unsupported-tor-version' return diff --git a/ooni/tests/test_onion.py b/ooni/tests/test_onion.py index 6dce5fc..30fd26d 100644 --- a/ooni/tests/test_onion.py +++ b/ooni/tests/test_onion.py @@ -1,6 +1,6 @@ from twisted.trial import unittest from ooni.utils import onion -from mock import Mock +from mock import Mock, patch sample_transport_lines = { 'fte': 'fte exec /fakebin --managed', @@ -30,3 +30,21 @@ class TestOnion(unittest.TestCase): for transport, exp_line in sample_transport_lines.iteritems(): self.assertEqual(onion.bridge_line(transport, '/log.txt'), exp_line) + + with patch.dict(onion.obfsproxy_details, + {'version': onion.OBFSProxyVersion('0.1.12')}): + self.assertRaises(onion.OutdatedObfsproxy, + onion.bridge_line, 'obfs2', '/log.txt') + + with patch.dict(onion.tor_details, + {'version': onion.TorVersion('0.2.4.20')}): + onion.bridge_line('fte', '/log.txt') + self.assertRaises(onion.OutdatedTor, + onion.bridge_line, 'scramblesuit', '/log.txt') + self.assertRaises(onion.OutdatedTor, + onion.bridge_line, 'obfs4', '/log.txt') + + with patch.dict(onion.tor_details, + {'version': onion.TorVersion('0.2.3.20')}): + self.assertRaises(onion.OutdatedTor, + onion.bridge_line, 'fte', '/log.txt') diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py index 6476636..ec0a8b4 100644 --- a/ooni/utils/onion.py +++ b/ooni/utils/onion.py @@ -105,6 +105,10 @@ class UnrecognizedTransport(Exception): pass class UninstalledTransport(Exception): pass +class OutdatedObfsproxy(Exception): + pass +class OutdatedTor(Exception): + pass def bridge_line(transport, log_file): bin_name = transport_bin_name.get(transport) @@ -115,4 +119,15 @@ def bridge_line(transport, log_file): if not bin_loc: raise UninstalledTransport + if OBFSProxyVersion('0.2') > obfsproxy_details['version']: + raise OutdatedObfsproxy + + if (transport == 'scramblesuit' or \ + bin_name == 'obfs4proxy') and \ + TorVersion('0.2.5.1') > tor_details['version']: + raise OutdatedTor + + if TorVersion('0.2.4.1') > tor_details['version']: + raise OutdatedTor + return _transport_line_templates[transport](bin_loc, log_file)

1 0

[translation/torbutton-torbuttonproperties] Update translations for torbutton-torbuttonproperties
by translation＠torproject.org 27 Mar '15

27 Mar '15

commit 5f5364d3662320c27f693d93f368c28d011717d2 Author: Translation commit bot <translation(a)torproject.org> Date: Fri Mar 27 08:45:51 2015 +0000 Update translations for torbutton-torbuttonproperties --- tr/torbutton.properties | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tr/torbutton.properties b/tr/torbutton.properties index 59dafbd..b42c784 100644 --- a/tr/torbutton.properties +++ b/tr/torbutton.properties @@ -55,10 +55,10 @@ torbutton.popup.confirm_plugins = Flash gibi eklentiler gizliliğinizi ve anonim torbutton.popup.never_ask_again = Bir daha sorma torbutton.popup.confirm_newnym = Tor Tarayıcısı tüm sekmeleri ve pencereleri kapatacak. Tüm web sitesi oturumları kaybolacaktır.\n\nOturumunuzu sıfırlamak için Tor tarayıcı yeniden başlatılsın mı?\n -torbutton.slider_notification = The green onion menu has now a security slider which lets you adjust your security level. Check it out! -torbutton.slider_notification_button = Open security settings +torbutton.slider_notification = Yeşil soğan menüsü artık güvenlik seviyesini ayarlayabileceğiniz bir güvenlik kaydırıcısını içeriyor. Deneyin! +torbutton.slider_notification_button = Güvenlik ayarlarını aç -torbutton.maximize_warning = Maximizing Tor Browser can allow websites to determine your monitor size, which can be used to track you. We recommend that you leave Tor Browser windows in their original default size. +torbutton.maximize_warning = Tor Tarayıcısı'nın ekranı kaplamasını etkinleştirmek web sitelerinin ekran boyutunuzu öğrenmesine; dolayısıyla sizi izlemede bu bilgiyi kullanmalarını sağlayabilir. Tor Tarayıcı pencerelerini özgün varsayılan boyutlarında bırakmanızı tavsiye ediyoruz. # Canvas permission prompt. Strings are kept here for ease of translation. canvas.siteprompt=Bu web sitesi (%S) bilgisayarınızı benzersiz olarak tanımlayabilmek için kullanılabilen HTML5 tuvalindeki resim verisine erişmeye çalıştı.\n\nTor Tarayıcı bu web sitesine HTML5 tuval resim verisini çıkarmasına izin vermeli mi?

1 0

[translation/gettor] Update translations for gettor
by translation＠torproject.org 27 Mar '15

27 Mar '15

commit 086253b94d9354f4ed9c27db2b896b6fadcfec4c Author: Translation commit bot <translation(a)torproject.org> Date: Fri Mar 27 08:45:04 2015 +0000 Update translations for gettor --- bn_IN/gettor.po | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bn_IN/gettor.po b/bn_IN/gettor.po index 91ac3db..23d4363 100644 --- a/bn_IN/gettor.po +++ b/bn_IN/gettor.po @@ -9,7 +9,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2013-01-19 13:40+0100\n" -"PO-Revision-Date: 2015-03-27 08:13+0000\n" +"PO-Revision-Date: 2015-03-27 08:25+0000\n" "Last-Translator: Rajarshi Bhowmick <raz.bhowmick(a)gmail.com>\n" "Language-Team: Bengali (India) (http://www.transifex.com/projects/p/torproject/language/bn_IN/)\n" "MIME-Version: 1.0\n" @@ -75,7 +75,7 @@ msgstr "আমি আপনাকে টর প্যাকেজ পাঠি msgid "" "Please reply to this mail, and tell me a single package name anywhere \n" "in the body of your email." -msgstr "" +msgstr "দয়াকরে, এই মেইল উত্তর দিন, এবং আপনার ইমেইল শরীরের যে কোন স্থানে\nএকটি প্যাকেজের নাম বলুন." #: lib/gettor/i18n.py:64 msgid "" @@ -426,7 +426,7 @@ msgstr "" msgid "" "source:\n" "The Tor source code, for experts. Most users do not want this package." -msgstr "" +msgstr "উৎস:\nটর সোর্স কোড, হল বিশেষজ্ঞদের জন্য. অধিকাংশ ব্যবহারকারীদের এই প্যাকেজ দরকার হয় না." #: lib/gettor/i18n.py:269 msgid "" @@ -440,7 +440,7 @@ msgstr "টর কি জিনিস ?" #: lib/gettor/i18n.py:274 msgid "The name \"Tor\" can refer to several different components." -msgstr "" +msgstr "\"টর\" নামটি বিভিন্ন বিষয়ের হতে পারে. " #: lib/gettor/i18n.py:276 msgid "" @@ -480,7 +480,7 @@ msgstr "" #: lib/gettor/i18n.py:299 msgid "How do I extract the file(s) you sent me?" -msgstr "" +msgstr "আমাকে যে ফাইল গুলি পাঠিয়েছেন সেগুলি কিভাবে এক্সট্র্যাক্ট করব?" #: lib/gettor/i18n.py:301 msgid "QUESTION:"

1 0

[translation/gettor] Update translations for gettor
by translation＠torproject.org 27 Mar '15

27 Mar '15

commit b9a8a2e188ea00948f8b76a2a42839d271cb3a1a Author: Translation commit bot <translation(a)torproject.org> Date: Fri Mar 27 08:15:05 2015 +0000 Update translations for gettor --- bn_IN/gettor.po | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bn_IN/gettor.po b/bn_IN/gettor.po index 5b1ae93..91ac3db 100644 --- a/bn_IN/gettor.po +++ b/bn_IN/gettor.po @@ -9,7 +9,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2013-01-19 13:40+0100\n" -"PO-Revision-Date: 2015-03-27 06:51+0000\n" +"PO-Revision-Date: 2015-03-27 08:13+0000\n" "Last-Translator: Rajarshi Bhowmick <raz.bhowmick(a)gmail.com>\n" "Language-Team: Bengali (India) (http://www.transifex.com/projects/p/torproject/language/bn_IN/)\n" "MIME-Version: 1.0\n" @@ -69,7 +69,7 @@ msgid "" " obfs-linux-i386\n" " obfs-linux-x86_64\n" " source" -msgstr "" +msgstr "আমি আপনাকে টর প্যাকেজ পাঠিয়ে দেব. যদি আপনি বলেন কোনটা আপনার লাগবে.\nনিম্নলিখিত প্যাকেজের থেকে একটি নাম নির্বাচন করুন.\n\nউইন্ডোস\nম্যাক ওয়েস -i386\nম্যাক ওয়েস -পি পি সি\nলিনাক্স-i386\nলিনাক্স-x86_64\nও বি এফ এস-উইন্ডোস\nও বি এফ এস-ম্যাক ওয়েস -i386\nও বি এফ এস-ম্যাক ওয়েস-x86_64\nও বি এফ এস-লিনাক্স-i386\nও বি এফ এস-লিনাক্স-x86_64\nউৎস" #: lib/gettor/i18n.py:61 msgid "" @@ -161,7 +161,7 @@ msgstr "" #: lib/gettor/i18n.py:113 msgid "1.) Save all received attachments into one folder on your disk." -msgstr "" +msgstr "১.) আপনার ডিস্কে, একটি ফোল্ডার মধ্যে, সব পাওয়া জিনিসগুলি সংরক্ষণ করুন." #: lib/gettor/i18n.py:115 msgid "" @@ -192,7 +192,7 @@ msgstr "" #: lib/gettor/i18n.py:130 msgid "6.) That's it. You're done. Thanks for using Tor and have fun!" -msgstr "" +msgstr "6.) ব্যাস, হয়ে গেল. এবার টর ব্যবহার করুন মজা করে. " #: lib/gettor/i18n.py:132 msgid "" @@ -204,7 +204,7 @@ msgstr "সহায়তা\n======= " msgid "" "Here's your requested software as a zip file. Please unzip the\n" "package and verify the signature." -msgstr "" +msgstr "আপনার অনুরোধ সফটওয়্যারটি একটি জিপ ফাইল হিসাবে দেওয়া হল. \nপ্যাকেজ আনজিপ করুন এবং স্বাক্ষর যাচাই করুন." #: lib/gettor/i18n.py:141 msgid "" @@ -235,7 +235,7 @@ msgstr "" msgid "" "BLOCKED ACCESS / CENSORSHIP\n" "===========================" -msgstr "" +msgstr "ব্যবহার বন্ধ / সেন্সর করা আছে\n===========================" #: lib/gettor/i18n.py:160 msgid ""

1 0

[onionoo/master] Add fractional time of having a flag assigned.
by karsten＠torproject.org 27 Mar '15

27 Mar '15

commit 43434dc5f4b8f9c588accc6c4e8e0b8981becc98 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Mar 19 22:44:20 2015 +0100 Add fractional time of having a flag assigned. Implements #15177. --- build.xml | 4 +- .../torproject/onionoo/docs/UptimeDocument.java | 10 ++ .../org/torproject/onionoo/docs/UptimeHistory.java | 37 ++++-- .../org/torproject/onionoo/docs/UptimeStatus.java | 74 +++++++----- .../torproject/onionoo/server/ResponseBuilder.java | 2 +- .../onionoo/updater/UptimeStatusUpdater.java | 94 +++++++++++---- .../onionoo/writer/UptimeDocumentWriter.java | 42 ++++++- .../torproject/onionoo/docs/UptimeStatusTest.java | 123 +++++++++++++++----- .../torproject/onionoo/updater/DummyConsensus.java | 7 +- .../onionoo/updater/UptimeStatusUpdaterTest.java | 5 +- web/protocol.html | 19 +++ 11 files changed, 319 insertions(+), 98 deletions(-) diff --git a/build.xml b/build.xml index a3f105e..5986416 100644 --- a/build.xml +++ b/build.xml @@ -1,8 +1,8 @@ <project default="dist" name="onionoo" basedir="."> - <property name="onionoo.protocol.version" value="2.2"/> + <property name="onionoo.protocol.version" value="2.3"/> <property name="release.version" - value="${onionoo.protocol.version}.1"/> + value="${onionoo.protocol.version}.0"/> <property name="javasources" value="src/main/java"/> <property name="tests" value="src/test/java"/> <property name="classes" value="classes"/> diff --git a/src/main/java/org/torproject/onionoo/docs/UptimeDocument.java b/src/main/java/org/torproject/onionoo/docs/UptimeDocument.java index 7f0bacc..9b84a31 100644 --- a/src/main/java/org/torproject/onionoo/docs/UptimeDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/UptimeDocument.java @@ -3,6 +3,7 @@ package org.torproject.onionoo.docs; import java.util.Map; +import java.util.SortedMap; public class UptimeDocument extends Document { @@ -19,5 +20,14 @@ public class UptimeDocument extends Document { public Map<String, GraphHistory> getUptime() { return this.uptime; } + + private SortedMap<String, Map<String, GraphHistory>> flags; + public void setFlags( + SortedMap<String, Map<String, GraphHistory>> flags) { + this.flags = flags; + } + public SortedMap<String, Map<String, GraphHistory>> getFlags() { + return this.flags; + } } diff --git a/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java b/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java index b686c7e..e98f72a 100644 --- a/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java +++ b/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java @@ -1,5 +1,8 @@ package org.torproject.onionoo.docs; +import java.util.SortedSet; +import java.util.TreeSet; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,27 +27,33 @@ public class UptimeHistory implements Comparable<UptimeHistory> { return this.uptimeHours; } + private SortedSet<String> flags; + public SortedSet<String> getFlags() { + return this.flags; + } + UptimeHistory(boolean relay, long startMillis, - int uptimeHours) { + int uptimeHours, SortedSet<String> flags) { this.relay = relay; this.startMillis = startMillis; this.uptimeHours = uptimeHours; + this.flags = flags; } public static UptimeHistory fromString(String uptimeHistoryString) { - String[] parts = uptimeHistoryString.split(" ", 3); - if (parts.length != 3) { + String[] parts = uptimeHistoryString.split(" ", -1); + if (parts.length < 3) { log.warn("Invalid number of space-separated strings in uptime " + "history: '" + uptimeHistoryString + "'. Skipping"); return null; } boolean relay = false; - if (parts[0].equals("r")) { + if (parts[0].equalsIgnoreCase("r")) { relay = true; } else if (!parts[0].equals("b")) { log.warn("Invalid node type in uptime history: '" - + uptimeHistoryString + "'. Supported types are 'r' and 'b'. " - + "Skipping."); + + uptimeHistoryString + "'. Supported types are 'r', 'R', and " + + "'b'. Skipping."); return null; } long startMillis = DateTimeHelper.parse(parts[1], @@ -62,15 +71,27 @@ public class UptimeHistory implements Comparable<UptimeHistory> { + uptimeHistoryString + "'. Skipping."); return null; } - return new UptimeHistory(relay, startMillis, uptimeHours); + SortedSet<String> flags = null; + if (parts[0].equals("R")) { + flags = new TreeSet<String>(); + for (int i = 3; i < parts.length; i++) { + flags.add(parts[i]); + } + } + return new UptimeHistory(relay, startMillis, uptimeHours, flags); } public String toString() { StringBuilder sb = new StringBuilder(); - sb.append(this.relay ? "r" : "b"); + sb.append(this.relay ? (this.flags == null ? "r" : "R") : "b"); sb.append(" " + DateTimeHelper.format(this.startMillis, DateTimeHelper.DATEHOUR_NOSPACE_FORMAT)); sb.append(" " + String.format("%d", this.uptimeHours)); + if (this.flags != null) { + for (String flag : this.flags) { + sb.append(" " + flag); + } + } return sb.toString(); } diff --git a/src/main/java/org/torproject/onionoo/docs/UptimeStatus.java b/src/main/java/org/torproject/onionoo/docs/UptimeStatus.java index e48fd09..266a245 100644 --- a/src/main/java/org/torproject/onionoo/docs/UptimeStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/UptimeStatus.java @@ -2,6 +2,7 @@ * See LICENSE for licensing information */ package org.torproject.onionoo.docs; +import java.util.NavigableSet; import java.util.Scanner; import java.util.SortedSet; import java.util.TreeSet; @@ -24,18 +25,12 @@ public class UptimeStatus extends Document { private SortedSet<UptimeHistory> relayHistory = new TreeSet<UptimeHistory>(); - public void setRelayHistory(SortedSet<UptimeHistory> relayHistory) { - this.relayHistory = relayHistory; - } public SortedSet<UptimeHistory> getRelayHistory() { return this.relayHistory; } private SortedSet<UptimeHistory> bridgeHistory = new TreeSet<UptimeHistory>(); - public void setBridgeHistory(SortedSet<UptimeHistory> bridgeHistory) { - this.bridgeHistory = bridgeHistory; - } public SortedSet<UptimeHistory> getBridgeHistory() { return this.bridgeHistory; } @@ -59,30 +54,50 @@ public class UptimeStatus extends Document { s.close(); } - public void addToHistory(boolean relay, SortedSet<Long> newIntervals) { - for (long startMillis : newIntervals) { - SortedSet<UptimeHistory> history = relay ? this.relayHistory - : this.bridgeHistory; - UptimeHistory interval = new UptimeHistory(relay, startMillis, 1); - if (!history.headSet(interval).isEmpty()) { - UptimeHistory prev = history.headSet(interval).last(); - if (prev.isRelay() == interval.isRelay() && - prev.getStartMillis() + DateTimeHelper.ONE_HOUR - * prev.getUptimeHours() > interval.getStartMillis()) { - continue; - } + public void addToHistory(boolean relay, long startMillis, + SortedSet<String> flags) { + SortedSet<UptimeHistory> history = relay ? this.relayHistory + : this.bridgeHistory; + UptimeHistory interval = new UptimeHistory(relay, startMillis, 1, + flags); + NavigableSet<UptimeHistory> existingIntervals = + new TreeSet<UptimeHistory>(history.headSet(new UptimeHistory( + relay, startMillis + DateTimeHelper.ONE_HOUR, 0, flags))); + for (UptimeHistory prev : existingIntervals.descendingSet()) { + if (prev.isRelay() != interval.isRelay() || + prev.getStartMillis() + DateTimeHelper.ONE_HOUR + * prev.getUptimeHours() <= interval.getStartMillis()) { + break; } - if (!history.tailSet(interval).isEmpty()) { - UptimeHistory next = history.tailSet(interval).first(); - if (next.isRelay() == interval.isRelay() && - next.getStartMillis() < interval.getStartMillis() - + DateTimeHelper.ONE_HOUR) { - continue; - } + if (prev.getFlags() == interval.getFlags() || + (prev.getFlags() != null && interval.getFlags() != null && + prev.getFlags().equals(interval.getFlags()))) { + /* The exact same interval is already contained in history. */ + return; + } + /* There is an interval that includes the new interval, but it + * contains different flags. Remove the old interval, put in any + * parts before or after the new interval, and add the new interval + * further down below. */ + history.remove(prev); + int hoursBefore = (int) ((interval.getStartMillis() + - prev.getStartMillis()) / DateTimeHelper.ONE_HOUR); + if (hoursBefore > 0) { + history.add(new UptimeHistory(relay, + prev.getStartMillis(), hoursBefore, prev.getFlags())); + } + int hoursAfter = (int) (prev.getStartMillis() + / DateTimeHelper.ONE_HOUR + prev.getUptimeHours() - + interval.getStartMillis() / DateTimeHelper.ONE_HOUR - 1); + if (hoursAfter > 0) { + history.add(new UptimeHistory(relay, + interval.getStartMillis() + DateTimeHelper.ONE_HOUR, + hoursAfter, prev.getFlags())); } - history.add(interval); - this.isDirty = true; + break; } + history.add(interval); + this.isDirty = true; } public void compressHistory() { @@ -99,7 +114,10 @@ public class UptimeStatus extends Document { if (lastInterval != null && lastInterval.getStartMillis() + DateTimeHelper.ONE_HOUR * lastInterval.getUptimeHours() == interval.getStartMillis() && - lastInterval.isRelay() == interval.isRelay()) { + lastInterval.isRelay() == interval.isRelay() && + (lastInterval.getFlags() == interval.getFlags() || + (lastInterval.getFlags() != null && interval.getFlags() != null + && lastInterval.getFlags().equals(interval.getFlags())))) { lastInterval.addUptime(interval); } else { if (lastInterval != null) { diff --git a/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java b/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java index d1e7f23..af0f67e 100644 --- a/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java +++ b/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java @@ -70,7 +70,7 @@ public class ResponseBuilder { return this.charsWritten; } - private static final String PROTOCOL_VERSION = "2.2"; + private static final String PROTOCOL_VERSION = "2.3"; private static final String NEXT_MAJOR_VERSION_SCHEDULED = null; diff --git a/src/main/java/org/torproject/onionoo/updater/UptimeStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/UptimeStatusUpdater.java index c3c98ed..e2cee78 100644 --- a/src/main/java/org/torproject/onionoo/updater/UptimeStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/UptimeStatusUpdater.java @@ -2,6 +2,8 @@ * See LICENSE for licensing information */ package org.torproject.onionoo.updater; +import java.util.BitSet; +import java.util.HashMap; import java.util.Map; import java.util.SortedMap; import java.util.SortedSet; @@ -48,41 +50,70 @@ public class UptimeStatusUpdater implements DescriptorListener, } } - private SortedSet<Long> newRelayStatuses = new TreeSet<Long>(), - newBridgeStatuses = new TreeSet<Long>(); + private static class Flags { + + private static Map<String, Integer> flagIndexes = + new HashMap<String, Integer>(); + + private static Map<Integer, String> flagStrings = + new HashMap<Integer, String>(); + + private BitSet flags; + + private Flags(SortedSet<String> flags) { + this.flags = new BitSet(flagIndexes.size()); + for (String flag : flags) { + if (!flagIndexes.containsKey(flag)) { + flagStrings.put(flagIndexes.size(), flag); + flagIndexes.put(flag, flagIndexes.size()); + } + this.flags.set(flagIndexes.get(flag)); + } + } + + public SortedSet<String> getFlags() { + SortedSet<String> result = new TreeSet<String>(); + if (this.flags != null) { + for (int i = this.flags.nextSetBit(0); i >= 0; + i = this.flags.nextSetBit(i + 1)) { + result.add(flagStrings.get(i)); + } + } + return result; + } + } + + private SortedMap<Long, Flags> + newRelayStatuses = new TreeMap<Long, Flags>(); + private SortedMap<String, SortedMap<Long, Flags>> + newRunningRelays = new TreeMap<String, SortedMap<Long, Flags>>(); + private SortedSet<Long> newBridgeStatuses = new TreeSet<Long>(); private SortedMap<String, SortedSet<Long>> - newRunningRelays = new TreeMap<String, SortedSet<Long>>(), newRunningBridges = new TreeMap<String, SortedSet<Long>>(); private void processRelayNetworkStatusConsensus( RelayNetworkStatusConsensus consensus) { - SortedSet<String> fingerprints = new TreeSet<String>(); + long dateHourMillis = (consensus.getValidAfterMillis() + / DateTimeHelper.ONE_HOUR) * DateTimeHelper.ONE_HOUR; for (NetworkStatusEntry entry : consensus.getStatusEntries().values()) { - if (entry.getFlags().contains("Running")) { - fingerprints.add(entry.getFingerprint()); - } - } - if (!fingerprints.isEmpty()) { - long dateHourMillis = (consensus.getValidAfterMillis() - / DateTimeHelper.ONE_HOUR) * DateTimeHelper.ONE_HOUR; - for (String fingerprint : fingerprints) { - if (!this.newRunningRelays.containsKey(fingerprint)) { - this.newRunningRelays.put(fingerprint, new TreeSet<Long>()); - } - this.newRunningRelays.get(fingerprint).add(dateHourMillis); + String fingerprint = entry.getFingerprint(); + if (!this.newRunningRelays.containsKey(fingerprint)) { + this.newRunningRelays.put(fingerprint, + new TreeMap<Long, Flags>()); } - this.newRelayStatuses.add(dateHourMillis); + this.newRunningRelays.get(fingerprint).put(dateHourMillis, + new Flags(entry.getFlags())); } + this.newRelayStatuses.put(dateHourMillis, + new Flags(consensus.getKnownFlags())); } private void processBridgeNetworkStatus(BridgeNetworkStatus status) { SortedSet<String> fingerprints = new TreeSet<String>(); for (NetworkStatusEntry entry : status.getStatusEntries().values()) { - if (entry.getFlags().contains("Running")) { - fingerprints.add(entry.getFingerprint()); - } + fingerprints.add(entry.getFingerprint()); } if (!fingerprints.isEmpty()) { long dateHourMillis = (status.getPublishedMillis() @@ -98,20 +129,30 @@ public class UptimeStatusUpdater implements DescriptorListener, } public void updateStatuses() { - for (Map.Entry<String, SortedSet<Long>> e : + for (Map.Entry<String, SortedMap<Long, Flags>> e : this.newRunningRelays.entrySet()) { this.updateStatus(true, e.getKey(), e.getValue()); } this.updateStatus(true, null, this.newRelayStatuses); for (Map.Entry<String, SortedSet<Long>> e : this.newRunningBridges.entrySet()) { - this.updateStatus(false, e.getKey(), e.getValue()); + SortedMap<Long, Flags> dateHourMillisNoFlags = + new TreeMap<Long, Flags>(); + for (long dateHourMillis : e.getValue()) { + dateHourMillisNoFlags.put(dateHourMillis, null); + } + this.updateStatus(false, e.getKey(), dateHourMillisNoFlags); + } + SortedMap<Long, Flags> dateHourMillisNoFlags = + new TreeMap<Long, Flags>(); + for (long dateHourMillis : this.newBridgeStatuses) { + dateHourMillisNoFlags.put(dateHourMillis, null); } - this.updateStatus(false, null, this.newBridgeStatuses); + this.updateStatus(false, null, dateHourMillisNoFlags); } private void updateStatus(boolean relay, String fingerprint, - SortedSet<Long> newUptimeHours) { + SortedMap<Long, Flags> dateHourMillisFlags) { UptimeStatus uptimeStatus = (fingerprint == null) ? this.documentStore.retrieve(UptimeStatus.class, true) : this.documentStore.retrieve(UptimeStatus.class, true, @@ -119,7 +160,10 @@ public class UptimeStatusUpdater implements DescriptorListener, if (uptimeStatus == null) { uptimeStatus = new UptimeStatus(); } - uptimeStatus.addToHistory(relay, newUptimeHours); + for (Map.Entry<Long, Flags> e : dateHourMillisFlags.entrySet()) { + uptimeStatus.addToHistory(relay, e.getKey(), + e.getValue() == null ? null : e.getValue().getFlags()); + } if (uptimeStatus.isDirty()) { uptimeStatus.compressHistory(); if (fingerprint == null) { diff --git a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java index c5f71a1..d9eb91f 100644 --- a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java @@ -6,7 +6,10 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.SortedMap; import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -107,18 +110,45 @@ public class UptimeDocumentWriter implements DocumentWriter { this.graphIntervals.length; graphIntervalIndex++) { String graphName = this.graphNames[graphIntervalIndex]; GraphHistory graphHistory = this.compileUptimeHistory( - graphIntervalIndex, relay, history, knownStatuses); + graphIntervalIndex, relay, history, knownStatuses, null); if (graphHistory != null) { uptime.put(graphName, graphHistory); } } uptimeDocument.setUptime(uptime); + SortedMap<String, Map<String, GraphHistory>> flags = + new TreeMap<String, Map<String, GraphHistory>>(); + SortedSet<String> allFlags = new TreeSet<String>(); + for (UptimeHistory hist : history) { + if (hist.getFlags() != null) { + allFlags.addAll(hist.getFlags()); + } + } + for (String flag : allFlags) { + Map<String, GraphHistory> graphsForFlags = + new LinkedHashMap<String, GraphHistory>(); + for (int graphIntervalIndex = 0; graphIntervalIndex < + this.graphIntervals.length; graphIntervalIndex++) { + String graphName = this.graphNames[graphIntervalIndex]; + GraphHistory graphHistory = this.compileUptimeHistory( + graphIntervalIndex, relay, history, knownStatuses, flag); + if (graphHistory != null) { + graphsForFlags.put(graphName, graphHistory); + } + } + if (!graphsForFlags.isEmpty()) { + flags.put(flag, graphsForFlags); + } + } + if (!flags.isEmpty()) { + uptimeDocument.setFlags(flags); + } return uptimeDocument; } private GraphHistory compileUptimeHistory(int graphIntervalIndex, boolean relay, SortedSet<UptimeHistory> history, - SortedSet<UptimeHistory> knownStatuses) { + SortedSet<UptimeHistory> knownStatuses, String flag) { long graphInterval = this.graphIntervals[graphIntervalIndex]; long dataPointInterval = this.dataPointIntervals[graphIntervalIndex]; @@ -130,7 +160,9 @@ public class UptimeDocumentWriter implements DocumentWriter { int uptimeHours = 0; long firstStatusStartMillis = -1L; for (UptimeHistory hist : history) { - if (hist.isRelay() != relay) { + if (hist.isRelay() != relay || + (flag != null && (hist.getFlags() == null || + !hist.getFlags().contains(flag)))) { continue; } if (firstStatusStartMillis < 0L) { @@ -170,7 +202,9 @@ public class UptimeDocumentWriter implements DocumentWriter { / dataPointInterval) * dataPointInterval; int statusHours = -1; for (UptimeHistory hist : knownStatuses) { - if (hist.isRelay() != relay) { + if (hist.isRelay() != relay || + (flag != null && (hist.getFlags() == null || + !hist.getFlags().contains(flag)))) { continue; } long histEndMillis = hist.getStartMillis() + DateTimeHelper.ONE_HOUR diff --git a/src/test/java/org/torproject/onionoo/docs/UptimeStatusTest.java b/src/test/java/org/torproject/onionoo/docs/UptimeStatusTest.java index b4bcdc8..7a48c25 100644 --- a/src/test/java/org/torproject/onionoo/docs/UptimeStatusTest.java +++ b/src/test/java/org/torproject/onionoo/docs/UptimeStatusTest.java @@ -7,12 +7,10 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.util.Arrays; +import java.util.SortedSet; import java.util.TreeSet; import org.junit.Test; -import org.torproject.onionoo.docs.DateTimeHelper; -import org.torproject.onionoo.docs.UptimeHistory; -import org.torproject.onionoo.docs.UptimeStatus; public class UptimeStatusTest { @@ -26,8 +24,8 @@ public class UptimeStatusTest { @Test() public void testSingleHourWriteToDisk() { UptimeStatus uptimeStatus = new UptimeStatus(); - uptimeStatus.addToHistory(true, new TreeSet<Long>(Arrays.asList( - new Long[] { DateTimeHelper.parse("2013-12-20 00:00:00") }))); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2013-12-20 00:00:00"), null); uptimeStatus.compressHistory(); assertTrue("Changed uptime status should say it's dirty.", uptimeStatus.isDirty()); @@ -47,9 +45,10 @@ public class UptimeStatusTest { @Test() public void testTwoConsecutiveHours() { UptimeStatus uptimeStatus = new UptimeStatus(); - uptimeStatus.addToHistory(true, new TreeSet<Long>(Arrays.asList( - new Long[] { DateTimeHelper.parse("2013-12-20 00:00:00"), - DateTimeHelper.parse("2013-12-20 01:00:00") }))); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2013-12-20 00:00:00"), null); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2013-12-20 01:00:00"), null); uptimeStatus.compressHistory(); assertEquals("History must contain single entry.", 1, uptimeStatus.getRelayHistory().size()); @@ -73,9 +72,10 @@ public class UptimeStatusTest { public void testGabelmooFillInGaps() { UptimeStatus uptimeStatus = new UptimeStatus(); uptimeStatus.setFromDocumentString(RELAY_UPTIME_SAMPLE); - uptimeStatus.addToHistory(true, new TreeSet<Long>(Arrays.asList( - new Long[] { DateTimeHelper.parse("2013-09-09 02:00:00"), - DateTimeHelper.parse("2013-12-20 00:00:00") }))); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2013-09-09 02:00:00"), null); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2013-12-20 00:00:00"), null); assertEquals("Uncompressed history must contain five entries.", 5, uptimeStatus.getRelayHistory().size()); uptimeStatus.compressHistory(); @@ -96,8 +96,8 @@ public class UptimeStatusTest { public void testAddExistingHourToIntervalStart() { UptimeStatus uptimeStatus = new UptimeStatus(); uptimeStatus.setFromDocumentString(RELAY_UPTIME_SAMPLE); - uptimeStatus.addToHistory(true, new TreeSet<Long>(Arrays.asList( - new Long[] { DateTimeHelper.parse("2013-07-22 17:00:00") }))); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2013-07-22 17:00:00"), null); uptimeStatus.compressHistory(); assertFalse("Unchanged history should not make uptime status dirty.", uptimeStatus.isDirty()); @@ -107,8 +107,8 @@ public class UptimeStatusTest { public void testAddExistingHourToIntervalEnd() { UptimeStatus uptimeStatus = new UptimeStatus(); uptimeStatus.setFromDocumentString(RELAY_UPTIME_SAMPLE); - uptimeStatus.addToHistory(true, new TreeSet<Long>(Arrays.asList( - new Long[] { DateTimeHelper.parse("2013-09-09 01:00:00") }))); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2013-09-09 01:00:00"), null); uptimeStatus.compressHistory(); assertFalse("Unchanged history should not make uptime status dirty.", uptimeStatus.isDirty()); @@ -118,9 +118,10 @@ public class UptimeStatusTest { public void testTwoHoursOverlappingWithIntervalStart() { UptimeStatus uptimeStatus = new UptimeStatus(); uptimeStatus.setFromDocumentString(RELAY_UPTIME_SAMPLE); - uptimeStatus.addToHistory(true, new TreeSet<Long>(Arrays.asList( - new Long[] { DateTimeHelper.parse("2013-07-22 16:00:00"), - DateTimeHelper.parse("2013-07-22 17:00:00")}))); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2013-07-22 16:00:00"), null); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2013-07-22 17:00:00"), null); uptimeStatus.compressHistory(); assertEquals("Compressed history must still contain three entries.", 3, uptimeStatus.getRelayHistory().size()); @@ -139,9 +140,10 @@ public class UptimeStatusTest { public void testTwoHoursOverlappingWithIntervalEnd() { UptimeStatus uptimeStatus = new UptimeStatus(); uptimeStatus.setFromDocumentString(RELAY_UPTIME_SAMPLE); - uptimeStatus.addToHistory(true, new TreeSet<Long>(Arrays.asList( - new Long[] { DateTimeHelper.parse("2013-09-09 01:00:00"), - DateTimeHelper.parse("2013-09-09 02:00:00")}))); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2013-09-09 01:00:00"), null); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2013-09-09 02:00:00"), null); uptimeStatus.compressHistory(); assertEquals("Compressed history must now contain two entries.", 2, uptimeStatus.getRelayHistory().size()); @@ -164,9 +166,10 @@ public class UptimeStatusTest { public void testAddRelayUptimeHours() { UptimeStatus uptimeStatus = new UptimeStatus(); uptimeStatus.setFromDocumentString(RELAYS_AND_BRIDGES_UPTIME_SAMPLE); - uptimeStatus.addToHistory(true, new TreeSet<Long>(Arrays.asList( - new Long[] { DateTimeHelper.parse("2013-07-22 16:00:00"), - DateTimeHelper.parse("2014-03-21 20:00:00")}))); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2013-07-22 16:00:00"), null); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2014-03-21 20:00:00"), null); uptimeStatus.compressHistory(); assertEquals("Compressed relay history must still contain one entry.", 1, uptimeStatus.getRelayHistory().size()); @@ -185,9 +188,10 @@ public class UptimeStatusTest { public void testAddBridgeUptimeHours() { UptimeStatus uptimeStatus = new UptimeStatus(); uptimeStatus.setFromDocumentString(RELAYS_AND_BRIDGES_UPTIME_SAMPLE); - uptimeStatus.addToHistory(false, new TreeSet<Long>(Arrays.asList( - new Long[] { DateTimeHelper.parse("2013-07-22 16:00:00"), - DateTimeHelper.parse("2014-03-21 20:00:00")}))); + uptimeStatus.addToHistory(false, + DateTimeHelper.parse("2013-07-22 16:00:00"), null); + uptimeStatus.addToHistory(false, + DateTimeHelper.parse("2014-03-21 20:00:00"), null); uptimeStatus.compressHistory(); assertEquals("Compressed bridge history must still contain one " + "entry.", 1, uptimeStatus.getBridgeHistory().size()); @@ -201,5 +205,70 @@ public class UptimeStatusTest { assertEquals("History uptime hours not 1+5811+1=5813.", 5813, newUptimeHistory.getUptimeHours()); } + + private static final SortedSet<String> RUNNING_FLAG = + new TreeSet<String>(Arrays.asList(new String[] { "Running" })); + + @Test() + public void testAddFlagsToNoFlagsEnd() { + UptimeStatus uptimeStatus = new UptimeStatus(); + uptimeStatus.setFromDocumentString(RELAYS_AND_BRIDGES_UPTIME_SAMPLE); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2014-03-21 20:00:00"), RUNNING_FLAG); + uptimeStatus.compressHistory(); + assertEquals("Mixed relay history must not be compressed.", 2, + uptimeStatus.getRelayHistory().size()); + } + + @Test() + public void testAddFlagsToNoFlagsBegin() { + UptimeStatus uptimeStatus = new UptimeStatus(); + uptimeStatus.setFromDocumentString(RELAYS_AND_BRIDGES_UPTIME_SAMPLE); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2013-07-22 16:00:00"), RUNNING_FLAG); + uptimeStatus.compressHistory(); + assertEquals("Mixed relay history must not be compressed.", 2, + uptimeStatus.getRelayHistory().size()); + } + + @Test() + public void testAddFlagsToNoFlagsMiddle() { + UptimeStatus uptimeStatus = new UptimeStatus(); + uptimeStatus.setFromDocumentString(RELAYS_AND_BRIDGES_UPTIME_SAMPLE); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2013-09-20 12:00:00"), RUNNING_FLAG); + uptimeStatus.compressHistory(); + assertEquals("Mixed relay history must not be compressed.", 3, + uptimeStatus.getRelayHistory().size()); + } + + private static final String RELAYS_FLAGS_UPTIME_SAMPLE = + "R 2013-07-22-17 5811 Running\n"; /* ends 2014-03-21 20:00:00 */ + + @Test() + public void testAddFlagsToFlagsEnd() { + UptimeStatus uptimeStatus = new UptimeStatus(); + uptimeStatus.setFromDocumentString(RELAYS_FLAGS_UPTIME_SAMPLE); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2014-03-21 20:00:00"), RUNNING_FLAG); + uptimeStatus.compressHistory(); + assertEquals("Relay history with flags must be compressed.", 1, + uptimeStatus.getRelayHistory().size()); + } + + private static final SortedSet<String> RUNNING_VALID_FLAGS = + new TreeSet<String>(Arrays.asList(new String[] { "Running", + "Valid" })); + + @Test() + public void testDontCompressDifferentFlags() { + UptimeStatus uptimeStatus = new UptimeStatus(); + uptimeStatus.setFromDocumentString(RELAYS_FLAGS_UPTIME_SAMPLE); + uptimeStatus.addToHistory(true, + DateTimeHelper.parse("2014-03-21 20:00:00"), RUNNING_VALID_FLAGS); + uptimeStatus.compressHistory(); + assertEquals("Relay history with different flags must not be " + + "compressed.", 2, uptimeStatus.getRelayHistory().size()); + } } diff --git a/src/test/java/org/torproject/onionoo/updater/DummyConsensus.java b/src/test/java/org/torproject/onionoo/updater/DummyConsensus.java index 8f164a0..a2d1e3e 100644 --- a/src/test/java/org/torproject/onionoo/updater/DummyConsensus.java +++ b/src/test/java/org/torproject/onionoo/updater/DummyConsensus.java @@ -6,6 +6,7 @@ import java.util.List; import java.util.SortedMap; import java.util.SortedSet; import java.util.TreeMap; +import java.util.TreeSet; import org.torproject.descriptor.DirSourceEntry; import org.torproject.descriptor.DirectorySignature; @@ -70,8 +71,12 @@ public class DummyConsensus implements RelayNetworkStatusConsensus { return null; } + private SortedSet<String> knownFlags = new TreeSet<String>(); + public void addKnownFlag(String flag) { + this.knownFlags.add(flag); + } public SortedSet<String> getKnownFlags() { - return null; + return this.knownFlags; } public SortedMap<String, Integer> getConsensusParams() { diff --git a/src/test/java/org/torproject/onionoo/updater/UptimeStatusUpdaterTest.java b/src/test/java/org/torproject/onionoo/updater/UptimeStatusUpdaterTest.java index e74dc5c..25399ab 100644 --- a/src/test/java/org/torproject/onionoo/updater/UptimeStatusUpdaterTest.java +++ b/src/test/java/org/torproject/onionoo/updater/UptimeStatusUpdaterTest.java @@ -52,6 +52,7 @@ public class UptimeStatusUpdaterTest { statusEntry.addFlag("Running"); DummyConsensus consensus = new DummyConsensus(); consensus.setValidAfterMillis(VALID_AFTER_SAMPLE); + consensus.addKnownFlag("Running"); consensus.addStatusEntry(statusEntry); this.descriptorSource.addDescriptor(DescriptorType.RELAY_CONSENSUSES, consensus); @@ -83,7 +84,7 @@ public class UptimeStatusUpdaterTest { private static final String ALL_RELAYS_AND_BRIDGES_FINGERPRINT = null; private static final String ALL_RELAYS_AND_BRIDGES_UPTIME_SAMPLE = - "r 2013-07-22-17 5811\n" /* ends 2014-03-21 20:00:00 */ + "R 2013-07-22-17 5811 Running\n" /* ends 2014-03-21 20:00:00 */ + "b 2013-07-22-17 5811\n"; /* ends 2014-03-21 20:00:00 */ private void addAllRelaysAndBridgesUptimeSample() { @@ -105,7 +106,7 @@ public class UptimeStatusUpdaterTest { 2, this.documentStore.getPerformedStoreOperations()); UptimeStatus status = this.documentStore.getDocument( UptimeStatus.class, ALL_RELAYS_AND_BRIDGES_FINGERPRINT); - assertEquals("Relay history must contain one entry", 1, + assertEquals("Relay history must contain one entry.", 1, status.getRelayHistory().size()); UptimeHistory history = status.getRelayHistory().first(); assertEquals("History not for relay.", true, history.isRelay()); diff --git a/web/protocol.html b/web/protocol.html index 2a13be8..c1dfa3f 100644 --- a/web/protocol.html +++ b/web/protocol.html @@ -174,6 +174,8 @@ field from details documents and optional "advertised_bandwidth" and <li>2.2: Removed optional "pool_assignment" field and added "transports" field to bridge details documents on December 8, 2014.</li> +<li>2.3: Added optional "flags" field to uptime +documents on March 22, 2015.</li> </ul> </div>  @@ -2206,6 +2208,23 @@ of network statuses have been processed for a given time period. </li> +<li> +flags +<code class="typeof">object</code> +optional + +Object containing fractional times of this relay having relay flags +assigned. +Keys are flag names like "Running" or +"Exit", values are objects similar to the +uptime field above, again with keys like +"1_week" etc. +If a relay never had a given relay flag assigned, no object is included +for that flag. +Added on March 22, 2015. + +</li> + </ul> <h4>Bridge uptime objects</h4>

1 0

[onionoo/master] Update Vagrant file to reflect switch to Jetty.
by karsten＠torproject.org 27 Mar '15

27 Mar '15

commit b5fcd049e6d58118435442be2dad326f798d0bf3 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Mar 27 08:41:55 2015 +0100 Update Vagrant file to reflect switch to Jetty. This update is long overdue. --- vagrant/bootstrap.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/vagrant/bootstrap.sh b/vagrant/bootstrap.sh index ad4507a..13138cc 100644 --- a/vagrant/bootstrap.sh +++ b/vagrant/bootstrap.sh @@ -13,12 +13,11 @@ echo "Installing required packages." apt-get install -y openjdk-7-jdk apt-get install -y libcommons-codec-java libcommons-compress-java \ libcommons-lang3-java libgoogle-gson-java junit4 libservlet3.0-java \ -ant tomcat6 libslf4j-java liblogback-java unzip +ant libslf4j-java liblogback-java unzip libjetty8-java echo "Setting up paths and creating symbolic links." mkdir -p /srv/onionoo.torproject.org/onionoo/ cd /srv/onionoo.torproject.org/onionoo/ -ln -s /vagrant/bin ln -s /vagrant/build.xml ln -s /vagrant/deps ln -s /vagrant/etc

1 0

[translation/gettor] Update translations for gettor
by translation＠torproject.org 27 Mar '15

27 Mar '15

commit 14af7a06590546631987e3c186979574a9ac7784 Author: Translation commit bot <translation(a)torproject.org> Date: Fri Mar 27 07:15:06 2015 +0000 Update translations for gettor --- bn_IN/gettor.po | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bn_IN/gettor.po b/bn_IN/gettor.po index d06378f..5b1ae93 100644 --- a/bn_IN/gettor.po +++ b/bn_IN/gettor.po @@ -9,7 +9,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2013-01-19 13:40+0100\n" -"PO-Revision-Date: 2015-03-27 06:43+0000\n" +"PO-Revision-Date: 2015-03-27 06:51+0000\n" "Last-Translator: Rajarshi Bhowmick <raz.bhowmick(a)gmail.com>\n" "Language-Team: Bengali (India) (http://www.transifex.com/projects/p/torproject/language/bn_IN/)\n" "MIME-Version: 1.0\n" @@ -81,7 +81,7 @@ msgstr "" msgid "" "OBTAINING LOCALIZED VERSIONS OF TOR\n" "===================================" -msgstr "" +msgstr " 'টর' স্থানীয় সংস্করণে পাওয়ার চেষ্টা চলছে\n===================================" #: lib/gettor/i18n.py:67 msgid ""

1 0