July 2014 - tor-commits - lists.torproject.org

[tor-browser-bundle/maint-3.6] Bug 12381: Added new obfsproxy tag.
by gk＠torproject.org 22 Jul '14

22 Jul '14

commit 3ecbd771e26aaa36f3603be6196d90d245c54e41 Author: Georg Koppen <gk(a)torproject.org> Date: Tue Jul 22 19:26:48 2014 +0000 Bug 12381: Added new obfsproxy tag. --- gitian/versions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitian/versions b/gitian/versions index 17e2e37..c0e5047 100755 --- a/gitian/versions +++ b/gitian/versions @@ -16,7 +16,7 @@ ZLIB_TAG=v1.2.8 LIBEVENT_TAG=release-2.0.21-stable MINGW_REV=6184 PYPTLIB_TAG=pyptlib-0.0.6 -OBFSPROXY_TAG=obfsproxy-0.2.9 +OBFSPROXY_TAG=obfsproxy-0.2.12 FLASHPROXY_TAG=1.6 LIBFTE_TAG=ee9e9ddf5c86e6940559a313d2bd22cc33b654c9 # tag 0.0.3 FTEPROXY_TAG=b1b863534b25a6c95b0a5eac76f8cb8be6266f88 # tag 0.2.17

1 0

[tor-browser-bundle/maint-3.6] Bug 12381, 12673: Squashed + merged kpdyer/master.
by gk＠torproject.org 22 Jul '14

22 Jul '14

commit e0e68e5713f6db5ca2756f7082094cb421c067a2 Author: Georg Koppen <gk(a)torproject.org> Date: Tue Jul 22 19:22:08 2014 +0000 Bug 12381, 12673: Squashed + merged kpdyer/master. This hopefully fixes fte over proxy support (12381) and adds new fte bridges (12673). --- Bundle-Data/PTConfigs/bridge_prefs.js | 22 +++++++++++++++++----- gitian/versions | 2 +- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/Bundle-Data/PTConfigs/bridge_prefs.js b/Bundle-Data/PTConfigs/bridge_prefs.js index 6d8ac38..9cb8a6a 100644 --- a/Bundle-Data/PTConfigs/bridge_prefs.js +++ b/Bundle-Data/PTConfigs/bridge_prefs.js @@ -16,11 +16,23 @@ pref("extensions.torlauncher.default_bridge.flashproxy.3", "flashproxy 0.0.1.0:3 pref("extensions.torlauncher.default_bridge.flashproxy.4", "flashproxy 0.0.1.0:4"); pref("extensions.torlauncher.default_bridge.flashproxy.5", "flashproxy 0.0.1.0:5"); -pref("extensions.torlauncher.default_bridge.fte.1", "fte 128.105.214.161:8080 1E326AAFB3FCB515015250D8FCCC8E37F91A153B"); -pref("extensions.torlauncher.default_bridge.fte.2", "fte 128.105.214.162:8080 FC562097E1951DCC41B7D7F324D88157119BB56D"); -pref("extensions.torlauncher.default_bridge.fte.3", "fte 128.105.214.163:8080 A17A40775FBD2CA1184BF80BFC330A77ECF9D0E9"); -pref("extensions.torlauncher.default_bridge.fte.4", "fte 131.252.210.150:8080 0E858AC201BF0F3FA3C462F64844CBFFC7297A42"); -pref("extensions.torlauncher.default_bridge.fte.5", "fte 79.125.3.12:8080 272465348803EE2546A9BB8EE37D462915531F09"); +// <ipv4> +pref("extensions.torlauncher.default_bridge.fte.1", "fte t5fPOQM3-1097909862.us-west-2.elb.amazonaws.com:80 6E6298B85B171FEA4E36ADB15D6FF95DA0B37C89"); +pref("extensions.torlauncher.default_bridge.fte.2", "fte 54.244.94.118:80 6E6298B85B171FEA4E36ADB15D6FF95DA0B37C89"); +pref("extensions.torlauncher.default_bridge.fte.3", "fte 131.252.210.150:8080 0E858AC201BF0F3FA3C462F64844CBFFC7297A42"); +pref("extensions.torlauncher.default_bridge.fte.4", "fte server-01.fteproxy.net:8080 0E858AC201BF0F3FA3C462F64844CBFFC7297A42"); +pref("extensions.torlauncher.default_bridge.fte.5", "fte 128.105.214.161:8080 1E326AAFB3FCB515015250D8FCCC8E37F91A153B"); +pref("extensions.torlauncher.default_bridge.fte.6", "fte server-02.fteproxy.net:8080 1E326AAFB3FCB515015250D8FCCC8E37F91A153B"); +pref("extensions.torlauncher.default_bridge.fte.7", "fte 128.105.214.162:8080 FC562097E1951DCC41B7D7F324D88157119BB56D"); +pref("extensions.torlauncher.default_bridge.fte.8", "fte server-03.fteproxy.net:8080 FC562097E1951DCC41B7D7F324D88157119BB56D"); +pref("extensions.torlauncher.default_bridge.fte.9", "fte 128.105.214.163:8080 A17A40775FBD2CA1184BF80BFC330A77ECF9D0E9"); +pref("extensions.torlauncher.default_bridge.fte.10", "fte server-04.fteproxy.net:8080 A17A40775FBD2CA1184BF80BFC330A77ECF9D0E9"); +pref("extensions.torlauncher.default_bridge.fte.11", "fte 79.125.3.12:8080 272465348803EE2546A9BB8EE37D462915531F09"); +pref("extensions.torlauncher.default_bridge.fte.12", "fte server-05.fteproxy.net:8080 272465348803EE2546A9BB8EE37D462915531F09"); +// </ipv4> +// <ipv6> +pref("extensions.torlauncher.default_bridge.fte.13", "fte ipv6.t5fPOQM3-1097909862.us-west-2.elb.amazonaws.com:80 6E6298B85B171FEA4E36ADB15D6FF95DA0B37C89"); +// </ipv6> //pref("extensions.torlauncher.default_bridge.scramblesuit.1", "scramblesuit 188.40.121.112:39707 5DE8D363D8F150C99E1A2D7237368D614838132C password=L5POGQONBPS2HZUR6GXBIDS4CMIYYOTI"); //pref("extensions.torlauncher.default_bridge.scramblesuit.2", "scramblesuit 188.226.213.208:54278 AA5A86C1490296EF4FACA946CC5A182FCD1C5B1E password=MD2VRP7WXAMSG7MKIGMHI4CB4BMSNO7T"); diff --git a/gitian/versions b/gitian/versions index a8a6fb7..17e2e37 100755 --- a/gitian/versions +++ b/gitian/versions @@ -19,7 +19,7 @@ PYPTLIB_TAG=pyptlib-0.0.6 OBFSPROXY_TAG=obfsproxy-0.2.9 FLASHPROXY_TAG=1.6 LIBFTE_TAG=ee9e9ddf5c86e6940559a313d2bd22cc33b654c9 # tag 0.0.3 -FTEPROXY_TAG=5e7a9fd498a948d17b0996275ef1b6f743251317 # tag 0.2.15 +FTEPROXY_TAG=b1b863534b25a6c95b0a5eac76f8cb8be6266f88 # tag 0.2.17 LIBDMG_TAG=dfd5e5cc3dc1191e37d3c3a6118975afdd1d7014 TXSOCKSX_TAG=216eb0894a1755872f4789f9458aa6cf543b8433 # unsigned habnabit/1.13.0.2

1 0

[tor-browser-bundle/master] Bug 12381, 12673: Squashed + merged kpdyer/master.
by gk＠torproject.org 22 Jul '14

22 Jul '14

commit 3135c79f586cc635fe6c8ab88b800ec9065dff90 Author: Georg Koppen <gk(a)torproject.org> Date: Tue Jul 22 19:04:32 2014 +0000 Bug 12381, 12673: Squashed + merged kpdyer/master. This hopefully fixes fte over proxy support (12381) and adds new fte bridges (12673). --- Bundle-Data/PTConfigs/bridge_prefs.js | 22 +++++++++++++++++----- gitian/versions | 2 +- gitian/versions.alpha | 2 +- gitian/versions.beta | 2 +- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/Bundle-Data/PTConfigs/bridge_prefs.js b/Bundle-Data/PTConfigs/bridge_prefs.js index 5a1532e..6ee714d 100644 --- a/Bundle-Data/PTConfigs/bridge_prefs.js +++ b/Bundle-Data/PTConfigs/bridge_prefs.js @@ -16,11 +16,23 @@ pref("extensions.torlauncher.default_bridge.flashproxy.3", "flashproxy 0.0.1.0:3 pref("extensions.torlauncher.default_bridge.flashproxy.4", "flashproxy 0.0.1.0:4"); pref("extensions.torlauncher.default_bridge.flashproxy.5", "flashproxy 0.0.1.0:5"); -pref("extensions.torlauncher.default_bridge.fte.1", "fte 128.105.214.161:8080 1E326AAFB3FCB515015250D8FCCC8E37F91A153B"); -pref("extensions.torlauncher.default_bridge.fte.2", "fte 128.105.214.162:8080 FC562097E1951DCC41B7D7F324D88157119BB56D"); -pref("extensions.torlauncher.default_bridge.fte.3", "fte 128.105.214.163:8080 A17A40775FBD2CA1184BF80BFC330A77ECF9D0E9"); -pref("extensions.torlauncher.default_bridge.fte.4", "fte 131.252.210.150:8080 0E858AC201BF0F3FA3C462F64844CBFFC7297A42"); -pref("extensions.torlauncher.default_bridge.fte.5", "fte 79.125.3.12:8080 272465348803EE2546A9BB8EE37D462915531F09"); +// <ipv4> +pref("extensions.torlauncher.default_bridge.fte.1", "fte t5fPOQM3-1097909862.us-west-2.elb.amazonaws.com:80 6E6298B85B171FEA4E36ADB15D6FF95DA0B37C89"); +pref("extensions.torlauncher.default_bridge.fte.2", "fte 54.244.94.118:80 6E6298B85B171FEA4E36ADB15D6FF95DA0B37C89"); +pref("extensions.torlauncher.default_bridge.fte.3", "fte 131.252.210.150:8080 0E858AC201BF0F3FA3C462F64844CBFFC7297A42"); +pref("extensions.torlauncher.default_bridge.fte.4", "fte server-01.fteproxy.net:8080 0E858AC201BF0F3FA3C462F64844CBFFC7297A42"); +pref("extensions.torlauncher.default_bridge.fte.5", "fte 128.105.214.161:8080 1E326AAFB3FCB515015250D8FCCC8E37F91A153B"); +pref("extensions.torlauncher.default_bridge.fte.6", "fte server-02.fteproxy.net:8080 1E326AAFB3FCB515015250D8FCCC8E37F91A153B"); +pref("extensions.torlauncher.default_bridge.fte.7", "fte 128.105.214.162:8080 FC562097E1951DCC41B7D7F324D88157119BB56D"); +pref("extensions.torlauncher.default_bridge.fte.8", "fte server-03.fteproxy.net:8080 FC562097E1951DCC41B7D7F324D88157119BB56D"); +pref("extensions.torlauncher.default_bridge.fte.9", "fte 128.105.214.163:8080 A17A40775FBD2CA1184BF80BFC330A77ECF9D0E9"); +pref("extensions.torlauncher.default_bridge.fte.10", "fte server-04.fteproxy.net:8080 A17A40775FBD2CA1184BF80BFC330A77ECF9D0E9"); +pref("extensions.torlauncher.default_bridge.fte.11", "fte 79.125.3.12:8080 272465348803EE2546A9BB8EE37D462915531F09"); +pref("extensions.torlauncher.default_bridge.fte.12", "fte server-05.fteproxy.net:8080 272465348803EE2546A9BB8EE37D462915531F09"); +// </ipv4> +// <ipv6> +pref("extensions.torlauncher.default_bridge.fte.13", "fte ipv6.t5fPOQM3-1097909862.us-west-2.elb.amazonaws.com:80 6E6298B85B171FEA4E36ADB15D6FF95DA0B37C89"); +// </ipv6> pref("extensions.torlauncher.default_bridge.scramblesuit.1", "scramblesuit 188.40.121.112:39707 5DE8D363D8F150C99E1A2D7237368D614838132C password=L5POGQONBPS2HZUR6GXBIDS4CMIYYOTI"); pref("extensions.torlauncher.default_bridge.scramblesuit.2", "scramblesuit 188.226.213.208:54278 AA5A86C1490296EF4FACA946CC5A182FCD1C5B1E password=MD2VRP7WXAMSG7MKIGMHI4CB4BMSNO7T"); diff --git a/gitian/versions b/gitian/versions index 0372cf7..b51c2af 100755 --- a/gitian/versions +++ b/gitian/versions @@ -19,7 +19,7 @@ PYPTLIB_TAG=pyptlib-0.0.6 OBFSPROXY_TAG=obfsproxy-0.2.9 FLASHPROXY_TAG=1.6 LIBFTE_TAG=ee9e9ddf5c86e6940559a313d2bd22cc33b654c9 # tag 0.0.3 -FTEPROXY_TAG=5e7a9fd498a948d17b0996275ef1b6f743251317 # tag 0.2.15 +FTEPROXY_TAG=b1b863534b25a6c95b0a5eac76f8cb8be6266f88 # tag 0.2.17 LIBDMG_TAG=dfd5e5cc3dc1191e37d3c3a6118975afdd1d7014 TXSOCKSX_TAG=216eb0894a1755872f4789f9458aa6cf543b8433 # unsigned habnabit/1.13.0.2 GOPTLIB_TAG=0.2 diff --git a/gitian/versions.alpha b/gitian/versions.alpha index 6be86bd..e8957a0 100755 --- a/gitian/versions.alpha +++ b/gitian/versions.alpha @@ -19,7 +19,7 @@ PYPTLIB_TAG=pyptlib-0.0.6 OBFSPROXY_TAG=obfsproxy-0.2.9 FLASHPROXY_TAG=1.6 LIBFTE_TAG=ee9e9ddf5c86e6940559a313d2bd22cc33b654c9 # tag 0.0.3 -FTEPROXY_TAG=5e7a9fd498a948d17b0996275ef1b6f743251317 # tag 0.2.15 +FTEPROXY_TAG=b1b863534b25a6c95b0a5eac76f8cb8be6266f88 # tag 0.2.17 LIBDMG_TAG=dfd5e5cc3dc1191e37d3c3a6118975afdd1d7014 TXSOCKSX_TAG=216eb0894a1755872f4789f9458aa6cf543b8433 # unsigned habnabit/1.13.0.2 GOPTLIB_TAG=0.2 diff --git a/gitian/versions.beta b/gitian/versions.beta index aedcf44..2e2754a 100755 --- a/gitian/versions.beta +++ b/gitian/versions.beta @@ -19,7 +19,7 @@ PYPTLIB_TAG=pyptlib-0.0.6 OBFSPROXY_TAG=obfsproxy-0.2.9 FLASHPROXY_TAG=1.6 LIBFTE_TAG=19f6b8ffafca2ec8fffbc418bc0f88518cea22ac # tag 0.0.2 -FTEPROXY_TAG=b5d7fba5c505907693fd2b5321f2aa57d4390cfa # tag 0.2.14 +FTEPROXY_TAG=b1b863534b25a6c95b0a5eac76f8cb8be6266f88 # tag 0.2.17 LIBDMG_TAG=dfd5e5cc3dc1191e37d3c3a6118975afdd1d7014 TXSOCKSX_TAG=216eb0894a1755872f4789f9458aa6cf543b8433 # unsigned habnabit/1.13.0.2 GOPTLIB_TAG=0.2

1 0

[translation/gettor] Update translations for gettor
by translation＠torproject.org 22 Jul '14

22 Jul '14

commit 23a39f6229cdafd899213094f285a9862c4ea60c Author: Translation commit bot <translation(a)torproject.org> Date: Tue Jul 22 16:45:05 2014 +0000 Update translations for gettor --- sk_SK/gettor.po | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sk_SK/gettor.po b/sk_SK/gettor.po index 568da55..7daddcd 100644 --- a/sk_SK/gettor.po +++ b/sk_SK/gettor.po @@ -9,7 +9,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2013-01-19 13:40+0100\n" -"PO-Revision-Date: 2014-07-22 16:01+0000\n" +"PO-Revision-Date: 2014-07-22 16:38+0000\n" "Last-Translator: once <matejbacik(a)gmail.com>\n" "Language-Team: Slovak (Slovakia) (http://www.transifex.com/projects/p/torproject/language/sk_SK/)\n" "MIME-Version: 1.0\n"

1 0

[flashproxy/master] Add some more ChangeLog for 1.7.
by dcf＠torproject.org 22 Jul '14

22 Jul '14

commit 3c4a3b4904120015d6995face2052123f8ed2a55 Author: David Fifield <david(a)bamsoftware.com> Date: Tue Jul 22 09:10:31 2014 -0700 Add some more ChangeLog for 1.7. --- ChangeLog | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ChangeLog b/ChangeLog index 759f7d8..b778a38 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,14 @@ Changes in version 1.7 + o Made the badge color reflect what's going on when it encounters a + network error and tries to reconnect. Fixes bug 11400. + + o Renamed facilitator programs: + facilitator → fp-facilitator + facilitator.cgi → fp-registrar.cgi + facilitator-email-poller → fp-registrar-email + facilitator-reg → fp-reg-decrypt + facilitator-reg-daemon → fp-reg-decryptd + o Fixed a bug in the browser proxy which caused it to stop accepting new connections once it had failed 5 previous connections.

1 0

[flashproxy/master] Increase version number to 1.7.
by dcf＠torproject.org 22 Jul '14

22 Jul '14

commit a3fe8c49c679051d90adb04e7d5aae51fcee9c72 Author: David Fifield <david(a)bamsoftware.com> Date: Tue Jul 22 08:58:41 2014 -0700 Increase version number to 1.7. --- ChangeLog | 1 + facilitator/configure.ac | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 18582ce..759f7d8 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,4 @@ +Changes in version 1.7 o Fixed a bug in the browser proxy which caused it to stop accepting new connections once it had failed 5 previous connections. diff --git a/facilitator/configure.ac b/facilitator/configure.ac index 3c034fb..2f56c1a 100644 --- a/facilitator/configure.ac +++ b/facilitator/configure.ac @@ -1,5 +1,5 @@ AC_PREREQ([2.68]) -AC_INIT([flashproxy-facilitator], [1.6]) +AC_INIT([flashproxy-facilitator], [1.7]) AM_INIT_AUTOMAKE([-Wall foreign]) AC_ARG_VAR(fpfacilitatoruser, [the user/group for the facilitator to run as])

1 0

r26884: {website} Remove duplication about outgoing firewalls by splitting use (website/trunk/docs/en)
by Lunar 22 Jul '14

22 Jul '14

Author: lunar Date: 2014-07-22 16:24:08 +0000 (Tue, 22 Jul 2014) New Revision: 26884 Modified: website/trunk/docs/en/faq.wml Log: Remove duplication about outgoing firewalls by splitting uses cases between client and relay Modified: website/trunk/docs/en/faq.wml =================================================================== --- website/trunk/docs/en/faq.wml 2014-07-22 13:46:17 UTC (rev 26883) +++ website/trunk/docs/en/faq.wml 2014-07-22 16:24:08 UTC (rev 26884) @@ -48,8 +48,8 @@ <li><a href="#IsItWorking">How can I tell if Tor is working, and that my connections really are anonymized?</a></li> <li><a href="#Mobile">Can I use Tor on my phone or mobile device?</a></li> - <li><a href="#OutboundPorts">Do I have to open all these outbound ports - on my firewall?</a></li> + <li><a href="#OutboundPorts">Which outbound ports must be open when + using Tor as a client?</a></li> <li><a href="#FTP">How do I use my browser for ftp with Tor?</a></li> <li><a href="#NoDataScrubbing">Does Tor remove personal information from the data my application sends?</a></li> @@ -882,10 +882,9 @@ <hr> - <a id="OutboundPorts"></a> - <h3><a class="anchor" href="#OutboundPorts">Do I have to open all these - outbound ports on my firewall?</a></h3> - + <a id="OutboundPorts"></a> + <h3><a class="anchor" href="#OutboundPorts">Which outbound ports must be open when + using Tor as a client?</a></h3> <p> Tor may attempt to connect to any port that is advertised in the directory as an ORPort (for making Tor connections) or a DirPort (for @@ -894,7 +893,7 @@ ports too. </p> <p> - As a client: you could probably get away with opening only those four + When using Tor as a client, you could probably get away with opening only those four ports. Since Tor does all its connections in the background, it will retry ones that fail, and hopefully you'll never have to know that it failed, as long as it finds a working one often enough. However, to get the most @@ -905,14 +904,6 @@ you want to explicitly tell your Tor client which ports are reachable for you. </p> - <p> - As a relay: you must allow outgoing connections to every other relay - and to anywhere your exit policy advertises that you allow. The - cleanest way to do that is simply to allow all outgoing connections - at your firewall. If you don't, clients will ask you to extend to - those relays, and those connections will fail, leading to complex - anonymity implications for the clients which we'd like to avoid. - </p> <hr> @@ -2334,7 +2325,7 @@ <hr> <a id="OutgoingFirewall"></a> - <h3><a class="anchor" href="#BandwidthShaping">How should I configure + <h3><a class="anchor" href="#OutgoingFirewall">How should I configure my outgoing filters?</a></h3> <p>

1 0

[translation/gettor] Update translations for gettor
by translation＠torproject.org 22 Jul '14

22 Jul '14

commit 01cb8d514ce43d296c63ee6e54b7619fb4ba7b74 Author: Translation commit bot <translation(a)torproject.org> Date: Tue Jul 22 16:15:05 2014 +0000 Update translations for gettor --- sk_SK/gettor.po | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/sk_SK/gettor.po b/sk_SK/gettor.po index 7dd4406..568da55 100644 --- a/sk_SK/gettor.po +++ b/sk_SK/gettor.po @@ -9,7 +9,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2013-01-19 13:40+0100\n" -"PO-Revision-Date: 2014-07-22 15:43+0000\n" +"PO-Revision-Date: 2014-07-22 16:01+0000\n" "Last-Translator: once <matejbacik(a)gmail.com>\n" "Language-Team: Slovak (Slovakia) (http://www.transifex.com/projects/p/torproject/language/sk_SK/)\n" "MIME-Version: 1.0\n" @@ -136,7 +136,7 @@ msgid "" "If your bandwith is low or your provider doesn't allow you to\n" "receive large attachments in your email, GetTor can send you several\n" "small packages instead of one big one." -msgstr "" +msgstr "Ak je váš prenosový rozsah nízky alebo váš poskytovateľ nepovoľuje veľké prílohy v e-mailoch, GetTor vám môže poslať viacero malých balíčkov namiesto jedného veľkého." #: lib/gettor/i18n.py:101 msgid "" @@ -145,7 +145,7 @@ msgid "" " \n" " windows\n" " split" -msgstr "" +msgstr "Stačí len uviesť kľúčové slovo 'split' na samostatnom novom riadku (toto \nje dôležíté) tak ako v príklade:\n\nwindows\nsplit" #: lib/gettor/i18n.py:107 msgid "" @@ -198,13 +198,13 @@ msgstr "6.) Hotovo. To je všetko. Ďakujeme, že používate Tor a prajeme veľ msgid "" "SUPPORT\n" "=======" -msgstr "" +msgstr "PODPORA\n=======" #: lib/gettor/i18n.py:138 msgid "" "Here's your requested software as a zip file. Please unzip the\n" "package and verify the signature." -msgstr "" +msgstr "Tu je vami požadovaný program v súbore zip. Prosím, rozbaľte balíček\na overte jeho podpis." #: lib/gettor/i18n.py:141 msgid "" @@ -214,14 +214,14 @@ msgid "" "tool as follows after unpacking the zip file:\n" "\n" " gpg --verify tor-browser-1.3.24_en-US.exe.asc tor-browser-1.3.24_en-US.exe" -msgstr "" +msgstr "VERIFY SIGNATURE\n================\nAk má váš počítač nainštalované GnuPG, použite po rozbalení príkazový riadok pre gpg tak ako je uvedené:\n\n gpg --verify tor-browser-1.3.24_en-US.exe.asc tor-browser-1.3.24_en-US.exe" #: lib/gettor/i18n.py:148 msgid "" "The output should look somewhat like this:\n" "\n" " gpg: Good signature from 'Erinn Clark <...>'" -msgstr "" +msgstr "Výstup by mal vyzerať asi takto:\n\n gpg: Good signature from 'Erinn Clark <...>'" #: lib/gettor/i18n.py:152 msgid "" @@ -229,13 +229,13 @@ msgid "" "a graphical user interface for GnuPG on this website:\n" "\n" " http://www.gnupg.org/related_software/frontends.html" -msgstr "" +msgstr "Ak neovládate nástroje príkazového riadka, rozhliadnite sa po grafickom užívateľskom rozhraní pre GnuPG na jeho stránkach:\n\n http://www.gnupg.org/related_software/frontends.html" #: lib/gettor/i18n.py:157 msgid "" "BLOCKED ACCESS / CENSORSHIP\n" "===========================" -msgstr "" +msgstr "BLOKOVANÝ PRÍSTUP / CENZÚRA\n===========================" #: lib/gettor/i18n.py:160 msgid ""

1 0

[translation/gettor] Update translations for gettor
by translation＠torproject.org 22 Jul '14

22 Jul '14

commit 9be51765dd0fa3e0cca623277efddb7e0e6b7934 Author: Translation commit bot <translation(a)torproject.org> Date: Tue Jul 22 15:45:10 2014 +0000 Update translations for gettor --- sk_SK/gettor.po | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/sk_SK/gettor.po b/sk_SK/gettor.po index 3c352ac..7dd4406 100644 --- a/sk_SK/gettor.po +++ b/sk_SK/gettor.po @@ -9,7 +9,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2013-01-19 13:40+0100\n" -"PO-Revision-Date: 2014-07-22 15:14+0000\n" +"PO-Revision-Date: 2014-07-22 15:43+0000\n" "Last-Translator: once <matejbacik(a)gmail.com>\n" "Language-Team: Slovak (Slovakia) (http://www.transifex.com/projects/p/torproject/language/sk_SK/)\n" "MIME-Version: 1.0\n" @@ -31,7 +31,7 @@ msgid "" "Unfortunately, we won't answer you at this address. You should make\n" "an account with GMAIL.COM, YAHOO.COM or YAHOO.CN and send the mail from\n" "one of those." -msgstr "" +msgstr "Bohužiaľ, nedokážeme vám na tejto adrese odpovedať. Mali by ste si vytvoriť účet\nna GMAIL.COM, YAHOO.COM alebo YAHOO.CN a z neho odoslať e-mail." #: lib/gettor/i18n.py:35 msgid "" @@ -51,7 +51,7 @@ msgstr "(Ospravedlňujeme sa, ak ste o tento e-mail nežiadal vy. Keďže váš\ msgid "" "If you have any questions or it doesn't work, you can contact a\n" "human at this support email address: help(a)rt.torproject.org" -msgstr "" +msgstr "Ak máte nejaké otázky alebo niečo nefunguje, môžete kontaktovať\nživú osobu na tejto adrese e-mailovej podpory: help(a)rt.torproject.org" #: lib/gettor/i18n.py:46 msgid "" @@ -69,19 +69,19 @@ msgid "" " obfs-linux-i386\n" " obfs-linux-x86_64\n" " source" -msgstr "" +msgstr "Odošlem vám balíček Tor, stačí povedať, ktorý chcete.\nVyberte si, prosím, jeden z nasledujúcich názvov balíčkov:\n\n windows\n macos-i386\n macos-ppc\n linux-i386\n linux-x86_64\n obfs-windows\n obfs-macos-i386\n obfs-macos-x86_64\n obfs-linux-i386\n obfs-linux-x86_64\n source" #: lib/gettor/i18n.py:61 msgid "" "Please reply to this mail, and tell me a single package name anywhere \n" "in the body of your email." -msgstr "" +msgstr "Odpovedzte, prosím, na tento e-mail a hocikde v obsahu vašej odpovede\nmi napíšte názov jedného balíčka." #: lib/gettor/i18n.py:64 msgid "" "OBTAINING LOCALIZED VERSIONS OF TOR\n" "===================================" -msgstr "" +msgstr "ZÍSKANIE LOKALIZOVANEJ VERZIE TOR\n===================================" #: lib/gettor/i18n.py:67 msgid "" @@ -89,14 +89,14 @@ msgid "" "language you want in the address you send the mail to:\n" "\n" " gettor+fa(a)torproject.org" -msgstr "" +msgstr "Pre získanie verzie Tor vo vašom jazyku, uveďte požadovaný jazyk do mena adresáta vášho e-mailu:\n\ngettor+fa@torproject.org" #: lib/gettor/i18n.py:72 msgid "" "This example will give you the requested package in a localized\n" "version for Farsi (Persian). Check below for a list of supported language\n" "codes. " -msgstr "" +msgstr "Tento príklad by vám dodal verziu balíčka preloženú do Farsi (perzštiny).\nNižšie nájdete zoznam podporovaných jazykových kódov." #: lib/gettor/i18n.py:76 msgid " List of supported locales:" @@ -119,7 +119,7 @@ msgid "" " gettor+pl(a)torproject.org: Polish\n" " gettor+ru(a)torproject.org: Russian\n" " gettor+zh(a)torproject.org: Chinese" -msgstr "" +msgstr " gettor+ar(a)torproject.org: Arabčina\n gettor+de(a)torproject.org: Nemčina\n gettor+en(a)torproject.org: Angličtina\n gettor+es(a)torproject.org: Španielčina\n gettor+fa(a)torproject.org: Farsi (perzčtina)\n gettor+fr(a)torproject.org: Francúzština\n gettor+it(a)torproject.org: Taliančina\n gettor+nl(a)torproject.org: Holandština\n gettor+pl(a)torproject.org: Polština\n gettor+ru(a)torproject.org: Ruština\n gettor+zh(a)torproject.org: Čínština" #: lib/gettor/i18n.py:92 msgid "If you select no language, you will receive the English version." @@ -129,7 +129,7 @@ msgstr "Ak si nevyberiete žiadny jazyk, bude zvolená anglická verzia." msgid "" "SMALLER SIZED PACKAGES\n" "======================" -msgstr "" +msgstr "BALÍČKY S MENŠOU VEĽKOSŤOU\n======================" #: lib/gettor/i18n.py:97 msgid "" @@ -432,7 +432,7 @@ msgstr "" msgid "" "FREQUENTLY ASKED QUESTIONS\n" "==========================" -msgstr "" +msgstr "ČASTO KLADENÉ OTÁZKY\n==========================" #: lib/gettor/i18n.py:272 msgid "What is Tor?" @@ -464,7 +464,7 @@ msgid "" "The Browser Bundle (TBB) is the package we recommend to most users. \n" "The bundle comes with everything you need to safely browse the Internet.\n" "Just extract it and run." -msgstr "" +msgstr "Tor Browser Bundle (TBB) je balíček, ktorý odporúčame väčšine užívateľov.\nZväzok obsahuje všetko, čo potrebujete na bezpečené surfovanie\npo Internete. Stačí rozbaliť a spustiť." #: lib/gettor/i18n.py:292 msgid "What package should I request?" @@ -500,4 +500,4 @@ msgid "" " https://www.oignon.net/dist/torbrowser/\n" " https://tor.beme-it.de/dist/torbrowser/\n" " https://www.torservers.net/mirrors/torproject.org/dist/torbrowser/" -msgstr "" +msgstr "Prepáčte, ale požadovaný balíček (%s) je pre vášho poskytovateľa príliš veľký.\nSkúste použiť iného poskytovateľa, ktorý povoľuje väčšie prílohy e-mailu.\nMôžete tiež skúsiť niektorý z nasledujúcich obrazov:\n\nhttps://www.oignon.net/dist/torbrowser/\nhttps://tor.beme-it.de…"

1 0

[onionoo/master] Move front-end parts of NodeStatus to SummaryDocument.
by karsten＠torproject.org 22 Jul '14

22 Jul '14

commit ab7f4ee58825a7980579e6cd4e334ee373b29047 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Jul 22 13:53:42 2014 +0200 Move front-end parts of NodeStatus to SummaryDocument. The NodeStatus class was seriously overloaded: 1. NodeDetailsStatusUpdater uses the list of NodeStatus instances as input to its GeoIP and rDNS look-up operations and to calculate path-selection probabilities. 2. DetailsDocumentWriter uses fields from NodeStatus instances (some of them being stored persistently, some only kept in memory) to write part of DetailsDocument instances. 3. NodeIndexer obtains all relevant data to populate the search index from NodeStatus instances. 4. ResponseBuilder uses NodeStatus instances to produce summary documents on-the-fly. As first step to reduce this complexity, let's copy the parts used for 3 and 4 to a currently unused class, SummaryDocument, and use that class in NodeIndexer and ResponseBuilder. It makes sense to combine those two purposes for performance reasons. Parts 1 and 2 will be simplified in later commits. --- src/org/torproject/onionoo/DocumentStore.java | 235 +++++++++++++------- src/org/torproject/onionoo/Main.java | 4 +- .../onionoo/NodeDetailsStatusUpdater.java | 2 +- src/org/torproject/onionoo/NodeIndexer.java | 38 ++-- src/org/torproject/onionoo/RequestHandler.java | 65 +++--- src/org/torproject/onionoo/ResponseBuilder.java | 52 ++--- src/org/torproject/onionoo/SummaryDocument.java | 195 +++++++++++++++- .../torproject/onionoo/SummaryDocumentWriter.java | 87 ++++++++ .../org/torproject/onionoo/DummyDocumentStore.java | 2 +- .../torproject/onionoo/ResourceServletTest.java | 107 +++++---- 10 files changed, 569 insertions(+), 218 deletions(-) diff --git a/src/org/torproject/onionoo/DocumentStore.java b/src/org/torproject/onionoo/DocumentStore.java index e3bf618..434ecb6 100644 --- a/src/org/torproject/onionoo/DocumentStore.java +++ b/src/org/torproject/onionoo/DocumentStore.java @@ -11,7 +11,9 @@ import java.io.FileInputStream; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.SortedMap; import java.util.SortedSet; @@ -43,46 +45,39 @@ public class DocumentStore { this.time = ApplicationFactory.getTime(); } - private boolean listedArchivedNodeStatuses = false, - listedCurrentNodeStatuses = false; - private long listOperations = 0L, listedFiles = 0L, storedFiles = 0L, storedBytes = 0L, retrievedFiles = 0L, retrievedBytes = 0L, removedFiles = 0L; - /* Node statuses are cached in memory, as opposed to all other document - * types. This cache is initialized when listing NodeStatus documents, - * either including or excluding archived node statuses. Later retrieve - * operations depend on which NodeStatus documents were listed. */ + /* Node statuses and summary documents are cached in memory, as opposed + * to all other document types. These caches are initialized when first + * accessing or modifying a NodeStatus or SummaryDocument document, + * respectively. */ private SortedMap<String, NodeStatus> cachedNodeStatuses; + private SortedMap<String, SummaryDocument> cachedSummaryDocuments; public <T extends Document> SortedSet<String> list( - Class<T> documentType, boolean includeArchive) { + Class<T> documentType) { if (documentType.equals(NodeStatus.class)) { - return this.listNodeStatuses(includeArchive); + return this.listNodeStatuses(); + } else if (documentType.equals(SummaryDocument.class)) { + return this.listSummaryDocuments(); } else { return this.listDocumentFiles(documentType); } } - private SortedSet<String> listNodeStatuses(boolean includeArchive) { - if ((includeArchive && listedCurrentNodeStatuses) || - (!includeArchive && listedArchivedNodeStatuses)) { - System.err.println("Listing node statuses is only permitted " - + "including the archive or excluding the archive, but not " - + "both. Returning empty list."); - return new TreeSet<String>(); - } + private SortedSet<String> listNodeStatuses() { if (this.cachedNodeStatuses == null) { - this.cacheNodeStatuses(includeArchive); + this.cacheNodeStatuses(); } return new TreeSet<String>(this.cachedNodeStatuses.keySet()); } - private void cacheNodeStatuses(boolean includeArchive) { + private void cacheNodeStatuses() { SortedMap<String, NodeStatus> parsedNodeStatuses = new TreeMap<String, NodeStatus>(); - File directory = includeArchive ? this.statusDir : this.outDir; + File directory = this.statusDir; if (directory != null) { File summaryFile = new File(directory, "summary"); if (summaryFile.exists()) { @@ -109,14 +104,56 @@ public class DocumentStore { } } } - if (includeArchive) { - this.listedArchivedNodeStatuses = true; - } else { - this.listedCurrentNodeStatuses = true; - } this.cachedNodeStatuses = parsedNodeStatuses; } + private SortedSet<String> listSummaryDocuments() { + if (this.cachedSummaryDocuments == null) { + this.cacheSummaryDocuments(); + } + return new TreeSet<String>(this.cachedSummaryDocuments.keySet()); + } + + private void cacheSummaryDocuments() { + SortedMap<String, SummaryDocument> parsedSummaryDocuments = + new TreeMap<String, SummaryDocument>(); + File directory = this.outDir; + if (directory != null) { + File summaryFile = new File(directory, "summary"); + if (summaryFile.exists()) { + String line = null; + try { + Gson gson = new Gson(); + BufferedReader br = new BufferedReader(new FileReader( + summaryFile)); + while ((line = br.readLine()) != null) { + if (line.length() == 0) { + continue; + } + SummaryDocument summaryDocument = gson.fromJson(line, + SummaryDocument.class); + if (summaryDocument != null) { + parsedSummaryDocuments.put(summaryDocument.getFingerprint(), + summaryDocument); + } + } + br.close(); + this.listedFiles += parsedSummaryDocuments.size(); + this.listOperations++; + } catch (IOException e) { + System.err.println("Could not read file '" + + summaryFile.getAbsolutePath() + "'."); + e.printStackTrace(); + } catch (JsonParseException e) { + System.err.println("Could not parse summary document '" + line + + "' in file '" + summaryFile.getAbsolutePath() + "'."); + e.printStackTrace(); + } + } + } + this.cachedSummaryDocuments = parsedSummaryDocuments; + } + private <T extends Document> SortedSet<String> listDocumentFiles( Class<T> documentType) { SortedSet<String> fingerprints = new TreeSet<String>(); @@ -178,6 +215,9 @@ public class DocumentStore { String fingerprint) { if (document instanceof NodeStatus) { return this.storeNodeStatus((NodeStatus) document, fingerprint); + } else if (document instanceof SummaryDocument) { + return this.storeSummaryDocument((SummaryDocument) document, + fingerprint); } else { return this.storeDocumentFile(document, fingerprint); } @@ -186,12 +226,21 @@ public class DocumentStore { private <T extends Document> boolean storeNodeStatus( NodeStatus nodeStatus, String fingerprint) { if (this.cachedNodeStatuses == null) { - this.cacheNodeStatuses(true); + this.cacheNodeStatuses(); } this.cachedNodeStatuses.put(fingerprint, nodeStatus); return true; } + private <T extends Document> boolean storeSummaryDocument( + SummaryDocument summaryDocument, String fingerprint) { + if (this.cachedSummaryDocuments == null) { + this.cacheSummaryDocuments(); + } + this.cachedSummaryDocuments.put(fingerprint, summaryDocument); + return true; + } + private <T extends Document> boolean storeDocumentFile(T document, String fingerprint) { File documentFile = this.getDocumentFile(document.getClass(), @@ -270,6 +319,8 @@ public class DocumentStore { boolean parse, String fingerprint) { if (documentType.equals(NodeStatus.class)) { return documentType.cast(this.retrieveNodeStatus(fingerprint)); + } else if (documentType.equals(SummaryDocument.class)) { + return documentType.cast(this.retrieveSummaryDocument(fingerprint)); } else { return this.retrieveDocumentFile(documentType, parse, fingerprint); } @@ -277,12 +328,17 @@ public class DocumentStore { private NodeStatus retrieveNodeStatus(String fingerprint) { if (this.cachedNodeStatuses == null) { - this.cacheNodeStatuses(true); + this.cacheNodeStatuses(); } - if (this.cachedNodeStatuses.containsKey(fingerprint)) { - return this.cachedNodeStatuses.get(fingerprint); - } else if (this.listedArchivedNodeStatuses) { - return null; + return this.cachedNodeStatuses.get(fingerprint); + } + + private SummaryDocument retrieveSummaryDocument(String fingerprint) { + if (this.cachedSummaryDocuments == null) { + this.cacheSummaryDocuments(); + } + if (this.cachedSummaryDocuments.containsKey(fingerprint)) { + return this.cachedSummaryDocuments.get(fingerprint); } /* TODO This is an evil hack to support looking up relays or bridges * that haven't been running for a week without having to load @@ -294,39 +350,35 @@ public class DocumentStore { return null; } boolean isRelay = detailsDocument.getHashedFingerprint() == null; + boolean running = false; String nickname = detailsDocument.getNickname(); - String address = null, countryCode = null, hostName = null, - defaultPolicy = null, portList = null, aSNumber = null, - contact = null; - SortedSet<String> orAddressesAndPorts = new TreeSet<String>(); + List<String> addresses = new ArrayList<String>(); + String countryCode = null, aSNumber = null, contact = null; for (String orAddressAndPort : detailsDocument.getOrAddresses()) { - if (address == null) { - if (!orAddressAndPort.contains(":")) { - return null; - } - address = orAddressAndPort.substring(0, - orAddressAndPort.lastIndexOf(":")); - } else { - orAddressesAndPorts.add(orAddressAndPort); + if (!orAddressAndPort.contains(":")) { + return null; + } + String orAddress = orAddressAndPort.substring(0, + orAddressAndPort.lastIndexOf(":")); + if (!addresses.contains(orAddress)) { + addresses.add(orAddress); } } - SortedSet<String> exitAddresses = new TreeSet<String>(); if (detailsDocument.getExitAddresses() != null) { - exitAddresses.addAll(detailsDocument.getExitAddresses()); + for (String exitAddress : detailsDocument.getExitAddresses()) { + if (!addresses.contains(exitAddress)) { + addresses.add(exitAddress); + } + } } SortedSet<String> relayFlags = new TreeSet<String>(), family = null; long lastSeenMillis = -1L, consensusWeight = -1L, - lastRdnsLookup = -1L, firstSeenMillis = -1L, - lastChangedAddresses = -1L; - int orPort = 0, dirPort = 0; - Boolean recommendedVersion = null; - NodeStatus nodeStatus = new NodeStatus(isRelay, nickname, fingerprint, - address, orAddressesAndPorts, exitAddresses, lastSeenMillis, - orPort, dirPort, relayFlags, consensusWeight, countryCode, - hostName, lastRdnsLookup, defaultPolicy, portList, - firstSeenMillis, lastChangedAddresses, aSNumber, contact, - recommendedVersion, family); - return nodeStatus; + firstSeenMillis = -1L; + SummaryDocument summaryDocument = new SummaryDocument(isRelay, + nickname, fingerprint, addresses, lastSeenMillis, running, + relayFlags, consensusWeight, countryCode, firstSeenMillis, + aSNumber, contact, family); + return summaryDocument; } private <T extends Document> T retrieveDocumentFile( @@ -455,6 +507,8 @@ public class DocumentStore { String fingerprint) { if (documentType.equals(NodeStatus.class)) { return this.removeNodeStatus(fingerprint); + } else if (documentType.equals(SummaryDocument.class)) { + return this.removeSummaryDocument(fingerprint); } else { return this.removeDocumentFile(documentType, fingerprint); } @@ -462,11 +516,18 @@ public class DocumentStore { private boolean removeNodeStatus(String fingerprint) { if (this.cachedNodeStatuses == null) { - this.cacheNodeStatuses(true); + this.cacheNodeStatuses(); } return this.cachedNodeStatuses.remove(fingerprint) != null; } + private boolean removeSummaryDocument(String fingerprint) { + if (this.cachedSummaryDocuments == null) { + this.cacheSummaryDocuments(); + } + return this.cachedSummaryDocuments.remove(fingerprint) != null; + } + private <T extends Document> boolean removeDocumentFile( Class<T> documentType, String fingerprint) { File documentFile = this.getDocumentFile(documentType, fingerprint); @@ -550,18 +611,20 @@ public class DocumentStore { * containing current time. It's important to write the update file * now, not earlier, because the front-end should not read new node * statuses until all details, bandwidths, and weights are ready. */ - if (this.listedArchivedNodeStatuses) { - this.writeNodeStatuses(false); - this.writeNodeStatuses(true); - this.writeUpdateStatus(); - } else if (this.listedCurrentNodeStatuses) { - this.writeNodeStatuses(false); + if (this.cachedNodeStatuses != null || + this.cachedSummaryDocuments != null) { + if (this.cachedNodeStatuses != null) { + this.writeNodeStatuses(); + } + if (this.cachedSummaryDocuments != null) { + this.writeSummaryDocuments(); + } this.writeUpdateStatus(); } } - private void writeNodeStatuses(boolean includeArchive) { - File directory = includeArchive ? this.statusDir : this.outDir; + private void writeNodeStatuses() { + File directory = this.statusDir; if (directory == null) { return; } @@ -569,21 +632,8 @@ public class DocumentStore { SortedMap<String, NodeStatus> cachedRelays = new TreeMap<String, NodeStatus>(), cachedBridges = new TreeMap<String, NodeStatus>(); - long cutoff = 0L; - if (!includeArchive) { - long maxLastSeenMillis = 0L; - for (NodeStatus node : this.cachedNodeStatuses.values()) { - if (node.getLastSeenMillis() > maxLastSeenMillis) { - maxLastSeenMillis = node.getLastSeenMillis(); - } - } - cutoff = maxLastSeenMillis - DateTimeHelper.ONE_WEEK; - } for (Map.Entry<String, NodeStatus> e : this.cachedNodeStatuses.entrySet()) { - if (e.getValue().getLastSeenMillis() < cutoff) { - continue; - } if (e.getValue().isRelay()) { cachedRelays.put(e.getKey(), e.getValue()); } else { @@ -624,6 +674,35 @@ public class DocumentStore { } } + private void writeSummaryDocuments() { + StringBuilder sb = new StringBuilder(); + Gson gson = new Gson(); + for (SummaryDocument summaryDocument : + this.cachedSummaryDocuments.values()) { + String line = gson.toJson(summaryDocument); + if (line != null) { + sb.append(line + "\n"); + } else { + System.err.println("Could not serialize relay summary document '" + + summaryDocument.getFingerprint() + "'"); + } + } + String documentString = sb.toString(); + File summaryFile = new File(this.outDir, "summary"); + try { + summaryFile.getParentFile().mkdirs(); + BufferedWriter bw = new BufferedWriter(new FileWriter(summaryFile)); + bw.write(documentString); + bw.close(); + this.storedFiles++; + this.storedBytes += documentString.length(); + } catch (IOException e) { + System.err.println("Could not write file '" + + summaryFile.getAbsolutePath() + "'."); + e.printStackTrace(); + } + } + private void writeUpdateStatus() { if (this.outDir == null) { return; diff --git a/src/org/torproject/onionoo/Main.java b/src/org/torproject/onionoo/Main.java index 0ae12e4..e9afe2f 100644 --- a/src/org/torproject/onionoo/Main.java +++ b/src/org/torproject/onionoo/Main.java @@ -45,6 +45,8 @@ public class Main { StatusUpdater[] sus = new StatusUpdater[] { ndsu, bsu, wsu, csu, usu }; + SummaryDocumentWriter sdw = new SummaryDocumentWriter(); + Logger.printStatusTime("Initialized summary document writer"); DetailsDocumentWriter ddw = new DetailsDocumentWriter(); Logger.printStatusTime("Initialized details document writer"); BandwidthDocumentWriter bdw = new BandwidthDocumentWriter(); @@ -55,7 +57,7 @@ public class Main { Logger.printStatusTime("Initialized clients document writer"); UptimeDocumentWriter udw = new UptimeDocumentWriter(); Logger.printStatusTime("Initialized uptime document writer"); - DocumentWriter[] dws = new DocumentWriter[] { ddw, bdw, wdw, cdw, + DocumentWriter[] dws = new DocumentWriter[] { sdw, ddw, bdw, wdw, cdw, udw }; Logger.printStatus("Downloading descriptors."); diff --git a/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java b/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java index e898638..a1149a5 100644 --- a/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java +++ b/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java @@ -310,7 +310,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, private void readStatusSummary() { SortedSet<String> fingerprints = this.documentStore.list( - NodeStatus.class, true); + NodeStatus.class); for (String fingerprint : fingerprints) { NodeStatus node = this.documentStore.retrieve(NodeStatus.class, true, fingerprint); diff --git a/src/org/torproject/onionoo/NodeIndexer.java b/src/org/torproject/onionoo/NodeIndexer.java index 8a04afb..d24284e 100644 --- a/src/org/torproject/onionoo/NodeIndexer.java +++ b/src/org/torproject/onionoo/NodeIndexer.java @@ -44,21 +44,21 @@ class NodeIndex { } - private Map<String, NodeStatus> relayFingerprintSummaryLines; + private Map<String, SummaryDocument> relayFingerprintSummaryLines; public void setRelayFingerprintSummaryLines( - Map<String, NodeStatus> relayFingerprintSummaryLines) { + Map<String, SummaryDocument> relayFingerprintSummaryLines) { this.relayFingerprintSummaryLines = relayFingerprintSummaryLines; } - public Map<String, NodeStatus> getRelayFingerprintSummaryLines() { + public Map<String, SummaryDocument> getRelayFingerprintSummaryLines() { return this.relayFingerprintSummaryLines; } - private Map<String, NodeStatus> bridgeFingerprintSummaryLines; + private Map<String, SummaryDocument> bridgeFingerprintSummaryLines; public void setBridgeFingerprintSummaryLines( - Map<String, NodeStatus> bridgeFingerprintSummaryLines) { + Map<String, SummaryDocument> bridgeFingerprintSummaryLines) { this.bridgeFingerprintSummaryLines = bridgeFingerprintSummaryLines; } - public Map<String, NodeStatus> getBridgeFingerprintSummaryLines() { + public Map<String, SummaryDocument> getBridgeFingerprintSummaryLines() { return this.bridgeFingerprintSummaryLines; } @@ -239,11 +239,11 @@ public class NodeIndexer implements ServletContextListener, Runnable { } } List<String> newRelaysByConsensusWeight = new ArrayList<String>(); - Map<String, NodeStatus> + Map<String, SummaryDocument> newRelayFingerprintSummaryLines = - new HashMap<String, NodeStatus>(), + new HashMap<String, SummaryDocument>(), newBridgeFingerprintSummaryLines = - new HashMap<String, NodeStatus>(); + new HashMap<String, SummaryDocument>(); Map<String, Set<String>> newRelaysByCountryCode = new HashMap<String, Set<String>>(), newRelaysByASNumber = new HashMap<String, Set<String>>(), @@ -256,14 +256,14 @@ public class NodeIndexer implements ServletContextListener, Runnable { newBridgesByFirstSeenDays = new TreeMap<Integer, Set<String>>(), newRelaysByLastSeenDays = new TreeMap<Integer, Set<String>>(), newBridgesByLastSeenDays = new TreeMap<Integer, Set<String>>(); - Set<NodeStatus> currentRelays = new HashSet<NodeStatus>(), - currentBridges = new HashSet<NodeStatus>(); - SortedSet<String> fingerprints = documentStore.list(NodeStatus.class, - false); + Set<SummaryDocument> currentRelays = new HashSet<SummaryDocument>(), + currentBridges = new HashSet<SummaryDocument>(); + SortedSet<String> fingerprints = documentStore.list( + SummaryDocument.class); long relaysLastValidAfterMillis = 0L, bridgesLastPublishedMillis = 0L; for (String fingerprint : fingerprints) { - NodeStatus node = documentStore.retrieve(NodeStatus.class, true, - fingerprint); + SummaryDocument node = documentStore.retrieve(SummaryDocument.class, + true, fingerprint); if (node.isRelay()) { relaysLastValidAfterMillis = Math.max( relaysLastValidAfterMillis, node.getLastSeenMillis()); @@ -276,12 +276,10 @@ public class NodeIndexer implements ServletContextListener, Runnable { } Time time = ApplicationFactory.getTime(); List<String> orderRelaysByConsensusWeight = new ArrayList<String>(); - for (NodeStatus entry : currentRelays) { + for (SummaryDocument entry : currentRelays) { String fingerprint = entry.getFingerprint().toUpperCase(); String hashedFingerprint = entry.getHashedFingerprint(). toUpperCase(); - entry.setRunning(entry.getLastSeenMillis() == - relaysLastValidAfterMillis); newRelayFingerprintSummaryLines.put(fingerprint, entry); newRelayFingerprintSummaryLines.put(hashedFingerprint, entry); long consensusWeight = entry.getConsensusWeight(); @@ -360,12 +358,10 @@ public class NodeIndexer implements ServletContextListener, Runnable { } e.getValue().retainAll(inMutualFamilyRelation); } - for (NodeStatus entry : currentBridges) { + for (SummaryDocument entry : currentBridges) { String hashedFingerprint = entry.getFingerprint().toUpperCase(); String hashedHashedFingerprint = entry.getHashedFingerprint(). toUpperCase(); - entry.setRunning(entry.getRelayFlags().contains("Running") && - entry.getLastSeenMillis() == bridgesLastPublishedMillis); newBridgeFingerprintSummaryLines.put(hashedFingerprint, entry); newBridgeFingerprintSummaryLines.put(hashedHashedFingerprint, entry); diff --git a/src/org/torproject/onionoo/RequestHandler.java b/src/org/torproject/onionoo/RequestHandler.java index 66559e6..39a66e1 100644 --- a/src/org/torproject/onionoo/RequestHandler.java +++ b/src/org/torproject/onionoo/RequestHandler.java @@ -10,6 +10,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.SortedMap; +import java.util.SortedSet; public class RequestHandler { @@ -109,11 +110,11 @@ public class RequestHandler { this.family = family; } - private Map<String, NodeStatus> filteredRelays = - new HashMap<String, NodeStatus>(); + private Map<String, SummaryDocument> filteredRelays = + new HashMap<String, SummaryDocument>(); - private Map<String, NodeStatus> filteredBridges = - new HashMap<String, NodeStatus>(); + private Map<String, SummaryDocument> filteredBridges = + new HashMap<String, SummaryDocument>(); public void handleRequest() { this.filteredRelays.putAll( @@ -164,8 +165,9 @@ public class RequestHandler { } boolean runningRequested = this.running.equals("true"); Set<String> removeRelays = new HashSet<String>(); - for (Map.Entry<String, NodeStatus> e : filteredRelays.entrySet()) { - if (e.getValue().getRunning() != runningRequested) { + for (Map.Entry<String, SummaryDocument> e : + filteredRelays.entrySet()) { + if (e.getValue().isRunning() != runningRequested) { removeRelays.add(e.getKey()); } } @@ -173,8 +175,9 @@ public class RequestHandler { this.filteredRelays.remove(fingerprint); } Set<String> removeBridges = new HashSet<String>(); - for (Map.Entry<String, NodeStatus> e : filteredBridges.entrySet()) { - if (e.getValue().getRunning() != runningRequested) { + for (Map.Entry<String, SummaryDocument> e : + filteredBridges.entrySet()) { + if (e.getValue().isRunning() != runningRequested) { removeBridges.add(e.getKey()); } } @@ -194,9 +197,10 @@ public class RequestHandler { private void filterBySearchTerm(String searchTerm) { Set<String> removeRelays = new HashSet<String>(); - for (Map.Entry<String, NodeStatus> e : filteredRelays.entrySet()) { + for (Map.Entry<String, SummaryDocument> e : + filteredRelays.entrySet()) { String fingerprint = e.getKey(); - NodeStatus entry = e.getValue(); + SummaryDocument entry = e.getValue(); boolean lineMatches = false; String nickname = entry.getNickname() != null ? entry.getNickname().toLowerCase() : "unnamed"; @@ -214,14 +218,7 @@ public class RequestHandler { /* Non-$-prefixed fingerprint matches. */ lineMatches = true; } else { - List<String> addresses = new ArrayList<String>(); - addresses.add(entry.getAddress().toLowerCase()); - for (String orAddress : entry.getOrAddresses()) { - addresses.add(orAddress.toLowerCase()); - } - for (String exitAddress : entry.getExitAddresses()) { - addresses.add(exitAddress.toLowerCase()); - } + List<String> addresses = entry.getAddresses(); for (String address : addresses) { if (address.startsWith(searchTerm.toLowerCase())) { /* Address matches. */ @@ -238,9 +235,10 @@ public class RequestHandler { this.filteredRelays.remove(fingerprint); } Set<String> removeBridges = new HashSet<String>(); - for (Map.Entry<String, NodeStatus> e : filteredBridges.entrySet()) { + for (Map.Entry<String, SummaryDocument> e : + filteredBridges.entrySet()) { String hashedFingerprint = e.getKey(); - NodeStatus entry = e.getValue(); + SummaryDocument entry = e.getValue(); boolean lineMatches = false; String nickname = entry.getNickname() != null ? entry.getNickname().toLowerCase() : "unnamed"; @@ -272,12 +270,12 @@ public class RequestHandler { return; } String fingerprint = this.lookup; - NodeStatus relayLine = this.filteredRelays.get(fingerprint); + SummaryDocument relayLine = this.filteredRelays.get(fingerprint); this.filteredRelays.clear(); if (relayLine != null) { this.filteredRelays.put(fingerprint, relayLine); } - NodeStatus bridgeLine = this.filteredBridges.get(fingerprint); + SummaryDocument bridgeLine = this.filteredBridges.get(fingerprint); this.filteredBridges.clear(); if (bridgeLine != null) { this.filteredBridges.put(fingerprint, bridgeLine); @@ -291,8 +289,8 @@ public class RequestHandler { this.filteredRelays.clear(); this.filteredBridges.clear(); String fingerprint = this.fingerprint; - NodeStatus entry = this.documentStore.retrieve(NodeStatus.class, true, - fingerprint); + SummaryDocument entry = this.documentStore.retrieve( + SummaryDocument.class, true, fingerprint); if (entry != null) { if (entry.isRelay()) { this.filteredRelays.put(fingerprint, entry); @@ -409,7 +407,8 @@ public class RequestHandler { this.nodeIndex.getBridgesByLastSeenDays(), this.lastSeenDays); } - private void filterNodesByDays(Map<String, NodeStatus> filteredNodes, + private void filterNodesByDays( + Map<String, SummaryDocument> filteredNodes, SortedMap<Integer, Set<String>> nodesByDays, int[] days) { Set<String> removeNodes = new HashSet<String>(); for (Set<String> nodes : nodesByDays.headMap(days[0]).values()) { @@ -483,14 +482,14 @@ public class RequestHandler { this.orderedRelays.add(this.filteredRelays.remove(relay)); } } - Set<NodeStatus> uniqueBridges = new HashSet<NodeStatus>( + Set<SummaryDocument> uniqueBridges = new HashSet<SummaryDocument>( this.filteredBridges.values()); this.orderedBridges.addAll(uniqueBridges); } else { - Set<NodeStatus> uniqueRelays = new HashSet<NodeStatus>( + Set<SummaryDocument> uniqueRelays = new HashSet<SummaryDocument>( this.filteredRelays.values()); this.orderedRelays.addAll(uniqueRelays); - Set<NodeStatus> uniqueBridges = new HashSet<NodeStatus>( + Set<SummaryDocument> uniqueBridges = new HashSet<SummaryDocument>( this.filteredBridges.values()); this.orderedBridges.addAll(uniqueBridges); } @@ -528,13 +527,15 @@ public class RequestHandler { } } - private List<NodeStatus> orderedRelays = new ArrayList<NodeStatus>(); - public List<NodeStatus> getOrderedRelays() { + private List<SummaryDocument> orderedRelays = + new ArrayList<SummaryDocument>(); + public List<SummaryDocument> getOrderedRelays() { return this.orderedRelays; } - private List<NodeStatus> orderedBridges = new ArrayList<NodeStatus>(); - public List<NodeStatus> getOrderedBridges() { + private List<SummaryDocument> orderedBridges = + new ArrayList<SummaryDocument>(); + public List<SummaryDocument> getOrderedBridges() { return this.orderedBridges; } diff --git a/src/org/torproject/onionoo/ResponseBuilder.java b/src/org/torproject/onionoo/ResponseBuilder.java index 77c4b6c..2086ea8 100644 --- a/src/org/torproject/onionoo/ResponseBuilder.java +++ b/src/org/torproject/onionoo/ResponseBuilder.java @@ -32,13 +32,15 @@ public class ResponseBuilder { this.bridgesPublishedString = bridgesPublishedString; } - private List<NodeStatus> orderedRelays = new ArrayList<NodeStatus>(); - public void setOrderedRelays(List<NodeStatus> orderedRelays) { + private List<SummaryDocument> orderedRelays = + new ArrayList<SummaryDocument>(); + public void setOrderedRelays(List<SummaryDocument> orderedRelays) { this.orderedRelays = orderedRelays; } - private List<NodeStatus> orderedBridges = new ArrayList<NodeStatus>(); - public void setOrderedBridges(List<NodeStatus> orderedBridges) { + private List<SummaryDocument> orderedBridges = + new ArrayList<SummaryDocument>(); + public void setOrderedBridges(List<SummaryDocument> orderedBridges) { this.orderedBridges = orderedBridges; } @@ -53,11 +55,11 @@ public class ResponseBuilder { writeBridges(orderedBridges, pw); } - private void writeRelays(List<NodeStatus> relays, PrintWriter pw) { + private void writeRelays(List<SummaryDocument> relays, PrintWriter pw) { pw.write("{\"relays_published\":\"" + relaysPublishedString + "\",\n\"relays\":["); int written = 0; - for (NodeStatus entry : relays) { + for (SummaryDocument entry : relays) { String lines = this.formatNodeStatus(entry); if (lines.length() > 0) { pw.print((written++ > 0 ? ",\n" : "\n") + lines); @@ -66,11 +68,12 @@ public class ResponseBuilder { pw.print("\n],\n"); } - private void writeBridges(List<NodeStatus> bridges, PrintWriter pw) { + private void writeBridges(List<SummaryDocument> bridges, + PrintWriter pw) { pw.write("\"bridges_published\":\"" + bridgesPublishedString + "\",\n\"bridges\":["); int written = 0; - for (NodeStatus entry : bridges) { + for (SummaryDocument entry : bridges) { String lines = this.formatNodeStatus(entry); if (lines.length() > 0) { pw.print((written++ > 0 ? ",\n" : "\n") + lines); @@ -79,7 +82,7 @@ public class ResponseBuilder { pw.print("\n]}\n"); } - private String formatNodeStatus(NodeStatus entry) { + private String formatNodeStatus(SummaryDocument entry) { if (this.resourceType == null) { return ""; } else if (this.resourceType.equals("summary")) { @@ -99,26 +102,17 @@ public class ResponseBuilder { } } - private String writeSummaryLine(NodeStatus entry) { + private String writeSummaryLine(SummaryDocument entry) { return entry.isRelay() ? writeRelaySummaryLine(entry) : writeBridgeSummaryLine(entry); } - private String writeRelaySummaryLine(NodeStatus entry) { + private String writeRelaySummaryLine(SummaryDocument entry) { String nickname = !entry.getNickname().equals("Unnamed") ? entry.getNickname() : null; String fingerprint = entry.getFingerprint(); - String running = entry.getRunning() ? "true" : "false"; - List<String> addresses = new ArrayList<String>(); - addresses.add(entry.getAddress()); - for (String orAddress : entry.getOrAddresses()) { - addresses.add(orAddress); - } - for (String exitAddress : entry.getExitAddresses()) { - if (!addresses.contains(exitAddress)) { - addresses.add(exitAddress); - } - } + String running = entry.isRunning() ? "true" : "false"; + List<String> addresses = entry.getAddresses(); StringBuilder addressesBuilder = new StringBuilder(); int written = 0; for (String address : addresses) { @@ -130,17 +124,17 @@ public class ResponseBuilder { fingerprint, addressesBuilder.toString(), running); } - private String writeBridgeSummaryLine(NodeStatus entry) { + private String writeBridgeSummaryLine(SummaryDocument entry) { String nickname = !entry.getNickname().equals("Unnamed") ? entry.getNickname() : null; String hashedFingerprint = entry.getFingerprint(); - String running = entry.getRunning() ? "true" : "false"; + String running = entry.isRunning() ? "true" : "false"; return String.format("{%s\"h\":\"%s\",\"r\":%s}", (nickname == null ? "" : "\"n\":\"" + nickname + "\","), hashedFingerprint, running); } - private String writeDetailsLines(NodeStatus entry) { + private String writeDetailsLines(SummaryDocument entry) { String fingerprint = entry.getFingerprint(); if (this.fields != null) { /* TODO Maybe there's a more elegant way (more maintainable, more @@ -267,7 +261,7 @@ public class ResponseBuilder { } } - private String writeBandwidthLines(NodeStatus entry) { + private String writeBandwidthLines(SummaryDocument entry) { String fingerprint = entry.getFingerprint(); BandwidthDocument bandwidthDocument = this.documentStore.retrieve( BandwidthDocument.class, false, fingerprint); @@ -279,7 +273,7 @@ public class ResponseBuilder { } } - private String writeWeightsLines(NodeStatus entry) { + private String writeWeightsLines(SummaryDocument entry) { String fingerprint = entry.getFingerprint(); WeightsDocument weightsDocument = this.documentStore.retrieve( WeightsDocument.class, false, fingerprint); @@ -291,7 +285,7 @@ public class ResponseBuilder { } } - private String writeClientsLines(NodeStatus entry) { + private String writeClientsLines(SummaryDocument entry) { String fingerprint = entry.getFingerprint(); ClientsDocument clientsDocument = this.documentStore.retrieve( ClientsDocument.class, false, fingerprint); @@ -303,7 +297,7 @@ public class ResponseBuilder { } } - private String writeUptimeLines(NodeStatus entry) { + private String writeUptimeLines(SummaryDocument entry) { String fingerprint = entry.getFingerprint(); UptimeDocument uptimeDocument = this.documentStore.retrieve( UptimeDocument.class, false, fingerprint); diff --git a/src/org/torproject/onionoo/SummaryDocument.java b/src/org/torproject/onionoo/SummaryDocument.java index 63862a2..3c0f700 100644 --- a/src/org/torproject/onionoo/SummaryDocument.java +++ b/src/org/torproject/onionoo/SummaryDocument.java @@ -1,8 +1,201 @@ -/* Copyright 2013 The Tor Project +/* Copyright 2013--2014 The Tor Project * See LICENSE for licensing information */ package org.torproject.onionoo; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.regex.Pattern; + +import org.apache.commons.codec.DecoderException; +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.codec.digest.DigestUtils; + class SummaryDocument extends Document { + private boolean t; + public void setRelay(boolean isRelay) { + this.t = isRelay; + } + public boolean isRelay() { + return this.t; + } + + private String f; + public void setFingerprint(String fingerprint) { + if (fingerprint != null) { + Pattern fingerprintPattern = Pattern.compile("^[0-9a-fA-F]{40}$"); + if (!fingerprintPattern.matcher(fingerprint).matches()) { + throw new IllegalArgumentException("Fingerprint '" + fingerprint + + "' is not a valid fingerprint."); + } + } + this.f = fingerprint; + } + public String getFingerprint() { + return this.f; + } + + public String getHashedFingerprint() { + String hashedFingerprint = null; + if (this.f != null) { + try { + hashedFingerprint = DigestUtils.shaHex(Hex.decodeHex( + this.f.toCharArray())).toUpperCase(); + } catch (DecoderException e) { + /* Format tested in setFingerprint(). */ + } + } + return hashedFingerprint; + } + + private String n; + public void setNickname(String nickname) { + if (nickname == null || nickname.equals("Unnamed")) { + this.n = null; + } else { + this.n = nickname; + } + } + public String getNickname() { + return this.n == null ? "Unnamed" : this.n; + } + + private String[] ad; + public void setAddresses(List<String> addresses) { + this.ad = this.collectionToStringArray(addresses); + } + public List<String> getAddresses() { + return this.stringArrayToList(this.ad); + } + + private String[] collectionToStringArray( + Collection<String> collection) { + String[] stringArray = null; + if (collection != null && !collection.isEmpty()) { + stringArray = new String[collection.size()]; + int i = 0; + for (String string : collection) { + stringArray[i++] = string; + } + } + return stringArray; + } + private List<String> stringArrayToList(String[] stringArray) { + List<String> list; + if (stringArray == null) { + list = new ArrayList<String>(); + } else { + list = Arrays.asList(stringArray); + } + return list; + } + private SortedSet<String> stringArrayToSortedSet(String[] stringArray) { + SortedSet<String> sortedSet = new TreeSet<String>(); + if (stringArray != null) { + sortedSet.addAll(Arrays.asList(stringArray)); + } + return sortedSet; + } + + private String cc; + public void setCountryCode(String countryCode) { + this.cc = countryCode; + } + public String getCountryCode() { + return this.cc; + } + + private String as; + public void setASNumber(String aSNumber) { + this.as = aSNumber; + } + public String getASNumber() { + return this.as; + } + + private String fs; + public void setFirstSeenMillis(long firstSeenMillis) { + this.fs = DateTimeHelper.format(firstSeenMillis); + } + public long getFirstSeenMillis() { + return DateTimeHelper.parse(this.fs); + } + + private String ls; + public void setLastSeenMillis(long lastSeenMillis) { + this.ls = DateTimeHelper.format(lastSeenMillis); + } + public long getLastSeenMillis() { + return DateTimeHelper.parse(this.ls); + } + + private String[] rf; + public void setRelayFlags(SortedSet<String> relayFlags) { + this.rf = this.collectionToStringArray(relayFlags); + } + public SortedSet<String> getRelayFlags() { + return this.stringArrayToSortedSet(this.rf); + } + + private long cw; + public void setConsensusWeight(long consensusWeight) { + this.cw = consensusWeight; + } + public long getConsensusWeight() { + return this.cw; + } + + private boolean r; + public void setRunning(boolean isRunning) { + this.r = isRunning; + } + public boolean isRunning() { + return this.r; + } + + private String c; + public void setContact(String contact) { + if (contact != null && contact.length() == 0) { + this.c = null; + } else { + this.c = contact; + } + } + public String getContact() { + return this.c; + } + + private String[] ff; + public void setFamilyFingerprints( + SortedSet<String> familyFingerprints) { + this.ff = this.collectionToStringArray(familyFingerprints); + } + public SortedSet<String> getFamilyFingerprints() { + return this.stringArrayToSortedSet(this.ff); + } + + public SummaryDocument(boolean isRelay, String nickname, + String fingerprint, List<String> addresses, long lastSeenMillis, + boolean running, SortedSet<String> relayFlags, long consensusWeight, + String countryCode, long firstSeenMillis, String aSNumber, + String contact, SortedSet<String> familyFingerprints) { + this.setRelay(isRelay); + this.setNickname(nickname); + this.setFingerprint(fingerprint); + this.setAddresses(addresses); + this.setLastSeenMillis(lastSeenMillis); + this.setRunning(running); + this.setRelayFlags(relayFlags); + this.setConsensusWeight(consensusWeight); + this.setCountryCode(countryCode); + this.setFirstSeenMillis(firstSeenMillis); + this.setASNumber(aSNumber); + this.setContact(contact); + this.setFamilyFingerprints(familyFingerprints); + } } diff --git a/src/org/torproject/onionoo/SummaryDocumentWriter.java b/src/org/torproject/onionoo/SummaryDocumentWriter.java new file mode 100644 index 0000000..7b257f5 --- /dev/null +++ b/src/org/torproject/onionoo/SummaryDocumentWriter.java @@ -0,0 +1,87 @@ +/* Copyright 2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo; + +import java.util.ArrayList; +import java.util.List; +import java.util.SortedSet; + +public class SummaryDocumentWriter implements DocumentWriter { + + private DocumentStore documentStore; + + public SummaryDocumentWriter() { + this.documentStore = ApplicationFactory.getDocumentStore(); + } + + private int writtenDocuments = 0, deletedDocuments = 0; + + public void writeDocuments() { + long maxLastSeenMillis = 0L; + for (String fingerprint : this.documentStore.list(NodeStatus.class)) { + NodeStatus nodeStatus = this.documentStore.retrieve( + NodeStatus.class, true, fingerprint); + if (nodeStatus != null && + nodeStatus.getLastSeenMillis() > maxLastSeenMillis) { + maxLastSeenMillis = nodeStatus.getLastSeenMillis(); + } + } + long cutoff = maxLastSeenMillis - DateTimeHelper.ONE_WEEK; + for (String fingerprint : this.documentStore.list(NodeStatus.class)) { + NodeStatus nodeStatus = this.documentStore.retrieve( + NodeStatus.class, + true, fingerprint); + if (nodeStatus == null) { + continue; + } + if (nodeStatus.getLastSeenMillis() < cutoff) { + if (this.documentStore.remove(SummaryDocument.class, + fingerprint)) { + this.deletedDocuments++; + } + continue; + } + boolean isRelay = nodeStatus.isRelay(); + String nickname = nodeStatus.getNickname(); + List<String> addresses = new ArrayList<String>(); + addresses.add(nodeStatus.getAddress()); + for (String orAddress : nodeStatus.getOrAddresses()) { + if (!addresses.contains(orAddress)) { + addresses.add(orAddress); + } + } + for (String exitAddress : nodeStatus.getExitAddresses()) { + if (!addresses.contains(exitAddress)) { + addresses.add(exitAddress); + } + } + long lastSeenMillis = nodeStatus.getLastSeenMillis(); + boolean running = nodeStatus.getRunning(); + SortedSet<String> relayFlags = nodeStatus.getRelayFlags(); + long consensusWeight = nodeStatus.getConsensusWeight(); + String countryCode = nodeStatus.getCountryCode(); + long firstSeenMillis = nodeStatus.getFirstSeenMillis(); + String aSNumber = nodeStatus.getASNumber(); + String contact = nodeStatus.getContact(); + SortedSet<String> familyFingerprints = + nodeStatus.getFamilyFingerprints(); + SummaryDocument summaryDocument = new SummaryDocument(isRelay, + nickname, fingerprint, addresses, lastSeenMillis, running, + relayFlags, consensusWeight, countryCode, firstSeenMillis, + aSNumber, contact, familyFingerprints); + if (this.documentStore.store(summaryDocument, fingerprint)) { + this.writtenDocuments++; + }; + } + Logger.printStatusTime("Wrote summary document files"); + } + + public String getStatsString() { + StringBuilder sb = new StringBuilder(); + sb.append(" " + Logger.formatDecimalNumber(this.writtenDocuments) + + " summary document files written\n"); + sb.append(" " + Logger.formatDecimalNumber(this.deletedDocuments) + + " summary document files deleted\n"); + return sb.toString(); + } +} diff --git a/test/org/torproject/onionoo/DummyDocumentStore.java b/test/org/torproject/onionoo/DummyDocumentStore.java index 6969b8f..5a5905b 100644 --- a/test/org/torproject/onionoo/DummyDocumentStore.java +++ b/test/org/torproject/onionoo/DummyDocumentStore.java @@ -51,7 +51,7 @@ public class DummyDocumentStore extends DocumentStore { } public <T extends Document> SortedSet<String> list( - Class<T> documentType, boolean includeArchive) { + Class<T> documentType) { this.performedListOperations++; return new TreeSet<String>(this.getStoredDocumentsByClass( documentType).keySet()); diff --git a/test/org/torproject/onionoo/ResourceServletTest.java b/test/org/torproject/onionoo/ResourceServletTest.java index f8d3281..e482f82 100644 --- a/test/org/torproject/onionoo/ResourceServletTest.java +++ b/test/org/torproject/onionoo/ResourceServletTest.java @@ -31,7 +31,8 @@ import com.google.gson.Gson; * which tests servlet specifics. */ public class ResourceServletTest { - private SortedMap<String, NodeStatus> relays, bridges; + private SortedMap<String, org.torproject.onionoo.SummaryDocument> + relays, bridges; private long currentTimeMillis = DateTimeHelper.parse( "2013-04-24 12:22:22"); @@ -100,77 +101,73 @@ public class ResourceServletTest { @Before public void createSampleRelaysAndBridges() { - NodeStatus relayTorkaZ = new NodeStatus(true, "TorkaZ", - "000C5F55BD4814B917CC474BD537F1A3B33CCE2A", "62.216.201.221", - new TreeSet<String>(), new TreeSet<String>(Arrays.asList( - new String[] { "62.216.201.222", "62.216.201.223" })), - DateTimeHelper.parse("2013-04-19 05:00:00"), 9001, 0, - new TreeSet<String>(Arrays.asList(new String[] { "Running", - "Valid" })), 20L, "de", null, -1L, "reject", "1-65535", - DateTimeHelper.parse("2013-04-18 05:00:00"), - DateTimeHelper.parse("2013-04-19 05:00:00"), "AS8767", + org.torproject.onionoo.SummaryDocument relayTorkaZ = + new org.torproject.onionoo.SummaryDocument(true, "TorkaZ", + "000C5F55BD4814B917CC474BD537F1A3B33CCE2A", Arrays.asList( + new String[] { "62.216.201.221", "62.216.201.222", + "62.216.201.223" }), DateTimeHelper.parse("2013-04-19 05:00:00"), + false, new TreeSet<String>(Arrays.asList(new String[] { "Running", + "Valid" })), 20L, "de", + DateTimeHelper.parse("2013-04-18 05:00:00"), "AS8767", "torkaz <klaus dot zufall at gmx dot de> " - + "<fb-token:np5_g_83jmf=>", null, new TreeSet<String>( - Arrays.asList(new String[] { - "001C13B3A55A71B977CA65EC85539D79C653A3FC", + + "<fb-token:np5_g_83jmf=>", new TreeSet<String>(Arrays.asList( + new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC", "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" }))); - NodeStatus relayFerrari458 = new NodeStatus(true, "Ferrari458", - "001C13B3A55A71B977CA65EC85539D79C653A3FC", "68.38.171.200", - new TreeSet<String>(Arrays.asList(new String[] { - "[2001:4f8:3:2e::51]:9001" })), new TreeSet<String>(), - DateTimeHelper.parse("2013-04-24 12:00:00"), 9001, 9030, + org.torproject.onionoo.SummaryDocument relayFerrari458 = + new org.torproject.onionoo.SummaryDocument(true, "Ferrari458", + "001C13B3A55A71B977CA65EC85539D79C653A3FC", Arrays.asList( + new String[] { "68.38.171.200", "[2001:4f8:3:2e::51]" }), + DateTimeHelper.parse("2013-04-24 12:00:00"), true, new TreeSet<String>(Arrays.asList(new String[] { "Fast", "Named", "Running", "V2Dir", "Valid" })), 1140L, "us", - "c-68-38-171-200.hsd1.pa.comcast.net", 1366805763009L, "reject", - "1-65535", DateTimeHelper.parse("2013-02-12 16:00:00"), - DateTimeHelper.parse("2013-02-26 18:00:00"), "AS7922", null, null, + DateTimeHelper.parse("2013-02-12 16:00:00"), "AS7922", null, new TreeSet<String>(Arrays.asList(new String[] { "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" }))); - relayFerrari458.setRunning(true); - NodeStatus relayTimMayTribute = new NodeStatus(true, "TimMayTribute", - "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B", "89.69.68.246", - new TreeSet<String>(), new TreeSet<String>(), - DateTimeHelper.parse("2013-04-22 20:00:00"), 9001, 9030, + org.torproject.onionoo.SummaryDocument relayTimMayTribute = + new org.torproject.onionoo.SummaryDocument(true, "TimMayTribute", + "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B", Arrays.asList( + new String[] { "89.69.68.246" }), + DateTimeHelper.parse("2013-04-22 20:00:00"), false, new TreeSet<String>(Arrays.asList(new String[] { "Fast", - "Running", "Unnamed", "V2Dir", "Valid" })), 63L, "a1", null, -1L, - "reject", "1-65535", DateTimeHelper.parse("2013-04-16 18:00:00"), + "Running", "Unnamed", "V2Dir", "Valid" })), 63L, "a1", DateTimeHelper.parse("2013-04-16 18:00:00"), "AS6830", "1024D/51E2A1C7 steven j. murdoch " + "<tor+steven.murdoch(a)cl.cam.ac.uk> <fb-token:5sr_k_zs2wm=>", - null, new TreeSet<String>()); - NodeStatus bridgeec2bridgercc7f31fe = new NodeStatus(false, + new TreeSet<String>()); + org.torproject.onionoo.SummaryDocument bridgeec2bridgercc7f31fe = + new org.torproject.onionoo.SummaryDocument(false, "ec2bridgercc7f31fe", "0000831B236DFF73D409AD17B40E2A728A53994F", - "10.199.7.176", new TreeSet<String>(), new TreeSet<String>(), - DateTimeHelper.parse("2013-04-21 18:07:03"), 443, 0, + Arrays.asList(new String[] { "10.199.7.176" }), + DateTimeHelper.parse("2013-04-21 18:07:03"), false, new TreeSet<String>(Arrays.asList(new String[] { "Valid" })), -1L, - "??", null, -1L, null, null, - DateTimeHelper.parse("2013-04-20 15:37:04"), -1L, null, null, - null, null); - NodeStatus bridgeUnnamed = new NodeStatus(false, "Unnamed", - "0002D9BDBBC230BD9C78FF502A16E0033EF87E0C", "10.0.52.84", - new TreeSet<String>(), new TreeSet<String>(), - DateTimeHelper.parse("2013-04-20 17:37:04"), 443, 0, + null, DateTimeHelper.parse("2013-04-20 15:37:04"), null, null, + null); + org.torproject.onionoo.SummaryDocument bridgeUnnamed = + new org.torproject.onionoo.SummaryDocument(false, "Unnamed", + "0002D9BDBBC230BD9C78FF502A16E0033EF87E0C", Arrays.asList( + new String[] { "10.0.52.84" }), + DateTimeHelper.parse("2013-04-20 17:37:04"), false, new TreeSet<String>(Arrays.asList(new String[] { "Valid" })), -1L, - "??", null, -1L, null, null, - DateTimeHelper.parse("2013-04-14 07:07:05"), -1L, null, null, - null, null); - NodeStatus bridgegummy = new NodeStatus(false, "gummy", - "1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB756", "10.63.169.98", - new TreeSet<String>(), new TreeSet<String>(), - DateTimeHelper.parse("2013-04-24 01:07:04"), 9001, 0, + null, DateTimeHelper.parse("2013-04-14 07:07:05"), null, null, + null); + org.torproject.onionoo.SummaryDocument bridgegummy = + new org.torproject.onionoo.SummaryDocument(false, "gummy", + "1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB756", Arrays.asList( + new String[] { "10.63.169.98" }), + DateTimeHelper.parse("2013-04-24 01:07:04"), true, new TreeSet<String>(Arrays.asList(new String[] { "Running", - "Valid" })), -1L, "??", null, -1L, null, null, - DateTimeHelper.parse("2013-01-16 21:07:04"), -1L, null, null, - null, null); - bridgegummy.setRunning(true); - this.relays = new TreeMap<String, NodeStatus>(); + "Valid" })), -1L, null, + DateTimeHelper.parse("2013-01-16 21:07:04"), null, null, null); + this.relays = + new TreeMap<String, org.torproject.onionoo.SummaryDocument>(); this.relays.put("000C5F55BD4814B917CC474BD537F1A3B33CCE2A", relayTorkaZ); this.relays.put("001C13B3A55A71B977CA65EC85539D79C653A3FC", relayFerrari458); this.relays.put("0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B", relayTimMayTribute); - this.bridges = new TreeMap<String, NodeStatus>(); + this.bridges = + new TreeMap<String, org.torproject.onionoo.SummaryDocument>(); this.bridges.put("0000831B236DFF73D409AD17B40E2A728A53994F", bridgeec2bridgercc7f31fe); this.bridges.put("0002D9BDBBC230BD9C78FF502A16E0033EF87E0C", @@ -203,10 +200,12 @@ public class ResourceServletTest { updateStatus.setDocumentString(String.valueOf( this.currentTimeMillis)); documentStore.addDocument(updateStatus, null); - for (Map.Entry<String, NodeStatus> e : relays.entrySet()) { + for (Map.Entry<String, org.torproject.onionoo.SummaryDocument> e : + this.relays.entrySet()) { documentStore.addDocument(e.getValue(), e.getKey()); } - for (Map.Entry<String, NodeStatus> e : bridges.entrySet()) { + for (Map.Entry<String, org.torproject.onionoo.SummaryDocument> e : + this.bridges.entrySet()) { documentStore.addDocument(e.getValue(), e.getKey()); } ApplicationFactory.setDocumentStore(documentStore);

1 0