- tor-commits - lists.torproject.org

[translation/tails-misc_completed] Update translations for tails-misc_completed
by translation＠torproject.org 01 May '16

01 May '16

commit d6250716e2dc8023a4e4e100cd7fe73ae9e1c02b Author: Translation commit bot <translation(a)torproject.org> Date: Sun May 1 08:15:55 2016 +0000 Update translations for tails-misc_completed --- en_GB.po | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/en_GB.po b/en_GB.po index 3b846ad..4019bb1 100644 --- a/en_GB.po +++ b/en_GB.po @@ -11,8 +11,8 @@ msgid "" msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2016-03-17 15:03+0100\n" -"PO-Revision-Date: 2016-03-21 16:27+0000\n" +"POT-Creation-Date: 2016-04-25 14:02+0200\n" +"PO-Revision-Date: 2016-05-01 07:49+0000\n" "Last-Translator: Andi Chandler <andi(a)gowling.com>\n" "Language-Team: English (United Kingdom) (http://www.transifex.com/otf/torproject/language/en_GB/)\n" "MIME-Version: 1.0\n" @@ -62,13 +62,13 @@ msgid "Do you want to start Electrum anyway?" msgstr "Do you want to start Electrum anyway?" #: config/chroot_local-includes/usr/local/bin/electrum:23 -#: config/chroot_local-includes/usr/local/bin/icedove:23 +#: config/chroot_local-includes/usr/local/bin/icedove:30 #: config/chroot_local-includes/usr/local/sbin/unsafe-browser:41 msgid "_Launch" msgstr "_Launch" #: config/chroot_local-includes/usr/local/bin/electrum:24 -#: config/chroot_local-includes/usr/local/bin/icedove:24 +#: config/chroot_local-includes/usr/local/bin/icedove:31 #: config/chroot_local-includes/usr/local/sbin/unsafe-browser:42 msgid "_Exit" msgstr "_Exit" @@ -230,17 +230,24 @@ msgstr "Output of GnuPG:" msgid "Other messages provided by GnuPG:" msgstr "Other messages provided by GnuPG:" -#: config/chroot_local-includes/usr/local/bin/icedove:19 +#: config/chroot_local-includes/usr/local/bin/icedove:20 msgid "The Claws Mail persistence feature is activated." msgstr "The Claws Mail persistence feature is activated." -#: config/chroot_local-includes/usr/local/bin/icedove:21 +#: config/chroot_local-includes/usr/local/bin/icedove:22 msgid "" "If you have emails saved in Claws Mail, you should <a " "href='https://tails.boum.org/doc/anonymous_internet/claws_mail_to_icedove'>migrate" " your data</a> before starting Icedove." msgstr "If you have emails saved in Claws Mail, you should <a href='https://tails.boum.org/doc/anonymous_internet/claws_mail_to_icedove'>migrate your data</a> before starting Icedove." +#: config/chroot_local-includes/usr/local/bin/icedove:27 +msgid "" +"If you already migrated your emails to Icedove, you should <a " +"href='https://tails.boum.org/doc/anonymous_internet/claws_mail_to_icedove#delete'>delete" +" all your Claws Mail data</a> to remove this warning." +msgstr "If you already migrated your emails to Icedove, you should <a href='https://tails.boum.org/doc/anonymous_internet/claws_mail_to_icedove#delete'>delete all your Claws Mail data</a> to remove this warning." + #: config/chroot_local-includes/usr/share/gnome-shell/extensions/shutdown-helper@tails.boum.org/extension.js:71 msgid "Restart" msgstr "Restart"

1 0

[translation/tails-misc] Update translations for tails-misc
by translation＠torproject.org 01 May '16

01 May '16

commit 45e85b67e47118b2da4f6b0f07b7985811b1c99d Author: Translation commit bot <translation(a)torproject.org> Date: Sun May 1 08:15:52 2016 +0000 Update translations for tails-misc --- en_GB.po | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/en_GB.po b/en_GB.po index e864a94..4019bb1 100644 --- a/en_GB.po +++ b/en_GB.po @@ -12,8 +12,8 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2016-04-25 14:02+0200\n" -"PO-Revision-Date: 2016-04-26 09:06+0000\n" -"Last-Translator: carolyn <carolyn(a)anhalt.org>\n" +"PO-Revision-Date: 2016-05-01 07:49+0000\n" +"Last-Translator: Andi Chandler <andi(a)gowling.com>\n" "Language-Team: English (United Kingdom) (http://www.transifex.com/otf/torproject/language/en_GB/)\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -246,7 +246,7 @@ msgid "" "If you already migrated your emails to Icedove, you should <a " "href='https://tails.boum.org/doc/anonymous_internet/claws_mail_to_icedove#delete'>delete" " all your Claws Mail data</a> to remove this warning." -msgstr "" +msgstr "If you already migrated your emails to Icedove, you should <a href='https://tails.boum.org/doc/anonymous_internet/claws_mail_to_icedove#delete'>delete all your Claws Mail data</a> to remove this warning." #: config/chroot_local-includes/usr/share/gnome-shell/extensions/shutdown-helper@tails.boum.org/extension.js:71 msgid "Restart"

1 0

[translation/tor-launcher-network-settings_completed] Update translations for tor-launcher-network-settings_completed
by translation＠torproject.org 01 May '16

01 May '16

commit 28cff4d97630e37c4c9f45bd22545a93f0ea09a3 Author: Translation commit bot <translation(a)torproject.org> Date: Sun May 1 08:15:49 2016 +0000 Update translations for tor-launcher-network-settings_completed --- en_GB/network-settings.dtd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/en_GB/network-settings.dtd b/en_GB/network-settings.dtd index 75ece3b..0d2e9dc 100644 --- a/en_GB/network-settings.dtd +++ b/en_GB/network-settings.dtd @@ -65,7 +65,7 @@ <!ENTITY torsettings.copyLog "Copy Tor Log To Clipboard"> <!ENTITY torsettings.bridgeHelpTitle "Bridge Relay Help"> <!ENTITY torsettings.bridgeHelp1 "If you are unable to connect to the Tor network, it could be that your Internet Service Provider (ISP) or another agency is blocking Tor.  Often, you can work around this problem by using Tor Bridges, which are unlisted relays that are more difficult to block."> -<!ENTITY torsettings.bridgeHelp1B "You may use the preconfigured, provided set of bridge addresses or you may obtain a custom set of addresses by using one of these three methods:"> +<!ENTITY torsettings.bridgeHelp1B "You may use the preconfigured, provided set of bridge addresses or you may obtain a custom set of addresses by using one of these methods:"> <!ENTITY torsettings.bridgeHelp2Heading "Through the Web"> <!ENTITY torsettings.bridgeHelp2 "Use a web browser to visit https://bridges.torproject.org"> <!ENTITY torsettings.bridgeHelp3Heading "Through the Email Autoresponder">

1 0

[translation/tor-launcher-properties_completed] Update translations for tor-launcher-properties_completed
by translation＠torproject.org 01 May '16

01 May '16

commit 267ced773db2ffc79507a4f010945985c2d4a184 Author: Translation commit bot <translation(a)torproject.org> Date: Sun May 1 08:15:41 2016 +0000 Update translations for tor-launcher-properties_completed --- en_GB/torlauncher.properties | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/en_GB/torlauncher.properties b/en_GB/torlauncher.properties index 02be756..6b471a3 100644 --- a/en_GB/torlauncher.properties +++ b/en_GB/torlauncher.properties @@ -1,4 +1,4 @@ -### Copyright (c) 2014, The Tor Project, Inc. +### Copyright (c) 2016, The Tor Project, Inc. ### See LICENSE for licensing information. torlauncher.error_title=Tor Launcher @@ -13,8 +13,8 @@ torlauncher.tor_bootstrap_failed_details=%1$S failed (%2$S). torlauncher.unable_to_start_tor=Unable to start Tor.\n\n%S torlauncher.tor_missing=The Tor executable is missing. -torlauncher.torrc_missing=The torrc file is missing. -torlauncher.datadir_missing=The Tor data directory does not exist. +torlauncher.torrc_missing=The torrc file is missing and could not be created. +torlauncher.datadir_missing=The Tor data directory does not exist and could not be created. torlauncher.password_hash_missing=Failed to get hashed password. torlauncher.failed_to_get_settings=Unable to retrieve Tor settings.\n\n%S @@ -36,6 +36,7 @@ torlauncher.quit_win=Exit torlauncher.done=Done torlauncher.forAssistance=For assistance, contact %S +torlauncher.forAssistance2=For assistance, visit %S torlauncher.copiedNLogMessages=Copy complete. %S Tor log messages are ready to be pasted into a text editor or an email message.

1 0

[translation/tails-persistence-setup_completed] Update translations for tails-persistence-setup_completed
by translation＠torproject.org 01 May '16

01 May '16

commit 31cd41899c61996e674512415db02a079343f519 Author: Translation commit bot <translation(a)torproject.org> Date: Sun May 1 08:15:27 2016 +0000 Update translations for tails-persistence-setup_completed --- en_GB/en_GB.po | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/en_GB/en_GB.po b/en_GB/en_GB.po index 49eed5e..f2ced47 100644 --- a/en_GB/en_GB.po +++ b/en_GB/en_GB.po @@ -3,7 +3,7 @@ # This file is distributed under the same license as the PACKAGE package. # # Translators: -# Andi Chandler <andi(a)gowling.com>, 2014-2015 +# Andi Chandler <andi(a)gowling.com>, 2014-2016 # Billy Humphreys <sysop(a)enderbbs.enderservices.tk>, 2014 # newharha ehrara <dbybgohg(a)yomail.info>, 2015 # Richard Shaylor <rshaylor(a)me.com>, 2014 @@ -11,9 +11,9 @@ msgid "" msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: Tails developers <tails(a)boum.org>\n" -"POT-Creation-Date: 2016-01-25 17:34+0100\n" -"PO-Revision-Date: 2016-03-21 16:32+0000\n" -"Last-Translator: carolyn <carolyn(a)anhalt.org>\n" +"POT-Creation-Date: 2016-04-24 16:40+0200\n" +"PO-Revision-Date: 2016-05-01 07:49+0000\n" +"Last-Translator: Andi Chandler <andi(a)gowling.com>\n" "Language-Team: English (United Kingdom) (http://www.transifex.com/otf/torproject/language/en_GB/)\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -118,10 +118,18 @@ msgid "Lists downloaded by APT" msgstr "Lists downloaded by APT" #: ../lib/Tails/Persistence/Configuration/Presets.pm:168 +msgid "Mumble server" +msgstr "Mumble server" + +#: ../lib/Tails/Persistence/Configuration/Presets.pm:170 +msgid "Mumble server configuration and its Tor Hidden Service address" +msgstr "Mumble server configuration and its Tor Hidden Service address" + +#: ../lib/Tails/Persistence/Configuration/Presets.pm:178 msgid "Dotfiles" msgstr "Dotfiles" -#: ../lib/Tails/Persistence/Configuration/Presets.pm:170 +#: ../lib/Tails/Persistence/Configuration/Presets.pm:180 msgid "" "Symlink into $HOME every file or directory found in the `dotfiles' directory" msgstr "Symlink into $HOME every file or directory found in the `dotfiles' directory"

1 0

[translation/tails-persistence-setup] Update translations for tails-persistence-setup
by translation＠torproject.org 01 May '16

01 May '16

commit 5dd4f605fb19b144bcbc8e7e26d58a376b12779c Author: Translation commit bot <translation(a)torproject.org> Date: Sun May 1 08:15:23 2016 +0000 Update translations for tails-persistence-setup --- en_GB/en_GB.po | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/en_GB/en_GB.po b/en_GB/en_GB.po index f50f773..f2ced47 100644 --- a/en_GB/en_GB.po +++ b/en_GB/en_GB.po @@ -3,7 +3,7 @@ # This file is distributed under the same license as the PACKAGE package. # # Translators: -# Andi Chandler <andi(a)gowling.com>, 2014-2015 +# Andi Chandler <andi(a)gowling.com>, 2014-2016 # Billy Humphreys <sysop(a)enderbbs.enderservices.tk>, 2014 # newharha ehrara <dbybgohg(a)yomail.info>, 2015 # Richard Shaylor <rshaylor(a)me.com>, 2014 @@ -12,8 +12,8 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: Tails developers <tails(a)boum.org>\n" "POT-Creation-Date: 2016-04-24 16:40+0200\n" -"PO-Revision-Date: 2016-04-25 08:33+0000\n" -"Last-Translator: carolyn <carolyn(a)anhalt.org>\n" +"PO-Revision-Date: 2016-05-01 07:49+0000\n" +"Last-Translator: Andi Chandler <andi(a)gowling.com>\n" "Language-Team: English (United Kingdom) (http://www.transifex.com/otf/torproject/language/en_GB/)\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -119,11 +119,11 @@ msgstr "Lists downloaded by APT" #: ../lib/Tails/Persistence/Configuration/Presets.pm:168 msgid "Mumble server" -msgstr "" +msgstr "Mumble server" #: ../lib/Tails/Persistence/Configuration/Presets.pm:170 msgid "Mumble server configuration and its Tor Hidden Service address" -msgstr "" +msgstr "Mumble server configuration and its Tor Hidden Service address" #: ../lib/Tails/Persistence/Configuration/Presets.pm:178 msgid "Dotfiles"

1 0

[collector/master] Relax CatchParameterName check.
by karsten＠torproject.org 30 Apr '16

30 Apr '16

commit c88783d6b52981d2e5cb300c4b3d5222c0f34976 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sat Apr 30 21:02:59 2016 +0200 Relax CatchParameterName check. --- resources/metrics_checks.xml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/resources/metrics_checks.xml b/resources/metrics_checks.xml index bd1d7ba..0777894 100644 --- a/resources/metrics_checks.xml +++ b/resources/metrics_checks.xml @@ -11,6 +11,9 @@ - Replaced com.google with org.torproject in import statement ordering [CustomImportOrder]. + - Relaxed requirement that catch parameters must be at least two + characters long [CatchParameterName]. + Checkstyle is very configurable. Be sure to read the documentation at http://checkstyle.sf.net (or in your downloaded distribution). @@ -111,7 +114,7 @@ value="Parameter name ''{0}'' must match pattern ''{1}''."/> </module> <module name="CatchParameterName"> - <property name="format" value="^[a-z][a-z0-9][a-zA-Z0-9]*$"/> + <property name="format" value="^[a-z][a-zA-Z0-9]*$"/> <message key="name.invalidPattern" value="Catch parameter name ''{0}'' must match pattern ''{1}''."/> </module>

1 0

[collector/master] Resolve a bunch of checkstyle warnings.
by karsten＠torproject.org 30 Apr '16

30 Apr '16

commit 8bd642198cd78c288516a68aa684eafefe86e074 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sat Apr 30 08:24:55 2016 +0200 Resolve a bunch of checkstyle warnings. --- resources/metrics_checks.xml | 8 +- .../bridgedescs/BridgeDescriptorParser.java | 7 +- .../bridgedescs/BridgeSnapshotReader.java | 44 +++-- .../bridgedescs/SanitizedBridgesWriter.java | 173 +++++++++--------- .../BridgePoolAssignmentsProcessor.java | 36 ++-- .../collector/exitlists/ExitListDownloader.java | 33 ++-- .../collector/index/CreateIndexJson.java | 21 ++- .../torproject/collector/main/Configuration.java | 31 +++- src/org/torproject/collector/main/LockFile.java | 3 +- .../collector/main/LoggingConfiguration.java | 1 + .../collector/relaydescs/ArchiveReader.java | 30 ++-- .../collector/relaydescs/ArchiveWriter.java | 135 ++++++++------ .../relaydescs/CachedRelayDescriptorReader.java | 55 +++--- .../collector/relaydescs/ReferenceChecker.java | 64 ++++--- .../relaydescs/RelayDescriptorDownloader.java | 198 +++++++++++++-------- .../relaydescs/RelayDescriptorParser.java | 61 ++++--- .../collector/torperf/TorperfDownloader.java | 62 ++++--- 17 files changed, 564 insertions(+), 398 deletions(-) diff --git a/resources/metrics_checks.xml b/resources/metrics_checks.xml index 4fa2737..bd1d7ba 100644 --- a/resources/metrics_checks.xml +++ b/resources/metrics_checks.xml @@ -5,7 +5,11 @@ <!-- Checkstyle configuration that checks the Google coding conventions from Google Java Style - that can be found at https://google.github.io/styleguide/javaguide.html. + that can be found at https://google.github.io/styleguide/javaguide.html with the following + modifications: + + - Replaced com.google with org.torproject in import statement ordering + [CustomImportOrder]. Checkstyle is very configurable. Be sure to read the documentation at http://checkstyle.sf.net (or in your downloaded distribution). @@ -159,7 +163,7 @@ <module name="OverloadMethodsDeclarationOrder"/> <module name="VariableDeclarationUsageDistance"/> <module name="CustomImportOrder"> - <property name="specialImportsRegExp" value="com.google"/> + <property name="specialImportsRegExp" value="org.torproject"/> <property name="sortImportsInGroupAlphabetically" value="true"/> <property name="customImportOrderRules" value="STATIC###SPECIAL_IMPORTS###THIRD_PARTY_PACKAGE###STANDARD_JAVA_PACKAGE"/> </module> diff --git a/src/org/torproject/collector/bridgedescs/BridgeDescriptorParser.java b/src/org/torproject/collector/bridgedescs/BridgeDescriptorParser.java index c14875b..f683ea0 100644 --- a/src/org/torproject/collector/bridgedescs/BridgeDescriptorParser.java +++ b/src/org/torproject/collector/bridgedescs/BridgeDescriptorParser.java @@ -1,5 +1,6 @@ -/* Copyright 2010--2012 The Tor Project +/* Copyright 2010--2016 The Tor Project * See LICENSE for licensing information */ + package org.torproject.collector.bridgedescs; import java.io.BufferedReader; @@ -9,13 +10,17 @@ import java.util.logging.Level; import java.util.logging.Logger; public class BridgeDescriptorParser { + private SanitizedBridgesWriter sbw; + private Logger logger; + public BridgeDescriptorParser(SanitizedBridgesWriter sbw) { this.sbw = sbw; this.logger = Logger.getLogger(BridgeDescriptorParser.class.getName()); } + public void parse(byte[] allData, String dateTime) { try { BufferedReader br = new BufferedReader(new StringReader( diff --git a/src/org/torproject/collector/bridgedescs/BridgeSnapshotReader.java b/src/org/torproject/collector/bridgedescs/BridgeSnapshotReader.java index b16938d..2d41d18 100644 --- a/src/org/torproject/collector/bridgedescs/BridgeSnapshotReader.java +++ b/src/org/torproject/collector/bridgedescs/BridgeSnapshotReader.java @@ -1,7 +1,13 @@ -/* Copyright 2010--2012 The Tor Project +/* Copyright 2010--2016 The Tor Project * See LICENSE for licensing information */ + package org.torproject.collector.bridgedescs; +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; + import java.io.BufferedInputStream; import java.io.BufferedReader; import java.io.BufferedWriter; @@ -20,11 +26,6 @@ import java.util.TreeSet; import java.util.logging.Level; import java.util.logging.Logger; -import org.apache.commons.codec.binary.Hex; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; -import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; - /** * Reads the half-hourly snapshots of bridge descriptors from Tonga. */ @@ -32,8 +33,8 @@ public class BridgeSnapshotReader { public BridgeSnapshotReader(BridgeDescriptorParser bdp, File bridgeDirectoriesDir, File statsDirectory) { - if (bdp == null || bridgeDirectoriesDir == null || - statsDirectory == null) { + if (bdp == null || bridgeDirectoriesDir == null + || statsDirectory == null) { throw new IllegalArgumentException(); } @@ -62,11 +63,15 @@ public class BridgeSnapshotReader { } } logger.fine("Importing files in directory " + bridgeDirectoriesDir - + "/..."); + + "/..."); Set<String> descriptorImportHistory = new HashSet<String>(); - int parsedFiles = 0, skippedFiles = 0, parsedStatuses = 0, - parsedServerDescriptors = 0, skippedServerDescriptors = 0, - parsedExtraInfoDescriptors = 0, skippedExtraInfoDescriptors = 0; + int parsedFiles = 0; + int skippedFiles = 0; + int parsedStatuses = 0; + int parsedServerDescriptors = 0; + int skippedServerDescriptors = 0; + int parsedExtraInfoDescriptors = 0; + int skippedExtraInfoDescriptors = 0; Stack<File> filesInInputDir = new Stack<File>(); filesInInputDir.add(bdDir); while (!filesInInputDir.isEmpty()) { @@ -118,9 +123,9 @@ public class BridgeSnapshotReader { break; } } - if (firstLine.startsWith("published ") || - firstLine.startsWith("flag-thresholds ") || - firstLine.startsWith("r ")) { + if (firstLine.startsWith("published ") + || firstLine.startsWith("flag-thresholds ") + || firstLine.startsWith("r ")) { bdp.parse(allData, dateTime); parsedStatuses++; } else if (descriptorImportHistory.contains(fileDigest)) { @@ -129,10 +134,11 @@ public class BridgeSnapshotReader { skippedFiles++; continue; } else { - int start = -1, sig = -1, end = -1; - String startToken = - firstLine.startsWith("router ") ? - "router " : "extra-info "; + int start = -1; + int sig = -1; + int end = -1; + String startToken = firstLine.startsWith("router ") + ? "router " : "extra-info "; String sigToken = "\nrouter-signature\n"; String endToken = "\n-----END SIGNATURE-----\n"; while (end < ascii.length()) { diff --git a/src/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java b/src/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java index 4aa13ea..f8d61c0 100644 --- a/src/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java +++ b/src/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java @@ -1,7 +1,17 @@ /* Copyright 2010--2016 The Tor Project * See LICENSE for licensing information */ + package org.torproject.collector.bridgedescs; +import org.torproject.collector.main.Configuration; +import org.torproject.collector.main.LockFile; +import org.torproject.collector.main.LoggingConfiguration; + +import org.apache.commons.codec.DecoderException; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.codec.digest.DigestUtils; + import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; @@ -25,14 +35,6 @@ import java.util.TreeMap; import java.util.logging.Level; import java.util.logging.Logger; -import org.apache.commons.codec.DecoderException; -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.codec.binary.Hex; -import org.apache.commons.codec.digest.DigestUtils; -import org.torproject.collector.main.Configuration; -import org.torproject.collector.main.LockFile; -import org.torproject.collector.main.LoggingConfiguration; - /** * Sanitizes bridge descriptors, i.e., removes all possibly sensitive * information from them, and writes them to a local directory structure. @@ -125,8 +127,8 @@ public class SanitizedBridgesWriter extends Thread { config.getLimitBridgeDescriptorMappings(); File statsDirectory = new File("stats"); - if (bridgeDirectoriesDirectory == null || - sanitizedBridgesDirectory == null || statsDirectory == null) { + if (bridgeDirectoriesDirectory == null + || sanitizedBridgesDirectory == null || statsDirectory == null) { throw new IllegalArgumentException(); } @@ -169,9 +171,9 @@ public class SanitizedBridgesWriter extends Thread { String line; while ((line = br.readLine()) != null) { String[] parts = line.split(","); - if ((line.length() != ("yyyy-MM,".length() + 31 * 2) && - line.length() != ("yyyy-MM,".length() + 50 * 2)) || - parts.length != 2) { + if ((line.length() != ("yyyy-MM,".length() + 31 * 2) + && line.length() != ("yyyy-MM,".length() + 50 * 2)) + || parts.length != 2) { this.logger.warning("Invalid line in bridge-ip-secrets file " + "starting with '" + line.substring(0, 7) + "'! " + "Not calculating any IP address hashes in this " @@ -364,8 +366,8 @@ public class SanitizedBridgesWriter extends Thread { } private byte[] getSecretForMonth(String month) throws IOException { - if (!this.secretsForHashingIPAddresses.containsKey(month) || - this.secretsForHashingIPAddresses.get(month).length == 31) { + if (!this.secretsForHashingIPAddresses.containsKey(month) + || this.secretsForHashingIPAddresses.get(month).length == 31) { byte[] secret = new byte[50]; this.secureRandom.nextBytes(secret); if (this.secretsForHashingIPAddresses.containsKey(month)) { @@ -420,8 +422,8 @@ public class SanitizedBridgesWriter extends Thread { maxNetworkStatusPublishedTime = publicationTime; } - if (this.bridgeSanitizingCutOffTimestamp. - compareTo(publicationTime) > 0) { + if (this.bridgeSanitizingCutOffTimestamp + .compareTo(publicationTime) > 0) { this.logger.log(!this.haveWarnedAboutInterval ? Level.WARNING : Level.FINE, "Sanitizing and storing network status with " + "publication time outside our descriptor sanitizing " @@ -476,9 +478,9 @@ public class SanitizedBridgesWriter extends Thread { String dirPort = parts[8]; /* Determine most recent descriptor publication time. */ - if (descPublicationTime.compareTo(publicationTime) <= 0 && - (mostRecentDescPublished == null || - descPublicationTime.compareTo( + if (descPublicationTime.compareTo(publicationTime) <= 0 + && (mostRecentDescPublished == null + || descPublicationTime.compareTo( mostRecentDescPublished) > 0)) { mostRecentDescPublished = descPublicationTime; } @@ -515,9 +517,9 @@ public class SanitizedBridgesWriter extends Thread { } /* Nothing special about s, w, and p lines; just copy them. */ - } else if (line.startsWith("s ") || line.equals("s") || - line.startsWith("w ") || line.equals("w") || - line.startsWith("p ") || line.equals("p")) { + } else if (line.startsWith("s ") || line.equals("s") + || line.startsWith("w ") || line.equals("w") + || line.startsWith("p ") || line.equals("p")) { scrubbed.append(line + "\n"); /* There should be nothing else but r, w, p, and s lines in the @@ -541,9 +543,9 @@ public class SanitizedBridgesWriter extends Thread { SimpleDateFormat formatter = new SimpleDateFormat( "yyyy-MM-dd HH:mm:ss"); formatter.setTimeZone(TimeZone.getTimeZone("UTC")); - if (formatter.parse(publicationTime).getTime() - - formatter.parse(mostRecentDescPublished).getTime() > - 60L * 60L * 1000L) { + if (formatter.parse(publicationTime).getTime() + - formatter.parse(mostRecentDescPublished).getTime() + > 60L * 60L * 1000L) { this.logger.warning("The most recent descriptor in the bridge " + "network status published at " + publicationTime + " was " + "published at " + mostRecentDescPublished + " which is " @@ -609,16 +611,21 @@ public class SanitizedBridgesWriter extends Thread { } /* Parse descriptor to generate a sanitized version. */ - String scrubbedDesc = null, published = null, - masterKeyEd25519FromIdentityEd25519 = null; + String scrubbedDesc = null; + String published = null; + String masterKeyEd25519FromIdentityEd25519 = null; try { BufferedReader br = new BufferedReader(new StringReader( new String(data, "US-ASCII"))); StringBuilder scrubbed = new StringBuilder(); - String line = null, hashedBridgeIdentity = null, address = null, - routerLine = null, scrubbedAddress = null, - masterKeyEd25519 = null; - List<String> orAddresses = null, scrubbedOrAddresses = null; + String line = null; + String hashedBridgeIdentity = null; + String address = null; + String routerLine = null; + String scrubbedAddress = null; + String masterKeyEd25519 = null; + List<String> orAddresses = null; + List<String> scrubbedOrAddresses = null; boolean skipCrypto = false; while ((line = br.readLine()) != null) { @@ -649,8 +656,8 @@ public class SanitizedBridgesWriter extends Thread { if (published.compareTo(maxServerDescriptorPublishedTime) > 0) { maxServerDescriptorPublishedTime = published; } - if (this.bridgeSanitizingCutOffTimestamp. - compareTo(published) > 0) { + if (this.bridgeSanitizingCutOffTimestamp + .compareTo(published) > 0) { this.logger.log(!this.haveWarnedAboutInterval ? Level.WARNING : Level.FINE, "Sanitizing and storing " + "server descriptor with publication time outside our " @@ -661,15 +668,15 @@ public class SanitizedBridgesWriter extends Thread { /* Parse the fingerprint to determine the hashed bridge * identity. */ - } else if (line.startsWith("opt fingerprint ") || - line.startsWith("fingerprint ")) { - String fingerprint = line.substring(line.startsWith("opt ") ? - "opt fingerprint".length() : "fingerprint".length()). - replaceAll(" ", "").toLowerCase(); + } else if (line.startsWith("opt fingerprint ") + || line.startsWith("fingerprint ")) { + String fingerprint = line.substring(line.startsWith("opt ") + ? "opt fingerprint".length() : "fingerprint".length()) + .replaceAll(" ", "").toLowerCase(); byte[] fingerprintBytes = Hex.decodeHex( fingerprint.toCharArray()); - hashedBridgeIdentity = DigestUtils.shaHex(fingerprintBytes). - toLowerCase(); + hashedBridgeIdentity = DigestUtils.shaHex(fingerprintBytes) + .toLowerCase(); try { scrubbedAddress = scrubIpv4Address(address, fingerprintBytes, published); @@ -695,9 +702,10 @@ public class SanitizedBridgesWriter extends Thread { } scrubbed.append((line.startsWith("opt ") ? "opt " : "") + "fingerprint"); - for (int i = 0; i < hashedBridgeIdentity.length() / 4; i++) + for (int i = 0; i < hashedBridgeIdentity.length() / 4; i++) { scrubbed.append(" " + hashedBridgeIdentity.substring(4 * i, 4 * (i + 1)).toUpperCase()); + } scrubbed.append("\n"); /* Replace the contact line (if present) with a generic one. */ @@ -722,8 +730,8 @@ public class SanitizedBridgesWriter extends Thread { /* Replace extra-info digest with the hashed digest of the * non-scrubbed descriptor. */ - } else if (line.startsWith("opt extra-info-digest ") || - line.startsWith("extra-info-digest ")) { + } else if (line.startsWith("opt extra-info-digest ") + || line.startsWith("extra-info-digest ")) { String[] parts = line.split(" "); if (line.startsWith("opt ")) { scrubbed.append("opt "); @@ -733,8 +741,8 @@ public class SanitizedBridgesWriter extends Thread { Hex.decodeHex(parts[1].toCharArray())).toUpperCase()); if (parts.length > 2) { scrubbed.append(" " + Base64.encodeBase64String( - DigestUtils.sha256(Base64.decodeBase64(parts[2]))). - replaceAll("=", "")); + DigestUtils.sha256(Base64.decodeBase64(parts[2]))) + .replaceAll("=", "")); } scrubbed.append("\n"); @@ -752,8 +760,8 @@ public class SanitizedBridgesWriter extends Thread { /* Extract master-key-ed25519 from identity-ed25519. */ } else if (line.equals("identity-ed25519")) { StringBuilder sb = new StringBuilder(); - while ((line = br.readLine()) != null && - !line.equals("-----END ED25519 CERT-----")) { + while ((line = br.readLine()) != null + && !line.equals("-----END ED25519 CERT-----")) { if (line.equals("-----BEGIN ED25519 CERT-----")) { continue; } @@ -764,8 +772,8 @@ public class SanitizedBridgesWriter extends Thread { sb.toString()); String sha256MasterKeyEd25519 = Base64.encodeBase64String( DigestUtils.sha256(Base64.decodeBase64( - masterKeyEd25519FromIdentityEd25519 + "="))). - replaceAll("=", ""); + masterKeyEd25519FromIdentityEd25519 + "="))) + .replaceAll("=", ""); scrubbed.append("master-key-ed25519 " + sha256MasterKeyEd25519 + "\n"); if (masterKeyEd25519 != null && !masterKeyEd25519.equals( @@ -778,8 +786,8 @@ public class SanitizedBridgesWriter extends Thread { /* Verify that identity-ed25519 and master-key-ed25519 match. */ } else if (line.startsWith("master-key-ed25519 ")) { masterKeyEd25519 = line.substring(line.indexOf(" ") + 1); - if (masterKeyEd25519FromIdentityEd25519 != null && - !masterKeyEd25519FromIdentityEd25519.equals( + if (masterKeyEd25519FromIdentityEd25519 != null + && !masterKeyEd25519FromIdentityEd25519.equals( masterKeyEd25519)) { this.logger.warning("Mismatch between identity-ed25519 and " + "master-key-ed25519. Skipping."); @@ -829,9 +837,9 @@ public class SanitizedBridgesWriter extends Thread { /* Skip all crypto parts that might leak the bridge's identity * fingerprint. */ } else if (line.startsWith("-----BEGIN ") - || line.equals("onion-key") || line.equals("signing-key") || - line.equals("onion-key-crosscert") || - line.startsWith("ntor-onion-key-crosscert ")) { + || line.equals("onion-key") || line.equals("signing-key") + || line.equals("onion-key-crosscert") + || line.startsWith("ntor-onion-key-crosscert ")) { skipCrypto = true; /* Stop skipping lines when the crypto parts are over. */ @@ -893,8 +901,8 @@ public class SanitizedBridgesWriter extends Thread { byte[] forDigest = new byte[sig - start]; System.arraycopy(data, start, forDigest, 0, sig - start); descriptorDigestSha256Base64 = Base64.encodeBase64String( - DigestUtils.sha256(DigestUtils.sha256(forDigest))). - replaceAll("=", ""); + DigestUtils.sha256(DigestUtils.sha256(forDigest))) + .replaceAll("=", ""); } } catch (UnsupportedEncodingException e) { /* Handle below. */ @@ -1010,14 +1018,16 @@ public class SanitizedBridgesWriter extends Thread { public void sanitizeAndStoreExtraInfoDescriptor(byte[] data) { /* Parse descriptor to generate a sanitized version. */ - String scrubbedDesc = null, published = null, - masterKeyEd25519FromIdentityEd25519 = null; + String scrubbedDesc = null; + String published = null; + String masterKeyEd25519FromIdentityEd25519 = null; try { BufferedReader br = new BufferedReader(new StringReader(new String( data, "US-ASCII"))); String line = null; StringBuilder scrubbed = null; - String hashedBridgeIdentity = null, masterKeyEd25519 = null; + String hashedBridgeIdentity = null; + String masterKeyEd25519 = null; while ((line = br.readLine()) != null) { /* Parse bridge identity from extra-info line and replace it with @@ -1054,8 +1064,8 @@ public class SanitizedBridgesWriter extends Thread { /* Extract master-key-ed25519 from identity-ed25519. */ } else if (line.equals("identity-ed25519")) { StringBuilder sb = new StringBuilder(); - while ((line = br.readLine()) != null && - !line.equals("-----END ED25519 CERT-----")) { + while ((line = br.readLine()) != null + && !line.equals("-----END ED25519 CERT-----")) { if (line.equals("-----BEGIN ED25519 CERT-----")) { continue; } @@ -1066,8 +1076,8 @@ public class SanitizedBridgesWriter extends Thread { sb.toString()); String sha256MasterKeyEd25519 = Base64.encodeBase64String( DigestUtils.sha256(Base64.decodeBase64( - masterKeyEd25519FromIdentityEd25519 + "="))). - replaceAll("=", ""); + masterKeyEd25519FromIdentityEd25519 + "="))) + .replaceAll("=", ""); scrubbed.append("master-key-ed25519 " + sha256MasterKeyEd25519 + "\n"); if (masterKeyEd25519 != null && !masterKeyEd25519.equals( @@ -1080,8 +1090,8 @@ public class SanitizedBridgesWriter extends Thread { /* Verify that identity-ed25519 and master-key-ed25519 match. */ } else if (line.startsWith("master-key-ed25519 ")) { masterKeyEd25519 = line.substring(line.indexOf(" ") + 1); - if (masterKeyEd25519FromIdentityEd25519 != null && - !masterKeyEd25519FromIdentityEd25519.equals( + if (masterKeyEd25519FromIdentityEd25519 != null + && !masterKeyEd25519FromIdentityEd25519.equals( masterKeyEd25519)) { this.logger.warning("Mismatch between identity-ed25519 and " + "master-key-ed25519. Skipping."); @@ -1169,8 +1179,8 @@ public class SanitizedBridgesWriter extends Thread { byte[] forDigest = new byte[sig - start]; System.arraycopy(data, start, forDigest, 0, sig - start); descriptorDigestSha256Base64 = Base64.encodeBase64String( - DigestUtils.sha256(DigestUtils.sha256(forDigest))). - replaceAll("=", ""); + DigestUtils.sha256(DigestUtils.sha256(forDigest))) + .replaceAll("=", ""); } } catch (UnsupportedEncodingException e) { /* Handle below. */ @@ -1230,11 +1240,12 @@ public class SanitizedBridgesWriter extends Thread { public void finishWriting() { /* Delete secrets that we don't need anymore. */ - if (!this.secretsForHashingIPAddresses.isEmpty() && - this.secretsForHashingIPAddresses.firstKey().compareTo( + if (!this.secretsForHashingIPAddresses.isEmpty() + && this.secretsForHashingIPAddresses.firstKey().compareTo( this.bridgeSanitizingCutOffTimestamp) < 0) { try { - int kept = 0, deleted = 0; + int kept = 0; + int deleted = 0; BufferedWriter bw = new BufferedWriter(new FileWriter( this.bridgeIpSecretsFile)); for (Map.Entry<String, byte[]> e : @@ -1267,26 +1278,26 @@ public class SanitizedBridgesWriter extends Thread { try { long maxNetworkStatusPublishedMillis = dateTimeFormat.parse(maxNetworkStatusPublishedTime).getTime(); - if (maxNetworkStatusPublishedMillis > 0L && - maxNetworkStatusPublishedMillis < tooOldMillis) { + if (maxNetworkStatusPublishedMillis > 0L + && maxNetworkStatusPublishedMillis < tooOldMillis) { this.logger.warning("The last known bridge network status was " + "published " + maxNetworkStatusPublishedTime + ", which is " + "more than 5:30 hours in the past."); } long maxServerDescriptorPublishedMillis = - dateTimeFormat.parse(maxServerDescriptorPublishedTime). - getTime(); - if (maxServerDescriptorPublishedMillis > 0L && - maxServerDescriptorPublishedMillis < tooOldMillis) { + dateTimeFormat.parse(maxServerDescriptorPublishedTime) + .getTime(); + if (maxServerDescriptorPublishedMillis > 0L + && maxServerDescriptorPublishedMillis < tooOldMillis) { this.logger.warning("The last known bridge server descriptor was " + "published " + maxServerDescriptorPublishedTime + ", which " + "is more than 5:30 hours in the past."); } long maxExtraInfoDescriptorPublishedMillis = - dateTimeFormat.parse(maxExtraInfoDescriptorPublishedTime). - getTime(); - if (maxExtraInfoDescriptorPublishedMillis > 0L && - maxExtraInfoDescriptorPublishedMillis < tooOldMillis) { + dateTimeFormat.parse(maxExtraInfoDescriptorPublishedTime) + .getTime(); + if (maxExtraInfoDescriptorPublishedMillis > 0L + && maxExtraInfoDescriptorPublishedMillis < tooOldMillis) { this.logger.warning("The last known bridge extra-info descriptor " + "was published " + maxExtraInfoDescriptorPublishedTime + ", which is more than 5:30 hours in the past."); diff --git a/src/org/torproject/collector/bridgepools/BridgePoolAssignmentsProcessor.java b/src/org/torproject/collector/bridgepools/BridgePoolAssignmentsProcessor.java index b682316..8afaa43 100644 --- a/src/org/torproject/collector/bridgepools/BridgePoolAssignmentsProcessor.java +++ b/src/org/torproject/collector/bridgepools/BridgePoolAssignmentsProcessor.java @@ -1,7 +1,17 @@ /* Copyright 2011--2016 The Tor Project * See LICENSE for licensing information */ + package org.torproject.collector.bridgepools; +import org.torproject.collector.main.Configuration; +import org.torproject.collector.main.LockFile; +import org.torproject.collector.main.LoggingConfiguration; + +import org.apache.commons.codec.DecoderException; +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; + import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; @@ -24,14 +34,6 @@ import java.util.TreeSet; import java.util.logging.Level; import java.util.logging.Logger; -import org.apache.commons.codec.DecoderException; -import org.apache.commons.codec.binary.Hex; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; -import org.torproject.collector.main.Configuration; -import org.torproject.collector.main.LockFile; -import org.torproject.collector.main.LoggingConfiguration; - public class BridgePoolAssignmentsProcessor extends Thread { public static void main(String[] args) { @@ -77,8 +79,8 @@ public class BridgePoolAssignmentsProcessor extends Thread { Logger logger = Logger.getLogger(BridgePoolAssignmentsProcessor.class.getName()); - if (assignmentsDirectory == null || - sanitizedAssignmentsDirectory == null) { + if (assignmentsDirectory == null + || sanitizedAssignmentsDirectory == null) { IllegalArgumentException e = new IllegalArgumentException("Neither " + "assignmentsDirectory nor sanitizedAssignmentsDirectory may " + "be null!"); @@ -117,9 +119,11 @@ public class BridgePoolAssignmentsProcessor extends Thread { } else { br = new BufferedReader(new FileReader(assignmentFile)); } - String line, bridgePoolAssignmentLine = null; + String line; + String bridgePoolAssignmentLine = null; SortedSet<String> sanitizedAssignments = new TreeSet<String>(); - boolean wroteLastLine = false, skipBefore20120504125947 = true; + boolean wroteLastLine = false; + boolean skipBefore20120504125947 = true; Set<String> hashedFingerprints = null; while ((line = br.readLine()) != null || !wroteLastLine) { if (line != null && line.startsWith("bridge-pool-assignment ")) { @@ -142,8 +146,8 @@ public class BridgePoolAssignmentsProcessor extends Thread { continue; } } - if (line == null || - line.startsWith("bridge-pool-assignment ")) { + if (line == null + || line.startsWith("bridge-pool-assignment ")) { if (bridgePoolAssignmentLine != null) { try { long bridgePoolAssignmentTime = assignmentFormat.parse( @@ -235,8 +239,8 @@ public class BridgePoolAssignmentsProcessor extends Thread { + "starting with '" + duplicateFingerprint + "'."); } - if (maxBridgePoolAssignmentTime > 0L && - maxBridgePoolAssignmentTime + 330L * 60L * 1000L + if (maxBridgePoolAssignmentTime > 0L + && maxBridgePoolAssignmentTime + 330L * 60L * 1000L < System.currentTimeMillis()) { SimpleDateFormat dateTimeFormat = new SimpleDateFormat( "yyyy-MM-dd HH:mm:ss"); diff --git a/src/org/torproject/collector/exitlists/ExitListDownloader.java b/src/org/torproject/collector/exitlists/ExitListDownloader.java index 7e019b0..d328850 100644 --- a/src/org/torproject/collector/exitlists/ExitListDownloader.java +++ b/src/org/torproject/collector/exitlists/ExitListDownloader.java @@ -1,7 +1,17 @@ /* Copyright 2010--2016 The Tor Project * See LICENSE for licensing information */ + package org.torproject.collector.exitlists; +import org.torproject.collector.main.Configuration; +import org.torproject.collector.main.LockFile; +import org.torproject.collector.main.LoggingConfiguration; +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.DescriptorParseException; +import org.torproject.descriptor.DescriptorParser; +import org.torproject.descriptor.DescriptorSourceFactory; +import org.torproject.descriptor.ExitList; + import java.io.BufferedInputStream; import java.io.BufferedWriter; import java.io.File; @@ -20,15 +30,6 @@ import java.util.TreeSet; import java.util.logging.Level; import java.util.logging.Logger; -import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.DescriptorParser; -import org.torproject.descriptor.DescriptorSourceFactory; -import org.torproject.descriptor.ExitList; -import org.torproject.descriptor.DescriptorParseException; -import org.torproject.collector.main.Configuration; -import org.torproject.collector.main.LockFile; -import org.torproject.collector.main.LoggingConfiguration; - public class ExitListDownloader extends Thread { public static void main(String[] args) { @@ -85,8 +86,8 @@ public class ExitListDownloader extends Thread { huc.connect(); int response = huc.getResponseCode(); if (response != 200) { - logger.warning("Could not download exit list. Response code " + - response); + logger.warning("Could not download exit list. Response code " + + response); return; } BufferedInputStream in = new BufferedInputStream( @@ -121,8 +122,8 @@ public class ExitListDownloader extends Thread { List<Descriptor> parsedDescriptors = descriptorParser.parseDescriptors(downloadedExitList.getBytes(), tarballFile.getName()); - if (parsedDescriptors.size() != 1 || - !(parsedDescriptors.get(0) instanceof ExitList)) { + if (parsedDescriptors.size() != 1 + || !(parsedDescriptors.get(0) instanceof ExitList)) { logger.warning("Could not parse downloaded exit list"); return; } @@ -136,12 +137,12 @@ public class ExitListDownloader extends Thread { logger.log(Level.WARNING, "Could not parse downloaded exit list", e); } - if (maxScanMillis > 0L && - maxScanMillis + 330L * 60L * 1000L < System.currentTimeMillis()) { + if (maxScanMillis > 0L + && maxScanMillis + 330L * 60L * 1000L < System.currentTimeMillis()) { logger.warning("The last reported scan in the downloaded exit list " + "took place at " + dateTimeFormat.format(maxScanMillis) + ", which is more than 5:30 hours in the past."); - } + } /* Write to disk. */ File rsyncFile = new File("recent/exit-lists/" diff --git a/src/org/torproject/collector/index/CreateIndexJson.java b/src/org/torproject/collector/index/CreateIndexJson.java index cf2e0bb..ac5adf5 100644 --- a/src/org/torproject/collector/index/CreateIndexJson.java +++ b/src/org/torproject/collector/index/CreateIndexJson.java @@ -1,7 +1,14 @@ -/* Copyright 2015 The Tor Project +/* Copyright 2015--2016 The Tor Project * See LICENSE for licensing information */ + package org.torproject.collector.index; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; +import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream; + import java.io.BufferedWriter; import java.io.File; import java.io.FileOutputStream; @@ -17,12 +24,6 @@ import java.util.TimeZone; import java.util.TreeSet; import java.util.zip.GZIPOutputStream; -import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; -import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; - /* Create a fresh index.json containing all directories and files in the * archive/ and recent/ directories. * @@ -53,12 +54,14 @@ public class CreateIndexJson { String path; SortedSet<FileNode> files; SortedSet<DirectoryNode> directories; + DirectoryNode(String path, SortedSet<FileNode> files, SortedSet<DirectoryNode> directories) { this.path = path; this.files = files; this.directories = directories; } + public int compareTo(DirectoryNode o) { return this.path.compareTo(o.path); } @@ -69,6 +72,7 @@ public class CreateIndexJson { String path; SortedSet<FileNode> files; SortedSet<DirectoryNode> directories; + IndexNode(String index_created, String path, SortedSet<FileNode> files, SortedSet<DirectoryNode> directories) { @@ -83,17 +87,20 @@ public class CreateIndexJson { String path; long size; String last_modified; + FileNode(String path, long size, String last_modified) { this.path = path; this.size = size; this.last_modified = last_modified; } + public int compareTo(FileNode o) { return this.path.compareTo(o.path); } } static DateFormat dateTimeFormat; + static { dateTimeFormat = new SimpleDateFormat(dateTimePattern, dateTimeLocale); diff --git a/src/org/torproject/collector/main/Configuration.java b/src/org/torproject/collector/main/Configuration.java index eaf3217..9a511d3 100644 --- a/src/org/torproject/collector/main/Configuration.java +++ b/src/org/torproject/collector/main/Configuration.java @@ -1,5 +1,6 @@ -/* Copyright 2010--2012 The Tor Project +/* Copyright 2010--2016 The Tor Project * See LICENSE for licensing information */ + package org.torproject.collector.main; import java.io.BufferedReader; @@ -67,6 +68,7 @@ public class Configuration { private String torperfOutputDirectory = "out/torperf/"; private SortedMap<String, String> torperfSources = null; private List<String> torperfFiles = null; + public Configuration() { /* Initialize logger. */ @@ -219,84 +221,111 @@ public class Configuration { System.exit(1); } } + public String getDirectoryArchivesOutputDirectory() { return this.directoryArchivesOutputDirectory; } + public boolean getImportCachedRelayDescriptors() { return this.importCachedRelayDescriptors; } + public List<String> getCachedRelayDescriptorDirectory() { return this.cachedRelayDescriptorsDirectory; } + public boolean getImportDirectoryArchives() { return this.importDirectoryArchives; } + public String getDirectoryArchivesDirectory() { return this.directoryArchivesDirectory; } + public boolean getKeepDirectoryArchiveImportHistory() { return this.keepDirectoryArchiveImportHistory; } + public boolean getReplaceIPAddressesWithHashes() { return this.replaceIPAddressesWithHashes; } + public long getLimitBridgeDescriptorMappings() { return this.limitBridgeDescriptorMappings; } + public String getSanitizedBridgesWriteDirectory() { return this.sanitizedBridgesWriteDirectory; } + public String getBridgeSnapshotsDirectory() { return this.bridgeSnapshotsDirectory; } + public boolean getDownloadRelayDescriptors() { return this.downloadRelayDescriptors; } + public List<String> getDownloadFromDirectoryAuthorities() { return this.downloadFromDirectoryAuthorities; } + public List<String> getDownloadVotesByFingerprint() { return this.downloadVotesByFingerprint; } + public boolean getDownloadCurrentConsensus() { return this.downloadCurrentConsensus; } + public boolean getDownloadCurrentMicrodescConsensus() { return this.downloadCurrentMicrodescConsensus; } + public boolean getDownloadCurrentVotes() { return this.downloadCurrentVotes; } + public boolean getDownloadMissingServerDescriptors() { return this.downloadMissingServerDescriptors; } + public boolean getDownloadMissingExtraInfoDescriptors() { return this.downloadMissingExtraInfoDescriptors; } + public boolean getDownloadMissingMicrodescriptors() { return this.downloadMissingMicrodescriptors; } + public boolean getDownloadAllServerDescriptors() { return this.downloadAllServerDescriptors; } + public boolean getDownloadAllExtraInfoDescriptors() { return this.downloadAllExtraInfoDescriptors; } + public boolean getCompressRelayDescriptorDownloads() { return this.compressRelayDescriptorDownloads; } + public String getAssignmentsDirectory() { return this.assignmentsDirectory; } + public String getSanitizedAssignmentsDirectory() { return this.sanitizedAssignmentsDirectory; } + public String getTorperfOutputDirectory() { return this.torperfOutputDirectory; } + public SortedMap<String, String> getTorperfSources() { return this.torperfSources; } + public List<String> getTorperfFiles() { return this.torperfFiles; } diff --git a/src/org/torproject/collector/main/LockFile.java b/src/org/torproject/collector/main/LockFile.java index cd850bd..b07d4b1 100644 --- a/src/org/torproject/collector/main/LockFile.java +++ b/src/org/torproject/collector/main/LockFile.java @@ -1,5 +1,6 @@ -/* Copyright 2010--2012 The Tor Project +/* Copyright 2010--2016 The Tor Project * See LICENSE for licensing information */ + package org.torproject.collector.main; import java.io.BufferedReader; diff --git a/src/org/torproject/collector/main/LoggingConfiguration.java b/src/org/torproject/collector/main/LoggingConfiguration.java index f3a620c..b8a91c7 100644 --- a/src/org/torproject/collector/main/LoggingConfiguration.java +++ b/src/org/torproject/collector/main/LoggingConfiguration.java @@ -1,5 +1,6 @@ /* Copyright 2010--2016 The Tor Project * See LICENSE for licensing information */ + package org.torproject.collector.main; import java.io.File; diff --git a/src/org/torproject/collector/relaydescs/ArchiveReader.java b/src/org/torproject/collector/relaydescs/ArchiveReader.java index 93bea7f..96409f4 100644 --- a/src/org/torproject/collector/relaydescs/ArchiveReader.java +++ b/src/org/torproject/collector/relaydescs/ArchiveReader.java @@ -1,7 +1,12 @@ -/* Copyright 2010--2014 The Tor Project +/* Copyright 2010--2016 The Tor Project * See LICENSE for licensing information */ + package org.torproject.collector.relaydescs; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; + import java.io.BufferedInputStream; import java.io.BufferedReader; import java.io.BufferedWriter; @@ -28,10 +33,6 @@ import java.util.TreeSet; import java.util.logging.Level; import java.util.logging.Logger; -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; - /** * Read in all files in a given directory and pass buffered readers of * them to the relay descriptor parser. @@ -40,13 +41,14 @@ public class ArchiveReader { public ArchiveReader(RelayDescriptorParser rdp, File archivesDirectory, File statsDirectory, boolean keepImportHistory) { - if (rdp == null || archivesDirectory == null || - statsDirectory == null) { + if (rdp == null || archivesDirectory == null + || statsDirectory == null) { throw new IllegalArgumentException(); } rdp.setArchiveReader(this); - int parsedFiles = 0, ignoredFiles = 0; + int parsedFiles = 0; + int ignoredFiles = 0; Logger logger = Logger.getLogger(ArchiveReader.class.getName()); SortedSet<String> archivesImportHistory = new TreeSet<String>(); File archivesImportHistoryFile = new File(statsDirectory, @@ -82,8 +84,8 @@ public class ArchiveReader { if (rdp != null) { try { BufferedInputStream bis = null; - if (keepImportHistory && - archivesImportHistory.contains(pop.getName())) { + if (keepImportHistory + && archivesImportHistory.contains(pop.getName())) { ignoredFiles++; continue; } else if (pop.getName().endsWith(".tar.bz2")) { @@ -176,7 +178,8 @@ public class ArchiveReader { } catch (UnsupportedEncodingException e) { /* No way that US-ASCII is not supported. */ } - int start = -1, end = -1; + int start = -1; + int end = -1; String startToken = "onion-key\n"; while (end < ascii.length()) { start = ascii.indexOf(startToken, end); @@ -198,8 +201,8 @@ public class ArchiveReader { if (!this.microdescriptorValidAfterTimes.containsKey( digest256Hex)) { logger.fine("Could not store microdescriptor '" - + digest256Hex + "', which was not contained in a " - + "microdesc consensus."); + + digest256Hex + "', which was not contained in a " + + "microdesc consensus."); continue; } for (String validAfterTime : @@ -265,6 +268,7 @@ public class ArchiveReader { private Map<String, Set<String>> microdescriptorValidAfterTimes = new HashMap<String, Set<String>>(); + public void haveParsedMicrodescConsensus(String validAfterTime, SortedSet<String> microdescriptorDigests) { for (String microdescriptor : microdescriptorDigests) { diff --git a/src/org/torproject/collector/relaydescs/ArchiveWriter.java b/src/org/torproject/collector/relaydescs/ArchiveWriter.java index a4bddff..370b5ea 100644 --- a/src/org/torproject/collector/relaydescs/ArchiveWriter.java +++ b/src/org/torproject/collector/relaydescs/ArchiveWriter.java @@ -1,7 +1,15 @@ /* Copyright 2010--2016 The Tor Project * See LICENSE for licensing information */ + package org.torproject.collector.relaydescs; +import org.torproject.collector.main.Configuration; +import org.torproject.collector.main.LockFile; +import org.torproject.collector.main.LoggingConfiguration; +import org.torproject.descriptor.DescriptorParseException; +import org.torproject.descriptor.DescriptorParser; +import org.torproject.descriptor.DescriptorSourceFactory; + import java.io.BufferedOutputStream; import java.io.BufferedReader; import java.io.BufferedWriter; @@ -27,13 +35,6 @@ import java.util.TreeMap; import java.util.logging.Level; import java.util.logging.Logger; -import org.torproject.descriptor.DescriptorParser; -import org.torproject.descriptor.DescriptorSourceFactory; -import org.torproject.descriptor.DescriptorParseException; -import org.torproject.collector.main.Configuration; -import org.torproject.collector.main.LockFile; -import org.torproject.collector.main.LoggingConfiguration; - public class ArchiveWriter extends Thread { public static void main(String[] args) { @@ -78,11 +79,13 @@ public class ArchiveWriter extends Thread { private File outputDirectory; private String rsyncCatString; private DescriptorParser descriptorParser; - private int storedConsensusesCounter = 0, - storedMicrodescConsensusesCounter = 0, storedVotesCounter = 0, - storedCertsCounter = 0, storedServerDescriptorsCounter = 0, - storedExtraInfoDescriptorsCounter = 0, - storedMicrodescriptorsCounter = 0; + private int storedConsensusesCounter = 0; + private int storedMicrodescConsensusesCounter = 0; + private int storedVotesCounter = 0; + private int storedCertsCounter = 0; + private int storedServerDescriptorsCounter = 0; + private int storedExtraInfoDescriptorsCounter = 0; + private int storedMicrodescriptorsCounter = 0; private SortedMap<Long, SortedSet<String>> storedConsensuses = new TreeMap<Long, SortedSet<String>>(); @@ -361,6 +364,7 @@ public class ArchiveWriter extends Thread { private static final byte[] CONSENSUS_ANNOTATION = "@type network-status-consensus-3 1.0\n".getBytes(); + public void storeConsensus(byte[] data, long validAfter, SortedSet<String> dirSources, SortedSet<String> serverDescriptorDigests) { @@ -376,8 +380,8 @@ public class ArchiveWriter extends Thread { if (this.store(CONSENSUS_ANNOTATION, data, outputFiles, null)) { this.storedConsensusesCounter++; } - if (!tarballFileExistedBefore && - this.now - validAfter < 3L * 60L * 60L * 1000L) { + if (!tarballFileExistedBefore + && this.now - validAfter < 3L * 60L * 60L * 1000L) { this.storedConsensuses.put(validAfter, serverDescriptorDigests); this.expectedVotes.put(validAfter, dirSources.size()); } @@ -385,6 +389,7 @@ public class ArchiveWriter extends Thread { private static final byte[] MICRODESCCONSENSUS_ANNOTATION = "@type network-status-microdesc-consensus-3 1.0\n".getBytes(); + public void storeMicrodescConsensus(byte[] data, long validAfter, SortedSet<String> microdescriptorDigests) { SimpleDateFormat yearMonthDirectoryFormat = new SimpleDateFormat( @@ -406,8 +411,8 @@ public class ArchiveWriter extends Thread { null)) { this.storedMicrodescConsensusesCounter++; } - if (!tarballFileExistedBefore && - this.now - validAfter < 3L * 60L * 60L * 1000L) { + if (!tarballFileExistedBefore + && this.now - validAfter < 3L * 60L * 60L * 1000L) { this.storedMicrodescConsensuses.put(validAfter, microdescriptorDigests); } @@ -415,6 +420,7 @@ public class ArchiveWriter extends Thread { private static final byte[] VOTE_ANNOTATION = "@type network-status-vote-3 1.0\n".getBytes(); + public void storeVote(byte[] data, long validAfter, String fingerprint, String digest, SortedSet<String> serverDescriptorDigests) { @@ -431,8 +437,8 @@ public class ArchiveWriter extends Thread { if (this.store(VOTE_ANNOTATION, data, outputFiles, null)) { this.storedVotesCounter++; } - if (!tarballFileExistedBefore && - this.now - validAfter < 3L * 60L * 60L * 1000L) { + if (!tarballFileExistedBefore + && this.now - validAfter < 3L * 60L * 60L * 1000L) { if (!this.storedVotes.containsKey(validAfter)) { this.storedVotes.put(validAfter, new TreeMap<String, SortedSet<String>>()); @@ -444,6 +450,7 @@ public class ArchiveWriter extends Thread { private static final byte[] CERTIFICATE_ANNOTATION = "@type dir-key-certificate-3 1.0\n".getBytes(); + public void storeCertificate(byte[] data, String fingerprint, long published) { SimpleDateFormat printFormat = new SimpleDateFormat( @@ -459,6 +466,7 @@ public class ArchiveWriter extends Thread { private static final byte[] SERVER_DESCRIPTOR_ANNOTATION = "@type server-descriptor 1.0\n".getBytes(); + public void storeServerDescriptor(byte[] data, String digest, long published, String extraInfoDigest) { SimpleDateFormat printFormat = new SimpleDateFormat("yyyy/MM/"); @@ -477,8 +485,8 @@ public class ArchiveWriter extends Thread { append)) { this.storedServerDescriptorsCounter++; } - if (!tarballFileExistedBefore && - this.now - published < 48L * 60L * 60L * 1000L) { + if (!tarballFileExistedBefore + && this.now - published < 48L * 60L * 60L * 1000L) { if (!this.storedServerDescriptors.containsKey(published)) { this.storedServerDescriptors.put(published, new HashMap<String, String>()); @@ -490,6 +498,7 @@ public class ArchiveWriter extends Thread { private static final byte[] EXTRA_INFO_ANNOTATION = "@type extra-info 1.0\n".getBytes(); + public void storeExtraInfoDescriptor(byte[] data, String extraInfoDigest, long published) { SimpleDateFormat descriptorFormat = new SimpleDateFormat("yyyy/MM/"); @@ -507,8 +516,8 @@ public class ArchiveWriter extends Thread { if (this.store(EXTRA_INFO_ANNOTATION, data, outputFiles, append)) { this.storedExtraInfoDescriptorsCounter++; } - if (!tarballFileExistedBefore && - this.now - published < 48L * 60L * 60L * 1000L) { + if (!tarballFileExistedBefore + && this.now - published < 48L * 60L * 60L * 1000L) { if (!this.storedExtraInfoDescriptors.containsKey(published)) { this.storedExtraInfoDescriptors.put(published, new HashSet<String>()); @@ -519,6 +528,7 @@ public class ArchiveWriter extends Thread { private static final byte[] MICRODESCRIPTOR_ANNOTATION = "@type microdescriptor 1.0\n".getBytes(); + public void storeMicrodescriptor(byte[] data, String microdescriptorDigest, long validAfter) { /* TODO We could check here whether we already stored the @@ -545,8 +555,8 @@ public class ArchiveWriter extends Thread { append)) { this.storedMicrodescriptorsCounter++; } - if (!tarballFileExistedBefore && - this.now - validAfter < 40L * 24L * 60L * 60L * 1000L) { + if (!tarballFileExistedBefore + && this.now - validAfter < 40L * 24L * 60L * 60L * 1000L) { if (!this.storedMicrodescriptors.containsKey(validAfter)) { this.storedMicrodescriptors.put(validAfter, new HashSet<String>()); @@ -557,6 +567,7 @@ public class ArchiveWriter extends Thread { } private StringBuilder intermediateStats = new StringBuilder(); + public void intermediateStats(String event) { intermediateStats.append("While " + event + ", we stored " + this.storedConsensusesCounter + " consensus(es), " @@ -600,8 +611,9 @@ public class ArchiveWriter extends Thread { for (Set<String> descriptors : this.storedMicrodescriptors.values()) { knownMicrodescriptors.addAll(descriptors); } - boolean missingDescriptors = false, missingVotes = false, - missingMicrodescConsensus = false; + boolean missingDescriptors = false; + boolean missingVotes = false; + boolean missingMicrodescConsensus = false; for (Map.Entry<Long, SortedSet<String>> c : this.storedConsensuses.entrySet()) { long validAfterMillis = c.getKey(); @@ -613,8 +625,10 @@ public class ArchiveWriter extends Thread { foundVotes = this.storedVotes.get(validAfterMillis).size(); for (Map.Entry<String, SortedSet<String>> v : this.storedVotes.get(validAfterMillis).entrySet()) { - int voteFoundServerDescs = 0, voteAllServerDescs = 0, - voteFoundExtraInfos = 0, voteAllExtraInfos = 0; + int voteFoundServerDescs = 0; + int voteAllServerDescs = 0; + int voteFoundExtraInfos = 0; + int voteAllExtraInfos = 0; for (String serverDescriptorDigest : v.getValue()) { voteAllServerDescs++; if (knownServerDescriptors.containsKey( @@ -636,32 +650,35 @@ public class ArchiveWriter extends Thread { if (voteAllServerDescs > 0) { sb.append(String.format(", %d/%d S (%.1f%%)", voteFoundServerDescs, voteAllServerDescs, - 100.0D * (double) voteFoundServerDescs / - (double) voteAllServerDescs)); + 100.0D * (double) voteFoundServerDescs + / (double) voteAllServerDescs)); } else { sb.append(", 0/0 S"); } if (voteAllExtraInfos > 0) { sb.append(String.format(", %d/%d E (%.1f%%)", voteFoundExtraInfos, voteAllExtraInfos, - 100.0D * (double) voteFoundExtraInfos / - (double) voteAllExtraInfos)); + 100.0D * (double) voteFoundExtraInfos + / (double) voteAllExtraInfos)); } else { sb.append(", 0/0 E"); } String fingerprint = v.getKey(); /* Ignore turtles when warning about missing descriptors. */ if (!fingerprint.equalsIgnoreCase( - "27B6B5996C426270A5C95488AA5BCEB6BCC86956") && - (voteFoundServerDescs * 1000 < voteAllServerDescs * 995 || - voteFoundExtraInfos * 1000 < voteAllExtraInfos * 995)) { + "27B6B5996C426270A5C95488AA5BCEB6BCC86956") + && (voteFoundServerDescs * 1000 < voteAllServerDescs * 995 + || voteFoundExtraInfos * 1000 < voteAllExtraInfos * 995)) { missingDescriptors = true; } } } - int foundServerDescs = 0, allServerDescs = 0, foundExtraInfos = 0, - allExtraInfos = 0, foundMicrodescriptors = 0, - allMicrodescriptors = 0; + int foundServerDescs = 0; + int allServerDescs = 0; + int foundExtraInfos = 0; + int allExtraInfos = 0; + int foundMicrodescriptors = 0; + int allMicrodescriptors = 0; for (String serverDescriptorDigest : c.getValue()) { allServerDescs++; if (knownServerDescriptors.containsKey( @@ -688,15 +705,15 @@ public class ArchiveWriter extends Thread { } if (allServerDescs > 0) { sb.append(String.format(", %d/%d S (%.1f%%)", foundServerDescs, - allServerDescs, 100.0D * (double) foundServerDescs / - (double) allServerDescs)); + allServerDescs, 100.0D * (double) foundServerDescs + / (double) allServerDescs)); } else { sb.append(", 0/0 S"); } if (allExtraInfos > 0) { sb.append(String.format(", %d/%d E (%.1f%%)", foundExtraInfos, - allExtraInfos, 100.0D * (double) foundExtraInfos / - (double) allExtraInfos)); + allExtraInfos, 100.0D * (double) foundExtraInfos + / (double) allExtraInfos)); } else { sb.append(", 0/0 E"); } @@ -712,17 +729,17 @@ public class ArchiveWriter extends Thread { if (allMicrodescriptors > 0) { sb.append(String.format(", %d/%d M (%.1f%%)", foundMicrodescriptors, allMicrodescriptors, - 100.0D * (double) foundMicrodescriptors / - (double) allMicrodescriptors)); + 100.0D * (double) foundMicrodescriptors + / (double) allMicrodescriptors)); } else { sb.append(", 0/0 M"); } } else { missingMicrodescConsensus = true; } - if (foundServerDescs * 1000 < allServerDescs * 995 || - foundExtraInfos * 1000 < allExtraInfos * 995 || - foundMicrodescriptors * 1000 < allMicrodescriptors * 995) { + if (foundServerDescs * 1000 < allServerDescs * 995 + || foundExtraInfos * 1000 < allExtraInfos * 995 + || foundMicrodescriptors * 1000 < allMicrodescriptors * 995) { missingDescriptors = true; } if (foundVotes < allVotes) { @@ -756,44 +773,44 @@ public class ArchiveWriter extends Thread { "yyyy-MM-dd HH:mm:ss"); dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); long tooOldMillis = this.now - 330L * 60L * 1000L; - if (!this.storedConsensuses.isEmpty() && - this.storedConsensuses.lastKey() < tooOldMillis) { + if (!this.storedConsensuses.isEmpty() + && this.storedConsensuses.lastKey() < tooOldMillis) { this.logger.warning("The last known relay network status " + "consensus was valid after " + dateTimeFormat.format(this.storedConsensuses.lastKey()) + ", which is more than 5:30 hours in the past."); } - if (!this.storedMicrodescConsensuses.isEmpty() && - this.storedMicrodescConsensuses.lastKey() < tooOldMillis) { + if (!this.storedMicrodescConsensuses.isEmpty() + && this.storedMicrodescConsensuses.lastKey() < tooOldMillis) { this.logger.warning("The last known relay network status " + "microdesc consensus was valid after " + dateTimeFormat.format( this.storedMicrodescConsensuses.lastKey()) + ", which is more than 5:30 hours in the past."); } - if (!this.storedVotes.isEmpty() && - this.storedVotes.lastKey() < tooOldMillis) { + if (!this.storedVotes.isEmpty() + && this.storedVotes.lastKey() < tooOldMillis) { this.logger.warning("The last known relay network status vote " + "was valid after " + dateTimeFormat.format( this.storedVotes.lastKey()) + ", which is more than 5:30 hours " + "in the past."); } - if (!this.storedServerDescriptors.isEmpty() && - this.storedServerDescriptors.lastKey() < tooOldMillis) { + if (!this.storedServerDescriptors.isEmpty() + && this.storedServerDescriptors.lastKey() < tooOldMillis) { this.logger.warning("The last known relay server descriptor was " + "published at " + dateTimeFormat.format(this.storedServerDescriptors.lastKey()) + ", which is more than 5:30 hours in the past."); } - if (!this.storedExtraInfoDescriptors.isEmpty() && - this.storedExtraInfoDescriptors.lastKey() < tooOldMillis) { + if (!this.storedExtraInfoDescriptors.isEmpty() + && this.storedExtraInfoDescriptors.lastKey() < tooOldMillis) { this.logger.warning("The last known relay extra-info descriptor " + "was published at " + dateTimeFormat.format( this.storedExtraInfoDescriptors.lastKey()) + ", which is more than 5:30 hours in the past."); } - if (!this.storedMicrodescriptors.isEmpty() && - this.storedMicrodescriptors.lastKey() < tooOldMillis) { + if (!this.storedMicrodescriptors.isEmpty() + && this.storedMicrodescriptors.lastKey() < tooOldMillis) { this.logger.warning("The last known relay microdescriptor was " + "contained in a microdesc consensus that was valid after " + dateTimeFormat.format(this.storedMicrodescriptors.lastKey()) diff --git a/src/org/torproject/collector/relaydescs/CachedRelayDescriptorReader.java b/src/org/torproject/collector/relaydescs/CachedRelayDescriptorReader.java index de0c060..b9001dd 100644 --- a/src/org/torproject/collector/relaydescs/CachedRelayDescriptorReader.java +++ b/src/org/torproject/collector/relaydescs/CachedRelayDescriptorReader.java @@ -1,7 +1,11 @@ -/* Copyright 2010--2012 The Tor Project +/* Copyright 2010--2016 The Tor Project * See LICENSE for licensing information */ + package org.torproject.collector.relaydescs; +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.codec.digest.DigestUtils; + import java.io.BufferedInputStream; import java.io.BufferedReader; import java.io.BufferedWriter; @@ -25,9 +29,6 @@ import java.util.TreeSet; import java.util.logging.Level; import java.util.logging.Logger; -import org.apache.commons.codec.binary.Hex; -import org.apache.commons.codec.digest.DigestUtils; - /** * Parses all descriptors in local directory cacheddesc/ and sorts them * into directory structure in directory-archive/. @@ -36,8 +37,8 @@ public class CachedRelayDescriptorReader { public CachedRelayDescriptorReader(RelayDescriptorParser rdp, List<String> inputDirectories, File statsDirectory) { - if (rdp == null || inputDirectories == null || - inputDirectories.isEmpty() || statsDirectory == null) { + if (rdp == null || inputDirectories == null + || inputDirectories.isEmpty() || statsDirectory == null) { throw new IllegalArgumentException(); } @@ -48,8 +49,8 @@ public class CachedRelayDescriptorReader { /* Read import history containing SHA-1 digests of previously parsed * statuses and descriptors, so that we can skip them in this run. */ - Set<String> lastImportHistory = new HashSet<String>(), - currentImportHistory = new HashSet<String>(); + Set<String> lastImportHistory = new HashSet<String>(); + Set<String> currentImportHistory = new HashSet<String>(); File importHistoryFile = new File(statsDirectory, "cacheddesc-import-history"); if (importHistoryFile.exists()) { @@ -114,8 +115,8 @@ public class CachedRelayDescriptorReader { SimpleDateFormat dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - if (dateTimeFormat.parse(line.substring("valid-after ". - length())).getTime() < System.currentTimeMillis() + if (dateTimeFormat.parse(line.substring("valid-after " + .length())).getTime() < System.currentTimeMillis() - 6L * 60L * 60L * 1000L) { logger.warning("Cached descriptor files in " + cachedDescDir.getAbsolutePath() + " are stale. " @@ -133,8 +134,8 @@ public class CachedRelayDescriptorReader { if (rdp != null) { String digest = Hex.encodeHexString(DigestUtils.sha( allData)); - if (!lastImportHistory.contains(digest) && - !currentImportHistory.contains(digest)) { + if (!lastImportHistory.contains(digest) + && !currentImportHistory.contains(digest)) { rdp.parse(allData); } else { dumpStats.append(" (skipped)"); @@ -142,7 +143,8 @@ public class CachedRelayDescriptorReader { currentImportHistory.add(digest); } } else if (f.getName().equals("v3-status-votes")) { - int parsedNum = 0, skippedNum = 0; + int parsedNum = 0; + int skippedNum = 0; String ascii = new String(allData, "US-ASCII"); String startToken = "network-status-version "; int end = ascii.length(); @@ -159,8 +161,8 @@ public class CachedRelayDescriptorReader { if (rdp != null) { String digest = Hex.encodeHexString(DigestUtils.sha( rawNetworkStatusBytes)); - if (!lastImportHistory.contains(digest) && - !currentImportHistory.contains(digest)) { + if (!lastImportHistory.contains(digest) + && !currentImportHistory.contains(digest)) { rdp.parse(rawNetworkStatusBytes); parsedNum++; } else { @@ -173,16 +175,19 @@ public class CachedRelayDescriptorReader { } dumpStats.append("\n" + f.getName() + ": parsed " + parsedNum + ", skipped " + skippedNum + " votes"); - } else if (f.getName().startsWith("cached-descriptors") || - f.getName().startsWith("cached-extrainfo")) { + } else if (f.getName().startsWith("cached-descriptors") + || f.getName().startsWith("cached-extrainfo")) { String ascii = new String(allData, "US-ASCII"); - int start = -1, sig = -1, end = -1; + int start = -1; + int sig = -1; + int end = -1; String startToken = - f.getName().startsWith("cached-descriptors") ? - "router " : "extra-info "; + f.getName().startsWith("cached-descriptors") + ? "router " : "extra-info "; String sigToken = "\nrouter-signature\n"; String endToken = "\n-----END SIGNATURE-----\n"; - int parsedNum = 0, skippedNum = 0; + int parsedNum = 0; + int skippedNum = 0; while (end < ascii.length()) { start = ascii.indexOf(startToken, end); if (start < 0) { @@ -203,8 +208,8 @@ public class CachedRelayDescriptorReader { if (rdp != null) { String digest = Hex.encodeHexString(DigestUtils.sha( descBytes)); - if (!lastImportHistory.contains(digest) && - !currentImportHistory.contains(digest)) { + if (!lastImportHistory.contains(digest) + && !currentImportHistory.contains(digest)) { rdp.parse(descBytes); parsedNum++; } else { @@ -215,8 +220,8 @@ public class CachedRelayDescriptorReader { } dumpStats.append("\n" + f.getName() + ": parsed " + parsedNum + ", skipped " + skippedNum + " " - + (f.getName().startsWith("cached-descriptors") ? - "server" : "extra-info") + " descriptors"); + + (f.getName().startsWith("cached-descriptors") + ? "server" : "extra-info") + " descriptors"); } } catch (IOException e) { logger.log(Level.WARNING, "Failed reading " diff --git a/src/org/torproject/collector/relaydescs/ReferenceChecker.java b/src/org/torproject/collector/relaydescs/ReferenceChecker.java index b9f36de..485fcea 100644 --- a/src/org/torproject/collector/relaydescs/ReferenceChecker.java +++ b/src/org/torproject/collector/relaydescs/ReferenceChecker.java @@ -1,5 +1,22 @@ +/* Copyright 2016 The Tor Project + * See LICENSE for licensing information */ + package org.torproject.collector.relaydescs; +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.DescriptorFile; +import org.torproject.descriptor.DescriptorReader; +import org.torproject.descriptor.DescriptorSourceFactory; +import org.torproject.descriptor.DirSourceEntry; +import org.torproject.descriptor.ExtraInfoDescriptor; +import org.torproject.descriptor.Microdescriptor; +import org.torproject.descriptor.NetworkStatusEntry; +import org.torproject.descriptor.RelayNetworkStatusConsensus; +import org.torproject.descriptor.RelayNetworkStatusVote; +import org.torproject.descriptor.ServerDescriptor; + +import com.google.gson.Gson; + import java.io.File; import java.io.FileReader; import java.io.FileWriter; @@ -17,20 +34,6 @@ import java.util.TreeSet; import java.util.logging.Level; import java.util.logging.Logger; -import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.DescriptorFile; -import org.torproject.descriptor.DescriptorReader; -import org.torproject.descriptor.DescriptorSourceFactory; -import org.torproject.descriptor.DirSourceEntry; -import org.torproject.descriptor.ExtraInfoDescriptor; -import org.torproject.descriptor.Microdescriptor; -import org.torproject.descriptor.NetworkStatusEntry; -import org.torproject.descriptor.RelayNetworkStatusConsensus; -import org.torproject.descriptor.RelayNetworkStatusVote; -import org.torproject.descriptor.ServerDescriptor; - -import com.google.gson.Gson; - public class ReferenceChecker { private Logger log = Logger.getLogger(ReferenceChecker.class.getName()); @@ -87,8 +90,8 @@ public class ReferenceChecker { return false; } Reference other = (Reference) otherObject; - return this.referencing.equals(other.referencing) && - this.referenced.equals(other.referenced); + return this.referencing.equals(other.referencing) + && this.referenced.equals(other.referenced); } @Override @@ -168,6 +171,7 @@ public class ReferenceChecker { } private static DateFormat dateTimeFormat; + static { dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US); @@ -175,9 +179,15 @@ public class ReferenceChecker { dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); } - private static final long ONE_HOUR = 60L * 60L * 1000L, - THREE_HOURS = 3L * ONE_HOUR, SIX_HOURS = 6L * ONE_HOUR, - ONE_DAY = 24L * ONE_HOUR, THIRTY_DAYS = 30L * ONE_DAY; + private static final long ONE_HOUR = 60L * 60L * 1000L; + + private static final long THREE_HOURS = 3L * ONE_HOUR; + + private static final long SIX_HOURS = 6L * ONE_HOUR; + + private static final long ONE_DAY = 24L * ONE_HOUR; + + private static final long THIRTY_DAYS = 30L * ONE_DAY; private void readRelayNetworkStatusConsensusUnflavored( RelayNetworkStatusConsensus consensus) { @@ -194,8 +204,8 @@ public class ReferenceChecker { consensus.getValidAfterMillis() + THREE_HOURS); } } - double entryWeight = 200.0 / - ((double) consensus.getStatusEntries().size()); + double entryWeight = 200.0 + / ((double) consensus.getStatusEntries().size()); for (NetworkStatusEntry entry : consensus.getStatusEntries().values()) { this.addReference(referencing, @@ -212,8 +222,8 @@ public class ReferenceChecker { String referencing = String.format("M-%s", validAfter); this.addReference(referencing, String.format("C-%s", validAfter), 1.0, consensus.getValidAfterMillis() + THREE_HOURS); - double entryWeight = 200.0 / - ((double) consensus.getStatusEntries().size()); + double entryWeight = 200.0 + / ((double) consensus.getStatusEntries().size()); for (NetworkStatusEntry entry : consensus.getStatusEntries().values()) { for (String digest : entry.getMicrodescriptorDigests()) { @@ -227,8 +237,8 @@ public class ReferenceChecker { String validAfter = dateTimeFormat.format(vote.getValidAfterMillis()); String referencing = String.format("V-%s-%s", validAfter, vote.getIdentity()); - double entryWeight = 200.0 / - ((double) vote.getStatusEntries().size()); + double entryWeight = 200.0 + / ((double) vote.getStatusEntries().size()); for (NetworkStatusEntry entry : vote.getStatusEntries().values()) { this.addReference(referencing, String.format("S-%s", entry.getDescriptor()), entryWeight, @@ -277,8 +287,8 @@ public class ReferenceChecker { StringBuilder sb = new StringBuilder("Missing referenced " + "descriptors:"); for (Reference reference : this.references) { - if (reference.referenced.length() > 0 && - !knownDescriptors.contains(reference.referenced)) { + if (reference.referenced.length() > 0 + && !knownDescriptors.contains(reference.referenced)) { if (!missingDescriptors.contains(reference.referenced)) { totalMissingDescriptorsWeight += reference.weight; } diff --git a/src/org/torproject/collector/relaydescs/RelayDescriptorDownloader.java b/src/org/torproject/collector/relaydescs/RelayDescriptorDownloader.java index 8cfbf10..458332a 100644 --- a/src/org/torproject/collector/relaydescs/RelayDescriptorDownloader.java +++ b/src/org/torproject/collector/relaydescs/RelayDescriptorDownloader.java @@ -1,7 +1,11 @@ -/* Copyright 2010--2014 The Tor Project +/* Copyright 2010--2016 The Tor Project * See LICENSE for licensing information */ + package org.torproject.collector.relaydescs; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.codec.digest.DigestUtils; + import java.io.BufferedInputStream; import java.io.BufferedReader; import java.io.BufferedWriter; @@ -31,9 +35,6 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.zip.InflaterInputStream; -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.codec.digest.DigestUtils; - /** * Downloads relay descriptors from the directory authorities via HTTP. * Keeps a list of missing descriptors that gets updated by parse results @@ -224,25 +225,61 @@ public class RelayDescriptorDownloader { * that we requested, and that we successfully downloaded in this * execution. */ - private int oldMissingConsensuses = 0, - oldMissingMicrodescConsensuses = 0, oldMissingVotes = 0, - oldMissingServerDescriptors = 0, oldMissingExtraInfoDescriptors = 0, - oldMissingMicrodescriptors = 0, newMissingConsensuses = 0, - newMissingMicrodescConsensuses = 0, newMissingVotes = 0, - newMissingServerDescriptors = 0, newMissingExtraInfoDescriptors = 0, - newMissingMicrodescriptors = 0, requestedConsensuses = 0, - requestedMicrodescConsensuses = 0, requestedVotes = 0, - requestedMissingServerDescriptors = 0, - requestedAllServerDescriptors = 0, - requestedMissingExtraInfoDescriptors = 0, - requestedAllExtraInfoDescriptors = 0, - requestedMissingMicrodescriptors = 0, downloadedConsensuses = 0, - downloadedMicrodescConsensuses = 0, downloadedVotes = 0, - downloadedMissingServerDescriptors = 0, - downloadedAllServerDescriptors = 0, - downloadedMissingExtraInfoDescriptors = 0, - downloadedAllExtraInfoDescriptors = 0, - downloadedMissingMicrodescriptors = 0; + private int oldMissingConsensuses = 0; + + private int oldMissingMicrodescConsensuses = 0; + + private int oldMissingVotes = 0; + + private int oldMissingServerDescriptors = 0; + + private int oldMissingExtraInfoDescriptors = 0; + + private int oldMissingMicrodescriptors = 0; + + private int newMissingConsensuses = 0; + + private int newMissingMicrodescConsensuses = 0; + + private int newMissingVotes = 0; + + private int newMissingServerDescriptors = 0; + + private int newMissingExtraInfoDescriptors = 0; + + private int newMissingMicrodescriptors = 0; + + private int requestedConsensuses = 0; + + private int requestedMicrodescConsensuses = 0; + + private int requestedVotes = 0; + + private int requestedMissingServerDescriptors = 0; + + private int requestedAllServerDescriptors = 0; + + private int requestedMissingExtraInfoDescriptors = 0; + + private int requestedAllExtraInfoDescriptors = 0; + + private int requestedMissingMicrodescriptors = 0; + + private int downloadedConsensuses = 0; + + private int downloadedMicrodescConsensuses = 0; + + private int downloadedVotes = 0; + + private int downloadedMissingServerDescriptors = 0; + + private int downloadedAllServerDescriptors = 0; + + private int downloadedMissingExtraInfoDescriptors = 0; + + private int downloadedAllExtraInfoDescriptors = 0; + + private int downloadedMissingMicrodescriptors = 0; /** * Initializes this class, including reading in missing descriptors from @@ -292,8 +329,8 @@ public class RelayDescriptorDownloader { SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); format.setTimeZone(TimeZone.getTimeZone("UTC")); long now = System.currentTimeMillis(); - this.currentValidAfter = format.format((now / (60L * 60L * 1000L)) * - (60L * 60L * 1000L)); + this.currentValidAfter = format.format((now / (60L * 60L * 1000L)) + * (60L * 60L * 1000L)); this.descriptorCutOff = format.format(now - 24L * 60L * 60L * 1000L); this.currentTimestamp = format.format(now); this.downloadAllDescriptorsCutOff = format.format(now @@ -317,14 +354,14 @@ public class RelayDescriptorDownloader { while ((line = br.readLine()) != null) { if (line.split(",").length > 2) { String published = line.split(",")[1]; - if (((line.startsWith("consensus,") || - line.startsWith("consensus-microdesc,") || - line.startsWith("vote,")) && - this.currentValidAfter.equals(published)) || - ((line.startsWith("server,") || - line.startsWith("extra,") || - line.startsWith("micro,")) && - this.descriptorCutOff.compareTo(published) < 0)) { + if (((line.startsWith("consensus,") + || line.startsWith("consensus-microdesc,") + || line.startsWith("vote,")) + && this.currentValidAfter.equals(published)) + || ((line.startsWith("server,") + || line.startsWith("extra,") + || line.startsWith("micro,")) + && this.descriptorCutOff.compareTo(published) < 0)) { if (!line.endsWith("NA")) { /* Not missing. */ } else if (line.startsWith("consensus,")) { @@ -352,8 +389,8 @@ public class RelayDescriptorDownloader { } this.microdescriptorKeys.get(microdescriptorDigest).add( microdescriptorKey); - if (line.endsWith("NA") && !this.missingMicrodescriptors. - contains(microdescriptorDigest)) { + if (line.endsWith("NA") && !this.missingMicrodescriptors + .contains(microdescriptorDigest)) { this.missingMicrodescriptors.add(microdescriptorDigest); oldMissingMicrodescriptors++; } @@ -418,8 +455,8 @@ public class RelayDescriptorDownloader { * download all server and extra-info descriptors from. */ this.downloadAllDescriptorsFromAuthorities = new HashSet<String>(); for (String authority : this.authorities) { - if (!this.lastDownloadedAllDescriptors.containsKey(authority) || - this.lastDownloadedAllDescriptors.get(authority).compareTo( + if (!this.lastDownloadedAllDescriptors.containsKey(authority) + || this.lastDownloadedAllDescriptors.get(authority).compareTo( this.downloadAllDescriptorsCutOff) < 0) { this.downloadAllDescriptorsFromAuthorities.add(authority); } @@ -523,8 +560,8 @@ public class RelayDescriptorDownloader { this.microdescriptorKeys.get(microdescriptorDigest).add( microdescriptorKey); this.missingDescriptors.put(microdescriptorKey, parsed); - if (parsed.equals("NA") && - !this.missingMicrodescriptors.contains(microdescriptorDigest)) { + if (parsed.equals("NA") + && !this.missingMicrodescriptors.contains(microdescriptorDigest)) { this.missingMicrodescriptors.add(microdescriptorDigest); this.newMissingMicrodescriptors++; } @@ -662,8 +699,8 @@ public class RelayDescriptorDownloader { /* Start with downloading the current consensus, unless we already * have it. */ if (downloadCurrentConsensus) { - if (this.missingDescriptors.containsKey(consensusKey) && - this.missingDescriptors.get(consensusKey).equals("NA")) { + if (this.missingDescriptors.containsKey(consensusKey) + && this.missingDescriptors.get(consensusKey).equals("NA")) { this.requestedConsensuses++; this.downloadedConsensuses += this.downloadResourceFromAuthority(authority, @@ -673,10 +710,9 @@ public class RelayDescriptorDownloader { /* Then try to download the microdesc consensus. */ if (downloadCurrentMicrodescConsensus) { - if (this.missingDescriptors.containsKey( - microdescConsensusKey) && - this.missingDescriptors.get(microdescConsensusKey). - equals("NA")) { + if (this.missingDescriptors.containsKey(microdescConsensusKey) + && this.missingDescriptors.get(microdescConsensusKey) + .equals("NA")) { this.requestedMicrodescConsensuses++; this.downloadedMicrodescConsensuses += this.downloadResourceFromAuthority(authority, @@ -690,8 +726,8 @@ public class RelayDescriptorDownloader { SortedSet<String> fingerprints = new TreeSet<String>(); for (Map.Entry<String, String> e : this.missingDescriptors.entrySet()) { - if (e.getValue().equals("NA") && - e.getKey().startsWith(voteKeyPrefix)) { + if (e.getValue().equals("NA") + && e.getKey().startsWith(voteKeyPrefix)) { String fingerprint = e.getKey().split(",")[2]; fingerprints.add(fingerprint); } @@ -714,9 +750,9 @@ public class RelayDescriptorDownloader { * authority if we haven't done so for 24 hours and if we're * configured to do so. */ if (this.downloadAllDescriptorsFromAuthorities.contains( - authority) && ((type.equals("server") && - this.downloadAllServerDescriptors) || - (type.equals("extra") && this.downloadAllExtraInfos))) { + authority) && ((type.equals("server") + && this.downloadAllServerDescriptors) + || (type.equals("extra") && this.downloadAllExtraInfos))) { int downloadedAllDescriptors = this.downloadResourceFromAuthority(authority, "/tor/" + type + "/all"); @@ -732,11 +768,11 @@ public class RelayDescriptorDownloader { /* Download missing server descriptors, extra-info descriptors, * and microdescriptors if we're configured to do so. */ - } else if ((type.equals("server") && - this.downloadMissingServerDescriptors) || - (type.equals("extra") && this.downloadMissingExtraInfos) || - (type.equals("micro") && - this.downloadMissingMicrodescriptors)) { + } else if ((type.equals("server") + && this.downloadMissingServerDescriptors) + || (type.equals("extra") && this.downloadMissingExtraInfos) + || (type.equals("micro") + && this.downloadMissingMicrodescriptors)) { /* Go through the list of missing descriptors of this type * and combine the descriptor identifiers to a URL of up to @@ -746,23 +782,24 @@ public class RelayDescriptorDownloader { new TreeSet<String>(); for (Map.Entry<String, String> e : this.missingDescriptors.entrySet()) { - if (e.getValue().equals("NA") && - e.getKey().startsWith(type + ",") && - this.descriptorCutOff.compareTo( + if (e.getValue().equals("NA") + && e.getKey().startsWith(type + ",") + && this.descriptorCutOff.compareTo( e.getKey().split(",")[1]) < 0) { String descriptorIdentifier = e.getKey().split(",")[3]; descriptorIdentifiers.add(descriptorIdentifier); } } StringBuilder combinedResource = null; - int descriptorsInCombinedResource = 0, - requestedDescriptors = 0, downloadedDescriptors = 0; + int descriptorsInCombinedResource = 0; + int requestedDescriptors = 0; + int downloadedDescriptors = 0; int maxDescriptorsInCombinedResource = type.equals("micro") ? 92 : 96; String separator = type.equals("micro") ? "-" : "+"; for (String descriptorIdentifier : descriptorIdentifiers) { - if (descriptorsInCombinedResource >= - maxDescriptorsInCombinedResource) { + if (descriptorsInCombinedResource + >= maxDescriptorsInCombinedResource) { requestedDescriptors += descriptorsInCombinedResource; downloadedDescriptors += this.downloadResourceFromAuthority(authority, @@ -836,8 +873,8 @@ public class RelayDescriptorDownloader { huc.connect(); int response = huc.getResponseCode(); if (response == 200) { - BufferedInputStream in = this.downloadCompressed && - !resource.startsWith("/tor/extra/") + BufferedInputStream in = this.downloadCompressed + && !resource.startsWith("/tor/extra/") ? new BufferedInputStream(new InflaterInputStream( huc.getInputStream())) : new BufferedInputStream(huc.getInputStream()); @@ -857,10 +894,10 @@ public class RelayDescriptorDownloader { if (resource.startsWith("/tor/status-vote/current/")) { this.rdp.parse(allData); receivedDescriptors = 1; - } else if (resource.startsWith("/tor/server/") || - resource.startsWith("/tor/extra/")) { - if (resource.equals("/tor/server/all") || - resource.equals("/tor/extra/all")) { + } else if (resource.startsWith("/tor/server/") + || resource.startsWith("/tor/extra/")) { + if (resource.equals("/tor/server/all") + || resource.equals("/tor/extra/all")) { this.lastDownloadedAllDescriptors.put(authority, this.currentTimestamp); } @@ -870,9 +907,11 @@ public class RelayDescriptorDownloader { } catch (UnsupportedEncodingException e) { /* No way that US-ASCII is not supported. */ } - int start = -1, sig = -1, end = -1; - String startToken = resource.startsWith("/tor/server/") ? - "router " : "extra-info "; + int start = -1; + int sig = -1; + int end = -1; + String startToken = resource.startsWith("/tor/server/") + ? "router " : "extra-info "; String sigToken = "\nrouter-signature\n"; String endToken = "\n-----END SIGNATURE-----\n"; while (end < ascii.length()) { @@ -910,7 +949,8 @@ public class RelayDescriptorDownloader { } catch (UnsupportedEncodingException e) { /* No way that US-ASCII is not supported. */ } - int start = -1, end = -1; + int start = -1; + int end = -1; String startToken = "onion-key\n"; while (end < ascii.length()) { start = ascii.indexOf(startToken, end); @@ -961,9 +1001,11 @@ public class RelayDescriptorDownloader { public void writeFile() { /* Write missing descriptors file to disk. */ - int missingConsensuses = 0, missingMicrodescConsensuses = 0, - missingVotes = 0, missingServerDescriptors = 0, - missingExtraInfoDescriptors = 0; + int missingConsensuses = 0; + int missingMicrodescConsensuses = 0; + int missingVotes = 0; + int missingServerDescriptors = 0; + int missingExtraInfoDescriptors = 0; try { this.logger.fine("Writing file " + this.missingDescriptorsFile.getAbsolutePath() + "..."); @@ -972,7 +1014,8 @@ public class RelayDescriptorDownloader { this.missingDescriptorsFile)); for (Map.Entry<String, String> e : this.missingDescriptors.entrySet()) { - String key = e.getKey(), value = e.getValue(); + String key = e.getKey(); + String value = e.getValue(); if (!value.equals("NA")) { /* Not missing. */ } else if (key.startsWith("consensus,")) { @@ -986,6 +1029,7 @@ public class RelayDescriptorDownloader { } else if (key.startsWith("extra,")) { missingExtraInfoDescriptors++; } else if (key.startsWith("micro,")) { + /* We're counting missing microdescriptors below. */ } bw.write(key + "," + value + "\n"); } @@ -1059,7 +1103,7 @@ public class RelayDescriptorDownloader { StringBuilder sb = new StringBuilder(); for (String authority : this.authorities) { sb.append(" " + authority + "=" - + this.requestsByAuthority.get(authority)); + + this.requestsByAuthority.get(authority)); } this.logger.info("We sent these numbers of requests to the directory " + "authorities:" + sb.toString()); diff --git a/src/org/torproject/collector/relaydescs/RelayDescriptorParser.java b/src/org/torproject/collector/relaydescs/RelayDescriptorParser.java index 4158d52..3f9b912 100644 --- a/src/org/torproject/collector/relaydescs/RelayDescriptorParser.java +++ b/src/org/torproject/collector/relaydescs/RelayDescriptorParser.java @@ -1,7 +1,12 @@ -/* Copyright 2010--2014 The Tor Project +/* Copyright 2010--2016 The Tor Project * See LICENSE for licensing information */ + package org.torproject.collector.relaydescs; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.codec.digest.DigestUtils; + import java.io.BufferedReader; import java.io.IOException; import java.io.StringReader; @@ -13,10 +18,6 @@ import java.util.TreeSet; import java.util.logging.Level; import java.util.logging.Logger; -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.codec.binary.Hex; -import org.apache.commons.codec.digest.DigestUtils; - /** * Parses relay descriptors including network status consensuses and * votes, server and extra-info descriptors, and passes the results to the @@ -93,9 +94,11 @@ public class RelayDescriptorParser { if (line.equals("network-status-version 3 microdesc")) { statusType = "consensus-microdesc"; } - String validAfterTime = null, fingerprint = null, - dirSource = null; - long validAfter = -1L, dirKeyPublished = -1L; + String validAfterTime = null; + String fingerprint = null; + String dirSource = null; + long validAfter = -1L; + long dirKeyPublished = -1L; SortedSet<String> dirSources = new TreeSet<String>(); SortedSet<String> serverDescriptors = new TreeSet<String>(); SortedSet<String> serverDescriptorDigests = new TreeSet<String>(); @@ -130,8 +133,8 @@ public class RelayDescriptorParser { } else if (line.startsWith("dir-key-published ")) { String dirKeyPublishedTime = line.substring( "dir-key-published ".length()); - dirKeyPublished = parseFormat.parse(dirKeyPublishedTime). - getTime(); + dirKeyPublished = parseFormat.parse(dirKeyPublishedTime) + .getTime(); } else if (line.startsWith("r ")) { String[] parts = line.split(" "); if (parts.length == 8) { @@ -158,12 +161,12 @@ public class RelayDescriptorParser { microdescriptorKeys.add(validAfterTime + "," + lastRelayIdentity + "," + digest256Base64); String digest256Hex = Hex.encodeHexString( - Base64.decodeBase64(digest256Base64 + "=")). - toLowerCase(); + Base64.decodeBase64(digest256Base64 + "=")) + .toLowerCase(); microdescriptorDigests.add(digest256Hex); - } else if (parts.length != 3 || - !parts[2].startsWith("sha256=") || - parts[2].length() != 50) { + } else if (parts.length != 3 + || !parts[2].startsWith("sha256=") + || parts[2].length() != 50) { this.logger.log(Level.WARNING, "Could not parse m line '" + line + "' in descriptor. Skipping."); break; @@ -226,23 +229,24 @@ public class RelayDescriptorParser { } } } else if (line.startsWith("router ")) { - String publishedTime = null, extraInfoDigest = null, - relayIdentifier = null; + String publishedTime = null; + String extraInfoDigest = null; + String relayIdentifier = null; long published = -1L; while ((line = br.readLine()) != null) { if (line.startsWith("published ")) { publishedTime = line.substring("published ".length()); published = parseFormat.parse(publishedTime).getTime(); - } else if (line.startsWith("opt fingerprint") || - line.startsWith("fingerprint")) { - relayIdentifier = line.substring(line.startsWith("opt ") ? - "opt fingerprint".length() : "fingerprint".length()). - replaceAll(" ", "").toLowerCase(); - } else if (line.startsWith("opt extra-info-digest ") || - line.startsWith("extra-info-digest ")) { - extraInfoDigest = line.startsWith("opt ") ? - line.split(" ")[2].toLowerCase() : - line.split(" ")[1].toLowerCase(); + } else if (line.startsWith("opt fingerprint") + || line.startsWith("fingerprint")) { + relayIdentifier = line.substring(line.startsWith("opt ") + ? "opt fingerprint".length() : "fingerprint".length()) + .replaceAll(" ", "").toLowerCase(); + } else if (line.startsWith("opt extra-info-digest ") + || line.startsWith("extra-info-digest ")) { + extraInfoDigest = line.startsWith("opt ") + ? line.split(" ")[2].toLowerCase() + : line.split(" ")[1].toLowerCase(); } } String ascii = new String(data, "US-ASCII"); @@ -266,7 +270,8 @@ public class RelayDescriptorParser { relayIdentifier, digest, extraInfoDigest); } } else if (line.startsWith("extra-info ")) { - String publishedTime = null, relayIdentifier = line.split(" ")[2]; + String publishedTime = null; + String relayIdentifier = line.split(" ")[2]; long published = -1L; while ((line = br.readLine()) != null) { if (line.startsWith("published ")) { diff --git a/src/org/torproject/collector/torperf/TorperfDownloader.java b/src/org/torproject/collector/torperf/TorperfDownloader.java index a1483c2..a2c94ef 100644 --- a/src/org/torproject/collector/torperf/TorperfDownloader.java +++ b/src/org/torproject/collector/torperf/TorperfDownloader.java @@ -1,7 +1,12 @@ /* Copyright 2012-2016 The Tor Project * See LICENSE for licensing information */ + package org.torproject.collector.torperf; +import org.torproject.collector.main.Configuration; +import org.torproject.collector.main.LockFile; +import org.torproject.collector.main.LoggingConfiguration; + import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; @@ -22,10 +27,6 @@ import java.util.TreeMap; import java.util.logging.Level; import java.util.logging.Logger; -import org.torproject.collector.main.Configuration; -import org.torproject.collector.main.LockFile; -import org.torproject.collector.main.LoggingConfiguration; - /* Download possibly truncated Torperf .data and .extradata files from * configured sources, append them to the files we already have, and merge * the two files into the .tpf format. */ @@ -97,8 +98,10 @@ public class TorperfDownloader extends Thread { private File torperfLastMergedFile = new File("stats/torperf-last-merged"); + SortedMap<String, String> lastMergedTimestamps = new TreeMap<String, String>(); + private void readLastMergedTimestamps() { if (!this.torperfLastMergedFile.exists()) { return; @@ -109,7 +112,8 @@ public class TorperfDownloader extends Thread { String line; while ((line = br.readLine()) != null) { String[] parts = line.split(" "); - String fileName = null, timestamp = null; + String fileName = null; + String timestamp = null; if (parts.length == 2) { try { Double.parseDouble(parts[1]); @@ -315,11 +319,14 @@ public class TorperfDownloader extends Thread { } this.logger.fine("Merging " + dataFile.getAbsolutePath() + " and " + extradataFile.getAbsolutePath() + " into .tpf format."); - BufferedReader brD = new BufferedReader(new FileReader(dataFile)), - brE = new BufferedReader(new FileReader(extradataFile)); - String lineD = brD.readLine(), lineE = brE.readLine(); - int d = 1, e = 1; - String maxDataComplete = null, maxUsedAt = null; + BufferedReader brD = new BufferedReader(new FileReader(dataFile)); + BufferedReader brE = new BufferedReader(new FileReader(extradataFile)); + String lineD = brD.readLine(); + String lineE = brE.readLine(); + int d = 1; + int e = 1; + String maxDataComplete = null; + String maxUsedAt = null; while (lineD != null) { /* Parse .data line. Every valid .data line will go into the .tpf @@ -363,8 +370,8 @@ public class TorperfDownloader extends Thread { + e++ + " which is a BUILDTIMEOUT_SET line."); lineE = brE.readLine(); continue; - } else if (lineE.startsWith("ok ") || - lineE.startsWith("error ")) { + } else if (lineE.startsWith("ok ") + || lineE.startsWith("error ")) { this.logger.finer("Skipping " + extradataFile.getName() + ":" + e++ + " which is in the old format."); lineE = brE.readLine(); @@ -446,6 +453,7 @@ public class TorperfDownloader extends Thread { } private SortedMap<Integer, String> dataTimestamps; + private SortedMap<String, String> parseDataLine(String line) { String[] parts = line.trim().split(" "); if (line.length() == 0 || parts.length < 20) { @@ -504,18 +512,18 @@ public class TorperfDownloader extends Thread { String key = keyAndValue[0]; previousKey = key; String value = keyAndValue[1]; - if (value.contains(".") && value.lastIndexOf(".") == - value.length() - 2) { + if (value.contains(".") && value.lastIndexOf(".") + == value.length() - 2) { /* Make sure that all floats have two trailing digits. */ value += "0"; } extradata.put(key, value); } else if (keyAndValue.length == 1 && previousKey != null) { String value = keyAndValue[0]; - if (previousKey.equals("STREAM_FAIL_REASONS") && - (value.equals("MISC") || value.equals("EXITPOLICY") || - value.equals("RESOURCELIMIT") || - value.equals("RESOLVEFAILED"))) { + if (previousKey.equals("STREAM_FAIL_REASONS") + && (value.equals("MISC") || value.equals("EXITPOLICY") + || value.equals("RESOURCELIMIT") + || value.equals("RESOLVEFAILED"))) { extradata.put(previousKey, extradata.get(previousKey) + ":" + value); } else { @@ -529,9 +537,13 @@ public class TorperfDownloader extends Thread { } private String cachedSource; + private int cachedFileSize; + private String cachedStartDate; + private SortedMap<String, String> cachedTpfLines; + private void writeTpfLine(String source, int fileSize, SortedMap<String, String> keysAndValues) throws IOException { StringBuilder sb = new StringBuilder(); @@ -547,14 +559,14 @@ public class TorperfDownloader extends Thread { long startMillis = Long.parseLong(startString.substring(0, startString.indexOf("."))) * 1000L; String startDate = dateFormat.format(startMillis); - if (this.cachedTpfLines == null || !source.equals(this.cachedSource) || - fileSize != this.cachedFileSize || - !startDate.equals(this.cachedStartDate)) { + if (this.cachedTpfLines == null || !source.equals(this.cachedSource) + || fileSize != this.cachedFileSize + || !startDate.equals(this.cachedStartDate)) { this.writeCachedTpfLines(); this.readTpfLinesToCache(source, fileSize, startDate); } - if (!this.cachedTpfLines.containsKey(startString) || - line.length() > this.cachedTpfLines.get(startString).length()) { + if (!this.cachedTpfLines.containsKey(startString) + || line.length() > this.cachedTpfLines.get(startString).length()) { this.cachedTpfLines.put(startString, line); } } @@ -588,8 +600,8 @@ public class TorperfDownloader extends Thread { } private void writeCachedTpfLines() throws IOException { - if (this.cachedSource == null || this.cachedFileSize == 0 || - this.cachedStartDate == null || this.cachedTpfLines == null) { + if (this.cachedSource == null || this.cachedFileSize == 0 + || this.cachedStartDate == null || this.cachedTpfLines == null) { return; } File tarballFile = new File(torperfOutputDirectory,

1 0

[webwml/master] Deprecate old Tor Browser versions
by gk＠torproject.org 30 Apr '16

30 Apr '16

commit 8f56c6bc02cb7af96b18efe7c57c10a94edf9b3d Author: Georg Koppen <gk(a)torproject.org> Date: Sat Apr 30 16:38:54 2016 +0000 Deprecate old Tor Browser versions --- projects/torbrowser/RecommendedTBBVersions | 6 ------ 1 file changed, 6 deletions(-) diff --git a/projects/torbrowser/RecommendedTBBVersions b/projects/torbrowser/RecommendedTBBVersions index ef7fcf1..61ac009 100644 --- a/projects/torbrowser/RecommendedTBBVersions +++ b/projects/torbrowser/RecommendedTBBVersions @@ -3,16 +3,10 @@ "5.5.5-Linux", "5.5.5-MacOS", "5.5.5-Windows", -"6.0a4", -"6.0a4-Linux", -"6.0a4-MacOS", -"6.0a4-Windows", "6.0a5", "6.0a5-Linux", "6.0a5-MacOS", "6.0a5-Windows", -"6.0a4-hardened", -"6.0a4-hardened-Linux", "6.0a5-hardened", "6.0a5-hardened-Linux" ]

1 0

[stem/master] Drop nickname from fallback directories
by atagar＠torproject.org 30 Apr '16

30 Apr '16

commit 2bc6282179705a58e4ed114b0895c2ab2d83ab01 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Apr 29 11:44:08 2016 -0700 Drop nickname from fallback directories Tor commit 8b90d45 complicates how fallback_dirs.inc is generated. Now that it no longer has hand-managed headers we don't have the nicknames. Dropping this attribute from the class. --- cache_fallback_directories.py | 1 - stem/descriptor/fallback_directories.cfg | 408 ++++++++++++++++++++++--------- stem/descriptor/remote.py | 75 +++--- test/unit/descriptor/remote.py | 18 +- 4 files changed, 323 insertions(+), 179 deletions(-) diff --git a/cache_fallback_directories.py b/cache_fallback_directories.py index 138f767..21d9fcf 100755 --- a/cache_fallback_directories.py +++ b/cache_fallback_directories.py @@ -59,6 +59,5 @@ if __name__ == '__main__': conf.set('%s.address' % fingerprint, directory.address) conf.set('%s.or_port' % fingerprint, str(directory.or_port)) conf.set('%s.dir_port' % fingerprint, str(directory.dir_port)) - conf.set('%s.nickname' % fingerprint, directory.nickname) conf.save(stem.descriptor.remote.CACHE_PATH) diff --git a/stem/descriptor/fallback_directories.cfg b/stem/descriptor/fallback_directories.cfg index d420fb5..50ad27b 100644 --- a/stem/descriptor/fallback_directories.cfg +++ b/stem/descriptor/fallback_directories.cfg @@ -1,130 +1,302 @@ -tor_commit e2202146d16af22502fd4166ac926fefa6528dff -stem_commit a96ec510539f2918bf93c50d63ce68e919f71209 -8FA37B93397015B2BC5A525C908485260BE9F422.address 178.254.44.135 -8FA37B93397015B2BC5A525C908485260BE9F422.or_port 9001 -8FA37B93397015B2BC5A525C908485260BE9F422.dir_port 9030 -8FA37B93397015B2BC5A525C908485260BE9F422.nickname Doedel22 -855BC2DABE24C861CD887DB9B2E950424B49FC34.address 84.219.173.60 -855BC2DABE24C861CD887DB9B2E950424B49FC34.or_port 443 -855BC2DABE24C861CD887DB9B2E950424B49FC34.dir_port 9030 -855BC2DABE24C861CD887DB9B2E950424B49FC34.nickname Logforme +tor_commit 1fd4340f827065485f8ce3fd03c5573f89880893 +stem_commit b5661f160a393984fd99b2cb67438ee4eb76e3a9 +9504CB22EEB25D344DE63CB7A6F2C46F895C3686.address 46.101.102.71 +9504CB22EEB25D344DE63CB7A6F2C46F895C3686.or_port 443 +9504CB22EEB25D344DE63CB7A6F2C46F895C3686.dir_port 80 +823AA81E277F366505545522CEDC2F529CE4DC3F.address 192.160.102.164 +823AA81E277F366505545522CEDC2F529CE4DC3F.or_port 9001 +823AA81E277F366505545522CEDC2F529CE4DC3F.dir_port 80 +AEA43CB1E47BE5F8051711B2BF01683DB1568E05.address 37.187.7.74 +AEA43CB1E47BE5F8051711B2BF01683DB1568E05.or_port 443 +AEA43CB1E47BE5F8051711B2BF01683DB1568E05.dir_port 80 +86C281AD135058238D7A337D546C902BE8505DDE.address 185.96.88.29 +86C281AD135058238D7A337D546C902BE8505DDE.or_port 443 +86C281AD135058238D7A337D546C902BE8505DDE.dir_port 80 +8664DC892540F3C789DB37008236C096C871734D.address 163.172.138.22 +8664DC892540F3C789DB37008236C096C871734D.or_port 443 +8664DC892540F3C789DB37008236C096C871734D.dir_port 80 +387B065A38E4DAA16D9D41C2964ECBC4B31D30FF.address 62.210.92.11 +387B065A38E4DAA16D9D41C2964ECBC4B31D30FF.or_port 9101 +387B065A38E4DAA16D9D41C2964ECBC4B31D30FF.dir_port 9130 +DE134FC8E5CC4EC8A5DE66934E70AC9D70267197.address 195.191.233.221 +DE134FC8E5CC4EC8A5DE66934E70AC9D70267197.or_port 443 +DE134FC8E5CC4EC8A5DE66934E70AC9D70267197.dir_port 80 +5714542DCBEE1DD9864824723638FD44B2122CEA.address 109.163.234.9 +5714542DCBEE1DD9864824723638FD44B2122CEA.or_port 443 +5714542DCBEE1DD9864824723638FD44B2122CEA.dir_port 80 +50586E25BE067FD1F739998550EDDCB1A14CA5B2.address 212.51.134.123 +50586E25BE067FD1F739998550EDDCB1A14CA5B2.or_port 9001 +50586E25BE067FD1F739998550EDDCB1A14CA5B2.dir_port 9030 +BCEDF6C193AA687AE471B8A22EBF6BC57C2D285E.address 198.96.155.3 +BCEDF6C193AA687AE471B8A22EBF6BC57C2D285E.or_port 5001 +BCEDF6C193AA687AE471B8A22EBF6BC57C2D285E.dir_port 8080 +F65E0196C94DFFF48AFBF2F5F9E3E19AAE583FD0.address 94.242.246.23 +F65E0196C94DFFF48AFBF2F5F9E3E19AAE583FD0.or_port 9001 +F65E0196C94DFFF48AFBF2F5F9E3E19AAE583FD0.dir_port 443 +616081EC829593AF4232550DE6FFAA1D75B37A90.address 95.128.43.164 +616081EC829593AF4232550DE6FFAA1D75B37A90.or_port 443 +616081EC829593AF4232550DE6FFAA1D75B37A90.dir_port 80 +58ED9C9C35E433EE58764D62892B4FFD518A3CD0.address 185.21.100.50 +58ED9C9C35E433EE58764D62892B4FFD518A3CD0.or_port 9001 +58ED9C9C35E433EE58764D62892B4FFD518A3CD0.dir_port 9030 +35E8B344F661F4F2E68B17648F35798B44672D7E.address 146.0.32.144 +35E8B344F661F4F2E68B17648F35798B44672D7E.or_port 9001 +35E8B344F661F4F2E68B17648F35798B44672D7E.dir_port 9030 +14419131033443AE6E21DA82B0D307F7CAE42BDB.address 144.76.14.145 +14419131033443AE6E21DA82B0D307F7CAE42BDB.or_port 143 +14419131033443AE6E21DA82B0D307F7CAE42BDB.dir_port 110 +79E169B25E4C7CE99584F6ED06F379478F23E2B8.address 62.210.129.246 +79E169B25E4C7CE99584F6ED06F379478F23E2B8.or_port 443 +79E169B25E4C7CE99584F6ED06F379478F23E2B8.dir_port 80 +FD1871854BFC06D7B02F10742073069F0528B5CC.address 192.187.124.98 +FD1871854BFC06D7B02F10742073069F0528B5CC.or_port 9001 +FD1871854BFC06D7B02F10742073069F0528B5CC.dir_port 9030 +EBE718E1A49EE229071702964F8DB1F318075FF8.address 131.188.40.188 +EBE718E1A49EE229071702964F8DB1F318075FF8.or_port 80 +EBE718E1A49EE229071702964F8DB1F318075FF8.dir_port 443 +489D94333DF66D57FFE34D9D59CC2D97E2CB0053.address 37.187.102.186 +489D94333DF66D57FFE34D9D59CC2D97E2CB0053.or_port 9001 +489D94333DF66D57FFE34D9D59CC2D97E2CB0053.dir_port 9030 +99E246DB480B313A3012BC3363093CC26CD209C7.address 81.7.10.93 +99E246DB480B313A3012BC3363093CC26CD209C7.or_port 31337 +99E246DB480B313A3012BC3363093CC26CD209C7.dir_port 31336 +3D7E274A87D9A89AF064C13D1EE4CA1F184F2600.address 176.10.107.180 +3D7E274A87D9A89AF064C13D1EE4CA1F184F2600.or_port 9001 +3D7E274A87D9A89AF064C13D1EE4CA1F184F2600.dir_port 9030 +D5039E1EBFD96D9A3F9846BF99EC9F75EDDE902A.address 37.187.115.157 +D5039E1EBFD96D9A3F9846BF99EC9F75EDDE902A.or_port 9001 +D5039E1EBFD96D9A3F9846BF99EC9F75EDDE902A.dir_port 9030 AE6A8C18E7499B586CD36246AC4BCAFFBBF93AB2.address 178.254.44.135 AE6A8C18E7499B586CD36246AC4BCAFFBBF93AB2.or_port 443 AE6A8C18E7499B586CD36246AC4BCAFFBBF93AB2.dir_port 80 -AE6A8C18E7499B586CD36246AC4BCAFFBBF93AB2.nickname Doedel21 -6FB38EB22E57EF7ED5EF00238F6A48E553735D88.address 212.47.237.95 -6FB38EB22E57EF7ED5EF00238F6A48E553735D88.or_port 9101 -6FB38EB22E57EF7ED5EF00238F6A48E553735D88.dir_port 9130 -6FB38EB22E57EF7ED5EF00238F6A48E553735D88.nickname kitten4 -3F5D8A879C58961BB45A3D26AC41B543B40236D6.address 212.47.237.95 -3F5D8A879C58961BB45A3D26AC41B543B40236D6.or_port 9001 -3F5D8A879C58961BB45A3D26AC41B543B40236D6.dir_port 9030 -3F5D8A879C58961BB45A3D26AC41B543B40236D6.nickname kitten3 -943C0C6841C1E914B9FCA796C6846620A5AF9BC7.address 217.12.210.214 -943C0C6841C1E914B9FCA796C6846620A5AF9BC7.or_port 443 -943C0C6841C1E914B9FCA796C6846620A5AF9BC7.dir_port 80 -943C0C6841C1E914B9FCA796C6846620A5AF9BC7.nickname Unnamed -A360C21FA87FFA2046D92C17086A6B47E5C68109.address 51.255.33.237 -A360C21FA87FFA2046D92C17086A6B47E5C68109.or_port 9001 -A360C21FA87FFA2046D92C17086A6B47E5C68109.dir_port 9091 -A360C21FA87FFA2046D92C17086A6B47E5C68109.nickname coby +262B66AD25C79588AD1FC8ED0E966395B47E5C1D.address 51.254.215.121 +262B66AD25C79588AD1FC8ED0E966395B47E5C1D.or_port 443 +262B66AD25C79588AD1FC8ED0E966395B47E5C1D.dir_port 80 +01A9258A46E97FF8B2CAC7910577862C14F2C524.address 193.171.202.146 +01A9258A46E97FF8B2CAC7910577862C14F2C524.or_port 9001 +01A9258A46E97FF8B2CAC7910577862C14F2C524.dir_port 9030 +9007C1D8E4F03D506A4A011B907A9E8D04E3C605.address 151.80.42.103 +9007C1D8E4F03D506A4A011B907A9E8D04E3C605.or_port 9001 +9007C1D8E4F03D506A4A011B907A9E8D04E3C605.dir_port 9030 +B143D439B72D239A419F8DCE07B8A8EB1B486FA7.address 212.129.62.232 +B143D439B72D239A419F8DCE07B8A8EB1B486FA7.or_port 443 +B143D439B72D239A419F8DCE07B8A8EB1B486FA7.dir_port 80 +32EE911D968BE3E016ECA572BB1ED0A9EE43FC2F.address 109.105.109.162 +32EE911D968BE3E016ECA572BB1ED0A9EE43FC2F.or_port 60784 +32EE911D968BE3E016ECA572BB1ED0A9EE43FC2F.dir_port 52860 +39F096961ED2576975C866D450373A9913AFDC92.address 198.50.191.95 +39F096961ED2576975C866D450373A9913AFDC92.or_port 443 +39F096961ED2576975C866D450373A9913AFDC92.dir_port 80 +64186650FFE4469EBBE52B644AE543864D32F43C.address 89.187.142.208 +64186650FFE4469EBBE52B644AE543864D32F43C.or_port 443 +64186650FFE4469EBBE52B644AE543864D32F43C.dir_port 80 +D64366987CB39F61AD21DBCF8142FA0577B92811.address 37.221.162.226 +D64366987CB39F61AD21DBCF8142FA0577B92811.or_port 9001 +D64366987CB39F61AD21DBCF8142FA0577B92811.dir_port 9030 +BC630CBBB518BE7E9F4E09712AB0269E9DC7D626.address 197.231.221.211 +BC630CBBB518BE7E9F4E09712AB0269E9DC7D626.or_port 9001 +BC630CBBB518BE7E9F4E09712AB0269E9DC7D626.dir_port 9030 +AC66FFA4AB35A59EBBF5BF4C70008BF24D8A7A5C.address 195.154.164.243 +AC66FFA4AB35A59EBBF5BF4C70008BF24D8A7A5C.or_port 443 +AC66FFA4AB35A59EBBF5BF4C70008BF24D8A7A5C.dir_port 80 +A10C4F666D27364036B562823E5830BC448E046A.address 171.25.193.77 +A10C4F666D27364036B562823E5830BC448E046A.or_port 443 +A10C4F666D27364036B562823E5830BC448E046A.dir_port 80 +CBEE0F3303C8C50462A12107CA2AE061831931BC.address 91.121.54.8 +CBEE0F3303C8C50462A12107CA2AE061831931BC.or_port 9001 +CBEE0F3303C8C50462A12107CA2AE061831931BC.dir_port 9030 +CBEFF7BA4A4062045133C053F2D70524D8BBE5BE.address 178.62.199.226 +CBEFF7BA4A4062045133C053F2D70524D8BBE5BE.or_port 443 +CBEFF7BA4A4062045133C053F2D70524D8BBE5BE.dir_port 80 +B87C84E38DAECFFFFDE98E5AEE5786AFDC748F2C.address 178.62.36.64 +B87C84E38DAECFFFFDE98E5AEE5786AFDC748F2C.or_port 9001 +B87C84E38DAECFFFFDE98E5AEE5786AFDC748F2C.dir_port 9030 +EC413181CEB1C8EDC17608BBB177CD5FD8535E99.address 91.219.236.222 +EC413181CEB1C8EDC17608BBB177CD5FD8535E99.or_port 443 +EC413181CEB1C8EDC17608BBB177CD5FD8535E99.dir_port 80 +49E7AD01BB96F6FE3AB8C3B15BD2470B150354DF.address 188.165.194.195 +49E7AD01BB96F6FE3AB8C3B15BD2470B150354DF.or_port 9001 +49E7AD01BB96F6FE3AB8C3B15BD2470B150354DF.dir_port 9030 +C697612CA5AED06B8D829FCC6065B9287212CB2F.address 195.154.79.128 +C697612CA5AED06B8D829FCC6065B9287212CB2F.or_port 443 +C697612CA5AED06B8D829FCC6065B9287212CB2F.dir_port 80 +8B7F47AE1A5D954A3E58ACDE0865D09DBA5B738D.address 178.217.184.32 +8B7F47AE1A5D954A3E58ACDE0865D09DBA5B738D.or_port 443 +8B7F47AE1A5D954A3E58ACDE0865D09DBA5B738D.dir_port 9030 +CBD0D1BD110EC52963082D839AC6A89D0AE243E7.address 37.59.46.159 +CBD0D1BD110EC52963082D839AC6A89D0AE243E7.or_port 9001 +CBD0D1BD110EC52963082D839AC6A89D0AE243E7.dir_port 9030 +322C6E3A973BC10FC36DE3037AD27BC89F14723B.address 212.83.154.33 +322C6E3A973BC10FC36DE3037AD27BC89F14723B.or_port 8443 +322C6E3A973BC10FC36DE3037AD27BC89F14723B.dir_port 8080 +DD85503F2D1F52EF9EAD621E942298F46CD2FC10.address 178.62.173.203 +DD85503F2D1F52EF9EAD621E942298F46CD2FC10.or_port 9001 +DD85503F2D1F52EF9EAD621E942298F46CD2FC10.dir_port 9030 +C414F28FD2BEC1553024299B31D4E726BEB8E788.address 188.138.112.60 +C414F28FD2BEC1553024299B31D4E726BEB8E788.or_port 1521 +C414F28FD2BEC1553024299B31D4E726BEB8E788.dir_port 1433 +FFA72BD683BC2FCF988356E6BEC1E490F313FB07.address 193.11.164.243 +FFA72BD683BC2FCF988356E6BEC1E490F313FB07.or_port 9001 +FFA72BD683BC2FCF988356E6BEC1E490F313FB07.dir_port 9030 +5665A3904C89E22E971305EE8C1997BCA4123C69.address 94.23.204.175 +5665A3904C89E22E971305EE8C1997BCA4123C69.or_port 9001 +5665A3904C89E22E971305EE8C1997BCA4123C69.dir_port 9030 +8844D87E9B038BE3270938F05AF797E1D3C74C0F.address 93.180.156.84 +8844D87E9B038BE3270938F05AF797E1D3C74C0F.or_port 9001 +8844D87E9B038BE3270938F05AF797E1D3C74C0F.dir_port 9030 +E781F4EC69671B3F1864AE2753E0890351506329.address 176.31.180.157 +E781F4EC69671B3F1864AE2753E0890351506329.or_port 22 +E781F4EC69671B3F1864AE2753E0890351506329.dir_port 143 +D62FB817B0288085FAC38A6DC8B36DCD85B70260.address 185.14.185.240 +D62FB817B0288085FAC38A6DC8B36DCD85B70260.or_port 443 +D62FB817B0288085FAC38A6DC8B36DCD85B70260.dir_port 9030 4F0DB7E687FC7C0AE55C8F243DA8B0EB27FBF1F2.address 108.53.208.157 4F0DB7E687FC7C0AE55C8F243DA8B0EB27FBF1F2.or_port 443 4F0DB7E687FC7C0AE55C8F243DA8B0EB27FBF1F2.dir_port 80 -4F0DB7E687FC7C0AE55C8F243DA8B0EB27FBF1F2.nickname Binnacle -F8D27B163B9247B232A2EEE68DD8B698695C28DE.address 78.47.18.110 -F8D27B163B9247B232A2EEE68DD8B698695C28DE.or_port 80 -F8D27B163B9247B232A2EEE68DD8B698695C28DE.dir_port 443 -F8D27B163B9247B232A2EEE68DD8B698695C28DE.nickname fluxe3 +0BEA4A88D069753218EAAAD6D22EA87B9A1319D6.address 5.39.92.199 +0BEA4A88D069753218EAAAD6D22EA87B9A1319D6.or_port 443 +0BEA4A88D069753218EAAAD6D22EA87B9A1319D6.dir_port 80 +CFECDDCA990E3EF7B7EC958B22441386B6B8D820.address 81.7.17.171 +CFECDDCA990E3EF7B7EC958B22441386B6B8D820.or_port 443 +CFECDDCA990E3EF7B7EC958B22441386B6B8D820.dir_port 80 +92ECC9E0E2AF81BB954719B189AC362E254AD4A5.address 91.219.237.244 +92ECC9E0E2AF81BB954719B189AC362E254AD4A5.or_port 443 +92ECC9E0E2AF81BB954719B189AC362E254AD4A5.dir_port 80 +D2A1703758A0FBBA026988B92C2F88BAB59F9361.address 185.13.38.75 +D2A1703758A0FBBA026988B92C2F88BAB59F9361.or_port 9001 +D2A1703758A0FBBA026988B92C2F88BAB59F9361.dir_port 9030 0756B7CD4DFC8182BE23143FAC0642F515182CEB.address 5.9.110.236 0756B7CD4DFC8182BE23143FAC0642F515182CEB.or_port 9001 0756B7CD4DFC8182BE23143FAC0642F515182CEB.dir_port 9030 -0756B7CD4DFC8182BE23143FAC0642F515182CEB.nickname rueckgrat -5BFDECCE9B4A23AE14EC767C5A2C1E10558B00B9.address 213.239.210.204 -5BFDECCE9B4A23AE14EC767C5A2C1E10558B00B9.or_port 443 -5BFDECCE9B4A23AE14EC767C5A2C1E10558B00B9.dir_port 22 -5BFDECCE9B4A23AE14EC767C5A2C1E10558B00B9.nickname Nurnberg01 -5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33.address 5.175.233.86 -5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33.or_port 443 -5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33.dir_port 80 -5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33.nickname wagner +2541759BEC04D37811C2209A88E863320271EC9C.address 185.61.138.18 +2541759BEC04D37811C2209A88E863320271EC9C.or_port 4443 +2541759BEC04D37811C2209A88E863320271EC9C.dir_port 8080 +9030DCF419F6E2FBF84F63CBACBA0097B06F557E.address 5.79.68.161 +9030DCF419F6E2FBF84F63CBACBA0097B06F557E.or_port 443 +9030DCF419F6E2FBF84F63CBACBA0097B06F557E.dir_port 81 +DAA39FC00B196B353C2A271459C305C429AF09E4.address 193.35.52.53 +DAA39FC00B196B353C2A271459C305C429AF09E4.or_port 9001 +DAA39FC00B196B353C2A271459C305C429AF09E4.dir_port 9030 +DDD7871C1B7FA32CB55061E08869A236E61BDDF8.address 5.34.183.205 +DDD7871C1B7FA32CB55061E08869A236E61BDDF8.or_port 443 +DDD7871C1B7FA32CB55061E08869A236E61BDDF8.dir_port 80 +E589316576A399C511A9781A73DA4545640B479D.address 46.252.26.2 +E589316576A399C511A9781A73DA4545640B479D.or_port 49991 +E589316576A399C511A9781A73DA4545640B479D.dir_port 45212 +A1EB8D8F1EE28DB98BBB1EAA3B4BEDD303BAB911.address 176.9.5.116 +A1EB8D8F1EE28DB98BBB1EAA3B4BEDD303BAB911.or_port 9001 +A1EB8D8F1EE28DB98BBB1EAA3B4BEDD303BAB911.dir_port 9030 +98F8D5F359949E41DE8DF3DBB1975A86E96A84A0.address 85.214.206.219 +98F8D5F359949E41DE8DF3DBB1975A86E96A84A0.or_port 9001 +98F8D5F359949E41DE8DF3DBB1975A86E96A84A0.dir_port 9030 +9F7D6E6420183C2B76D3CE99624EBC98A21A967E.address 46.28.110.244 +9F7D6E6420183C2B76D3CE99624EBC98A21A967E.or_port 443 +9F7D6E6420183C2B76D3CE99624EBC98A21A967E.dir_port 80 +A0F06C2FADF88D3A39AA3072B406F09D7095AC9E.address 46.165.230.5 +A0F06C2FADF88D3A39AA3072B406F09D7095AC9E.or_port 443 +A0F06C2FADF88D3A39AA3072B406F09D7095AC9E.dir_port 80 +844AE9CAD04325E955E2BE1521563B79FE7094B7.address 192.87.28.82 +844AE9CAD04325E955E2BE1521563B79FE7094B7.or_port 9001 +844AE9CAD04325E955E2BE1521563B79FE7094B7.dir_port 9030 +30648BC64CEDB3020F4A405E4AB2A6347FB8FA22.address 213.61.66.118 +30648BC64CEDB3020F4A405E4AB2A6347FB8FA22.or_port 9001 +30648BC64CEDB3020F4A405E4AB2A6347FB8FA22.dir_port 9031 +08DC0F3C6E3D9C527C1FC8745D35DD1B0DE1875D.address 37.187.1.149 +08DC0F3C6E3D9C527C1FC8745D35DD1B0DE1875D.or_port 9001 +08DC0F3C6E3D9C527C1FC8745D35DD1B0DE1875D.dir_port 9030 +2BA2C8E96B2590E1072AECE2BDB5C48921BF8510.address 144.76.26.175 +2BA2C8E96B2590E1072AECE2BDB5C48921BF8510.or_port 9011 +2BA2C8E96B2590E1072AECE2BDB5C48921BF8510.dir_port 9012 +7350AB9ED7568F22745198359373C04AC783C37C.address 176.31.191.26 +7350AB9ED7568F22745198359373C04AC783C37C.or_port 9001 +7350AB9ED7568F22745198359373C04AC783C37C.dir_port 9030 +794D8EA8343A4E820320265D05D4FA83AB6D1778.address 185.11.180.67 +794D8EA8343A4E820320265D05D4FA83AB6D1778.or_port 9001 +794D8EA8343A4E820320265D05D4FA83AB6D1778.dir_port 80 +6A640018EABF3DA9BAD9321AA37C2C87BBE1F907.address 144.76.73.140 +6A640018EABF3DA9BAD9321AA37C2C87BBE1F907.or_port 9001 +6A640018EABF3DA9BAD9321AA37C2C87BBE1F907.dir_port 9030 +0D3EBA17E1C78F1E9900BABDB23861D46FCAF163.address 178.62.197.82 +0D3EBA17E1C78F1E9900BABDB23861D46FCAF163.or_port 443 +0D3EBA17E1C78F1E9900BABDB23861D46FCAF163.dir_port 80 +BF0FB582E37F738CD33C3651125F2772705BB8E8.address 148.251.190.229 +BF0FB582E37F738CD33C3651125F2772705BB8E8.or_port 9010 +BF0FB582E37F738CD33C3651125F2772705BB8E8.dir_port 9030 +6FC6F08270D565BE89B7C819DD8E2D487397C073.address 94.126.23.174 +6FC6F08270D565BE89B7C819DD8E2D487397C073.or_port 9001 +6FC6F08270D565BE89B7C819DD8E2D487397C073.dir_port 9030 +75F1992FD3F403E9C082A5815EB5D12934CDF46C.address 46.101.237.246 +75F1992FD3F403E9C082A5815EB5D12934CDF46C.or_port 9001 +75F1992FD3F403E9C082A5815EB5D12934CDF46C.dir_port 9030 +855BC2DABE24C861CD887DB9B2E950424B49FC34.address 84.219.173.60 +855BC2DABE24C861CD887DB9B2E950424B49FC34.or_port 443 +855BC2DABE24C861CD887DB9B2E950424B49FC34.dir_port 9030 +29F1020B94BE25E6BE1AD13E93CE19D2131B487C.address 194.150.168.79 +29F1020B94BE25E6BE1AD13E93CE19D2131B487C.or_port 11111 +29F1020B94BE25E6BE1AD13E93CE19D2131B487C.dir_port 11112 +7A32C9519D80CA458FC8B034A28F5F6815649A98.address 82.223.21.74 +7A32C9519D80CA458FC8B034A28F5F6815649A98.or_port 9001 +7A32C9519D80CA458FC8B034A28F5F6815649A98.dir_port 9030 +0C2C599AFCB26F5CFC2C7592435924C1D63D9484.address 5.196.88.122 +0C2C599AFCB26F5CFC2C7592435924C1D63D9484.or_port 9001 +0C2C599AFCB26F5CFC2C7592435924C1D63D9484.dir_port 9030 +F960DF50F0FD4075AC9B505C1D4FFC8384C490FB.address 46.101.143.173 +F960DF50F0FD4075AC9B505C1D4FFC8384C490FB.or_port 443 +F960DF50F0FD4075AC9B505C1D4FFC8384C490FB.dir_port 80 +17898F9A2EBC7D69DAF87C00A1BD2FABF3C9E1D2.address 178.32.216.146 +17898F9A2EBC7D69DAF87C00A1BD2FABF3C9E1D2.or_port 9001 +17898F9A2EBC7D69DAF87C00A1BD2FABF3C9E1D2.dir_port 9030 +D1B8AAA98C65F3DF7D8BB3AF881CAEB84A33D8EE.address 134.119.3.164 +D1B8AAA98C65F3DF7D8BB3AF881CAEB84A33D8EE.or_port 9001 +D1B8AAA98C65F3DF7D8BB3AF881CAEB84A33D8EE.dir_port 9030 +3B33F6FCA645AD4E91428A3AF7DC736AD9FB727B.address 164.132.77.175 +3B33F6FCA645AD4E91428A3AF7DC736AD9FB727B.or_port 9001 +3B33F6FCA645AD4E91428A3AF7DC736AD9FB727B.dir_port 9030 +9231DF741915AA1630031A93026D88726877E93A.address 51.255.41.65 +9231DF741915AA1630031A93026D88726877E93A.or_port 9001 +9231DF741915AA1630031A93026D88726877E93A.dir_port 9030 2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E.address 62.210.124.124 2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E.or_port 9101 2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E.dir_port 9130 -2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E.nickname kitten2 -BEE2317AE127EB681C5AE1551C1EA0630580638A.address 213.239.220.25 -BEE2317AE127EB681C5AE1551C1EA0630580638A.or_port 443 -BEE2317AE127EB681C5AE1551C1EA0630580638A.dir_port 22 -BEE2317AE127EB681C5AE1551C1EA0630580638A.nickname Nurnberg02 -DF3AED4322B1824BF5539AE54B2D1B38E080FF05.address 217.12.199.208 -DF3AED4322B1824BF5539AE54B2D1B38E080FF05.or_port 443 -DF3AED4322B1824BF5539AE54B2D1B38E080FF05.dir_port 80 -DF3AED4322B1824BF5539AE54B2D1B38E080FF05.nickname Unnamed -6DE61A6F72C1E5418A66BFED80DFB63E4C77668F.address 85.25.138.93 -6DE61A6F72C1E5418A66BFED80DFB63E4C77668F.or_port 4029 -6DE61A6F72C1E5418A66BFED80DFB63E4C77668F.dir_port 9030 -6DE61A6F72C1E5418A66BFED80DFB63E4C77668F.nickname eriador -9F5068310818ED7C70B0BC4087AB55CB12CB4377.address 178.254.20.134 -9F5068310818ED7C70B0BC4087AB55CB12CB4377.or_port 443 -9F5068310818ED7C70B0BC4087AB55CB12CB4377.dir_port 80 -9F5068310818ED7C70B0BC4087AB55CB12CB4377.nickname Doedel26 -86E78DD3720C78DA8673182EF96C54B162CD660C.address 62.210.124.124 -86E78DD3720C78DA8673182EF96C54B162CD660C.or_port 9001 -86E78DD3720C78DA8673182EF96C54B162CD660C.dir_port 9030 -86E78DD3720C78DA8673182EF96C54B162CD660C.nickname kitten1 -92CFD9565B24646CAC2D172D3DB503D69E777B8A.address 178.16.208.57 -92CFD9565B24646CAC2D172D3DB503D69E777B8A.or_port 443 -92CFD9565B24646CAC2D172D3DB503D69E777B8A.dir_port 80 -92CFD9565B24646CAC2D172D3DB503D69E777B8A.nickname bakunin -14419131033443AE6E21DA82B0D307F7CAE42BDB.address 144.76.14.145 -14419131033443AE6E21DA82B0D307F7CAE42BDB.or_port 143 -14419131033443AE6E21DA82B0D307F7CAE42BDB.dir_port 110 -14419131033443AE6E21DA82B0D307F7CAE42BDB.nickname PedicaboMundi -CBEFF7BA4A4062045133C053F2D70524D8BBE5BE.address 178.62.199.226 -CBEFF7BA4A4062045133C053F2D70524D8BBE5BE.or_port 443 -CBEFF7BA4A4062045133C053F2D70524D8BBE5BE.dir_port 80 -CBEFF7BA4A4062045133C053F2D70524D8BBE5BE.nickname tornoderdednl -FA3415659444AE006E7E9E5375E82F29700CFDFD.address 170.130.1.7 -FA3415659444AE006E7E9E5375E82F29700CFDFD.or_port 9001 -FA3415659444AE006E7E9E5375E82F29700CFDFD.dir_port 9030 -FA3415659444AE006E7E9E5375E82F29700CFDFD.nickname ratchet -0C475BA4D3AA3C289B716F95954CAD616E50C4E5.address 178.254.13.126 -0C475BA4D3AA3C289B716F95954CAD616E50C4E5.or_port 9001 -0C475BA4D3AA3C289B716F95954CAD616E50C4E5.dir_port 9030 -0C475BA4D3AA3C289B716F95954CAD616E50C4E5.nickname Freebird32 -F9246DEF2B653807236DA134F2AEAB103D58ABFE.address 178.254.13.126 -F9246DEF2B653807236DA134F2AEAB103D58ABFE.or_port 443 -F9246DEF2B653807236DA134F2AEAB103D58ABFE.dir_port 80 -F9246DEF2B653807236DA134F2AEAB103D58ABFE.nickname Freebird31 -00C4B4731658D3B4987132A3F77100CFCB190D97.address 81.7.17.171 -00C4B4731658D3B4987132A3F77100CFCB190D97.or_port 443 -00C4B4731658D3B4987132A3F77100CFCB190D97.dir_port 80 -00C4B4731658D3B4987132A3F77100CFCB190D97.nickname 12xBTME1 -DD85503F2D1F52EF9EAD621E942298F46CD2FC10.address 178.62.173.203 -DD85503F2D1F52EF9EAD621E942298F46CD2FC10.or_port 9001 -DD85503F2D1F52EF9EAD621E942298F46CD2FC10.dir_port 9030 -DD85503F2D1F52EF9EAD621E942298F46CD2FC10.nickname kili -EBE718E1A49EE229071702964F8DB1F318075FF8.address 131.188.40.188 -EBE718E1A49EE229071702964F8DB1F318075FF8.or_port 80 -EBE718E1A49EE229071702964F8DB1F318075FF8.dir_port 443 -EBE718E1A49EE229071702964F8DB1F318075FF8.nickname fluxe4 -CE75BF0972ADD52AF8807602374E495C815DB304.address 81.7.14.246 -CE75BF0972ADD52AF8807602374E495C815DB304.or_port 443 -CE75BF0972ADD52AF8807602374E495C815DB304.dir_port 80 -CE75BF0972ADD52AF8807602374E495C815DB304.nickname pixelminer -4B9E2C56FB42B891794FE2CD2FCAD08A320CC3BB.address 88.198.38.226 -4B9E2C56FB42B891794FE2CD2FCAD08A320CC3BB.or_port 443 -4B9E2C56FB42B891794FE2CD2FCAD08A320CC3BB.dir_port 22 -4B9E2C56FB42B891794FE2CD2FCAD08A320CC3BB.nickname Nurnberg04 -F6279A203C1950ACF592322A235647A05BFBCF91.address 85.10.201.38 -F6279A203C1950ACF592322A235647A05BFBCF91.or_port 443 -F6279A203C1950ACF592322A235647A05BFBCF91.dir_port 22 -F6279A203C1950ACF592322A235647A05BFBCF91.nickname Nurnberg03 -2CE96A8A1DA032664C90F574AFFBECE18A6E8DFC.address 178.254.20.134 -2CE96A8A1DA032664C90F574AFFBECE18A6E8DFC.or_port 9001 -2CE96A8A1DA032664C90F574AFFBECE18A6E8DFC.dir_port 9030 -2CE96A8A1DA032664C90F574AFFBECE18A6E8DFC.nickname Doedel24 -E65D300F11E1DB12C534B0146BDAB6972F1A8A48.address 167.114.35.28 -E65D300F11E1DB12C534B0146BDAB6972F1A8A48.or_port 9001 -E65D300F11E1DB12C534B0146BDAB6972F1A8A48.dir_port 9030 -E65D300F11E1DB12C534B0146BDAB6972F1A8A48.nickname horizons +AD253B49E303C6AB1E048B014392AC569E8A7DAE.address 163.172.131.88 +AD253B49E303C6AB1E048B014392AC569E8A7DAE.or_port 443 +AD253B49E303C6AB1E048B014392AC569E8A7DAE.dir_port 80 +ACD889D86E02EDDAB1AFD81F598C0936238DC6D0.address 86.59.119.88 +ACD889D86E02EDDAB1AFD81F598C0936238DC6D0.or_port 443 +ACD889D86E02EDDAB1AFD81F598C0936238DC6D0.dir_port 80 +FCB6695F8F2DC240E974510A4B3A0F2B12AB5B64.address 195.154.8.111 +FCB6695F8F2DC240E974510A4B3A0F2B12AB5B64.or_port 443 +FCB6695F8F2DC240E974510A4B3A0F2B12AB5B64.dir_port 80 +20462CBA5DA4C2D963567D17D0B7249718114A68.address 212.47.229.2 +20462CBA5DA4C2D963567D17D0B7249718114A68.or_port 9001 +20462CBA5DA4C2D963567D17D0B7249718114A68.dir_port 9030 +5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33.address 5.175.233.86 +5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33.or_port 443 +5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33.dir_port 80 +5C4DF16A0029CC4F67D3E127356E68F219269859.address 185.100.85.138 +5C4DF16A0029CC4F67D3E127356E68F219269859.or_port 46356 +5C4DF16A0029CC4F67D3E127356E68F219269859.dir_port 80 +B1726B94885CE3AC3910CA8B60622B97B98E2529.address 185.66.250.141 +B1726B94885CE3AC3910CA8B60622B97B98E2529.or_port 9001 +B1726B94885CE3AC3910CA8B60622B97B98E2529.dir_port 9030 +B83DC1558F0D34353BB992EF93AFEAFDB226A73E.address 193.11.114.46 +B83DC1558F0D34353BB992EF93AFEAFDB226A73E.or_port 9003 +B83DC1558F0D34353BB992EF93AFEAFDB226A73E.dir_port 9032 +1DBAED235E3957DE1ABD25B4206BE71406FB61F8.address 46.101.151.222 +1DBAED235E3957DE1ABD25B4206BE71406FB61F8.or_port 443 +1DBAED235E3957DE1ABD25B4206BE71406FB61F8.dir_port 80 +5A5E03355C1908EBF424CAF1F3ED70782C0D2F74.address 78.142.142.246 +5A5E03355C1908EBF424CAF1F3ED70782C0D2F74.or_port 443 +5A5E03355C1908EBF424CAF1F3ED70782C0D2F74.dir_port 80 +ABCB4965F1FEE193602B50A365425105C889D3F8.address 192.34.63.137 +ABCB4965F1FEE193602B50A365425105C889D3F8.or_port 443 +ABCB4965F1FEE193602B50A365425105C889D3F8.dir_port 9030 +5CF8AFA5E4B0BB88942A44A3F3AAE08C3BDFD60B.address 178.16.208.62 +5CF8AFA5E4B0BB88942A44A3F3AAE08C3BDFD60B.or_port 443 +5CF8AFA5E4B0BB88942A44A3F3AAE08C3BDFD60B.dir_port 80 +1ECD73B936CB6E6B3CD647CC204F108D9DF2C9F7.address 91.219.237.229 +1ECD73B936CB6E6B3CD647CC204F108D9DF2C9F7.or_port 443 +1ECD73B936CB6E6B3CD647CC204F108D9DF2C9F7.dir_port 80 +361D33C96D0F161275EE67E2C91EE10B276E778B.address 217.79.190.25 +361D33C96D0F161275EE67E2C91EE10B276E778B.or_port 9090 +361D33C96D0F161275EE67E2C91EE10B276E778B.dir_port 9030 diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index 08183ef..2a40d2d 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -709,21 +709,19 @@ class Directory(object): :var int or_port: port on which the relay services relay traffic :var int dir_port: port on which directory information is available :var str fingerprint: relay fingerprint - :var str nickname: nickname of the authority """ - def __init__(self, address, or_port, dir_port, fingerprint, nickname): + def __init__(self, address, or_port, dir_port, fingerprint): self.address = address self.or_port = or_port self.dir_port = dir_port self.fingerprint = fingerprint - self.nickname = nickname def __eq__(self, other): if not isinstance(other, Directory): return False - for attr in ('nickname', 'address', 'or_port', 'dir_port', 'fingerprint'): + for attr in ('address', 'or_port', 'dir_port', 'fingerprint'): if getattr(self, attr) != getattr(other, attr): return False @@ -761,22 +759,30 @@ class DirectoryAuthority(Directory): .. versionchanged:: 1.3.0 Added the is_bandwidth_authority attribute. + :var str nickname: nickname of the authority :var str v3ident: identity key fingerprint used to sign votes and consensus :var bool is_bandwidth_authority: **True** if this is a bandwidth authority, **False** otherwise """ def __init__(self, address = None, or_port = None, dir_port = None, fingerprint = None, nickname = None, v3ident = None, is_bandwidth_authority = False): - super(DirectoryAuthority, self).__init__(address, or_port, dir_port, fingerprint, nickname) + super(DirectoryAuthority, self).__init__(address, or_port, dir_port, fingerprint) + self.nickname = nickname self.v3ident = v3ident self.is_bandwidth_authority = is_bandwidth_authority def __eq__(self, other): - if isinstance(other, DirectoryAuthority) and super(DirectoryAuthority, self).__eq__(other): - return self.v3ident == other.v3ident and self.is_bandwidth_authority == other.is_bandwidth_authority - else: + if not isinstance(other, DirectoryAuthority): + return False + elif not super(DirectoryAuthority, self).__eq__(other): return False + for attr in ('nickname', 'v3ident', 'is_bandwidth_authority'): + if getattr(self, attr) != getattr(other, attr): + return False + + return True + DIRECTORY_AUTHORITIES = { 'moria1': DirectoryAuthority( @@ -887,7 +893,8 @@ def get_authorities(): class FallbackDirectory(Directory): """ Particularly stable relays tor can instead of authorities when - bootstrapping. These relays are `hardcoded in tor <https://gitweb.torproject.org/tor.git/tree/src/or/fallback_dirs.inc>`_. + bootstrapping. These relays are `hardcoded in tor + <https://gitweb.torproject.org/tor.git/tree/src/or/fallback_dirs.inc>`_. For example, the following checks the performance of tor's fallback directories... @@ -901,23 +908,21 @@ class FallbackDirectory(Directory): for fallback_directory in FallbackDirectory.from_cache().values(): start = time.time() downloader.get_consensus(endpoints = [(fallback_directory.address, fallback_directory.dir_port)]).run() - print('Downloading the consensus took %0.2f from %s' % (time.time() - start, fallback_directory.nickname)) + print('Downloading the consensus took %0.2f from %s' % (time.time() - start, fallback_directory.fingerprint)) :: % python example.py - Downloading the consensus took 5.07 from Doedel22 - Downloading the consensus took 3.59 from tornoderdednl - Downloading the consensus took 4.16 from Logforme - Downloading the consensus took 6.76 from Doedel21 - Downloading the consensus took 5.21 from kitten4 + Downloading the consensus took 5.07 from 0AD3FA884D18F89EEA2D89C019379E0E7FD94417 + Downloading the consensus took 3.59 from C871C91489886D5E2E94C13EA1A5FDC4B6DC5204 + Downloading the consensus took 4.16 from 74A910646BCEEFBCD2E874FC1DC997430F968145 ... .. versionadded:: 1.5.0 """ - def __init__(self, address = None, or_port = None, dir_port = None, fingerprint = None, nickname = None): - super(FallbackDirectory, self).__init__(address, or_port, dir_port, fingerprint, nickname) + def __init__(self, address = None, or_port = None, dir_port = None, fingerprint = None): + super(FallbackDirectory, self).__init__(address, or_port, dir_port, fingerprint) @staticmethod def from_cache(): @@ -943,7 +948,7 @@ class FallbackDirectory(Directory): attr = {} - for attr_name in ('address', 'or_port', 'dir_port', 'nickname'): + for attr_name in ('address', 'or_port', 'dir_port'): key = '%s.%s' % (fingerprint, attr_name) attr[attr_name] = conf.get(key) @@ -956,15 +961,12 @@ class FallbackDirectory(Directory): raise IOError("'%s.or_port' was an invalid port (%s)" % (fingerprint, attr['or_port'])) elif not connection.is_valid_port(attr['dir_port']): raise IOError("'%s.dir_port' was an invalid port (%s)" % (fingerprint, attr['dir_port'])) - elif not tor_tools.is_valid_nickname(attr['nickname']): - raise IOError("'%s.nickname' was an invalid nickname (%s)" % (fingerprint, attr['nickname'])) results[fingerprint] = FallbackDirectory( address = attr['address'], or_port = int(attr['or_port']), dir_port = int(attr['dir_port']), fingerprint = fingerprint, - nickname = attr['nickname'], ) return results @@ -1001,46 +1003,34 @@ class FallbackDirectory(Directory): # Example of an entry... # - # /* - # wagner - # Flags: Fast Guard Running Stable V2Dir Valid - # Fallback Weight: 43680 / 491920 (8.879%) - # Consensus Weight: 62600 / 546000 (11.465%) - # Rarely used email <trff914 AT gmail DOT com> - # */ # "5.175.233.86:80 orport=443 id=5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33" # " weight=43680", - results, nickname, last_line = {}, None, None + results = {} for line in fallback_dir_page.splitlines(): - if last_line == '/*': - nickname = line - elif line.startswith('"'): + if line.startswith('"'): addr_line_match = re.match('"([\d\.]+):(\d+) orport=(\d+) id=([\dA-F]{40}).*', line) if addr_line_match: address, dir_port, or_port, fingerprint = addr_line_match.groups() if not connection.is_valid_ipv4_address(address): - raise IOError('%s has an invalid address: %s' % (nickname, address)) + raise IOError('%s has an invalid address: %s' % (fingerprint, address)) elif not connection.is_valid_port(or_port): - raise IOError('%s has an invalid or_port: %s' % (nickname, or_port)) + raise IOError('%s has an invalid or_port: %s' % (fingerprint, or_port)) elif not connection.is_valid_port(dir_port): - raise IOError('%s has an invalid dir_port: %s' % (nickname, dir_port)) + raise IOError('%s has an invalid dir_port: %s' % (fingerprint, dir_port)) elif not tor_tools.is_valid_fingerprint(fingerprint): - raise IOError('%s has an invalid fingerprint: %s' % (nickname, fingerprint)) + raise IOError('%s has an invalid fingerprint: %s' % (fingerprint, fingerprint)) results[fingerprint] = FallbackDirectory( address = address, or_port = int(or_port), dir_port = int(dir_port), fingerprint = fingerprint, - nickname = nickname, ) - last_line = line - return results @@ -1058,16 +1048,15 @@ def _fallback_directory_differences(previous_directories, new_directories): directory = new_directories[fp] lines += [ - '* Added %s as a new fallback directory:' % directory.nickname, + '* Added %s as a new fallback directory:' % directory.fingerprint, ' address: %s' % directory.address, ' or_port: %s' % directory.or_port, ' dir_port: %s' % directory.dir_port, - ' fingerprint: %s' % directory.fingerprint, '', ] for fp in removed_fp: - lines.append('* Removed %s as a fallback directory' % previous_directories[fp].nickname) + lines.append('* Removed %s as a fallback directory' % fp) for fp in new_directories: if fp in added_fp or fp in removed_fp: @@ -1077,7 +1066,7 @@ def _fallback_directory_differences(previous_directories, new_directories): new_directory = new_directories[fp] if previous_directory != new_directory: - for attr in ('nickname', 'address', 'or_port', 'dir_port', 'fingerprint'): + for attr in ('address', 'or_port', 'dir_port', 'fingerprint'): old_attr = getattr(previous_directory, attr) new_attr = getattr(new_directory, attr) diff --git a/test/unit/descriptor/remote.py b/test/unit/descriptor/remote.py index f0e0dea..7309b57 100644 --- a/test/unit/descriptor/remote.py +++ b/test/unit/descriptor/remote.py @@ -63,22 +63,8 @@ FALLBACK_DIR_CONTENT = b"""\ * This works around an issue where relays post a descriptor without a DirPort * when restarted. If these relays stay up, they will have been up for 120 days * by the 0.2.8 stable release -- teor */ -/* -wagner -Flags: Fast Guard Running Stable V2Dir Valid -Fallback Weight: 43680 / 491920 (8.879%) -Consensus Weight: 62600 / 546000 (11.465%) -Rarely used email <trff914 AT gmail DOT com> -*/ "5.175.233.86:80 orport=443 id=5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33" " weight=43680", -/* -kitten2 -Flags: Fast Guard HSDir Running Stable V2Dir Valid -Fallback Weight: 43680 / 491920 (8.879%) -Consensus Weight: 59100 / 546000 (10.824%) -0xEFB74277ECE4E222 Aeris <aeris+tor AT imirhil DOT fr> - 1aerisnnLWPchhDSXpxWGYWwLiSFUVFnd -*/ "62.210.124.124:9130 orport=9101 id=2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E" " ipv6=[2001:bc8:3f23:100::1]:9101" " weight=43680", @@ -185,7 +171,7 @@ class TestDescriptorDownloader(unittest.TestCase): # quick sanity test that we can load cached content fallback_directories = stem.descriptor.remote.FallbackDirectory.from_cache() self.assertTrue(len(fallback_directories) > 10) - self.assertEqual('wagner', fallback_directories['5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33'].nickname) + self.assertEqual('5.175.233.86', fallback_directories['5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33'].address) @patch(URL_OPEN) def test_fallback_directories_from_remote(self, urlopen_mock): @@ -194,14 +180,12 @@ class TestDescriptorDownloader(unittest.TestCase): expected = { '5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33': stem.descriptor.remote.FallbackDirectory( - nickname = 'wagner', address = '5.175.233.86', or_port = 443, dir_port = 80, fingerprint = '5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33', ), '2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E': stem.descriptor.remote.FallbackDirectory( - nickname = 'kitten2', address = '62.210.124.124', or_port = 9101, dir_port = 9130,

1 0