tor-commits June 2017

tor-commits@lists.torproject.org

13 participants
1918 discussions

[stem/master] Add a make_router_status_entry method
by atagar＠torproject.org 30 Jun '17

30 Jun '17

commit 984497610ebc1efbf474b5dbd3fcc7e89c88fedd Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jun 30 11:06:51 2017 -0700 Add a make_router_status_entry method Little helper to make router status entries that are derived from a server descriptor. This presently populates just the base router status entry fields, not the optional v3 fields. Happy to have it expanded if folks need that though. --- stem/descriptor/server_descriptor.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 643831b..edd1612 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -21,6 +21,7 @@ etc). This information is provided from a few sources... ServerDescriptor - Tor server descriptor. |- RelayDescriptor - Server descriptor for a relay. + | +- make_router_status_entry - Creates a router status entry for this descriptor. | |- BridgeDescriptor - Scrubbed server descriptor for a bridge. | |- is_scrubbed - checks if our content has been properly scrubbed @@ -46,6 +47,7 @@ import stem.util.str_tools import stem.util.tor_tools import stem.version +from stem.descriptor.router_status_entry import RouterStatusEntryV3 from stem.util import str_type from stem.descriptor import ( @@ -855,6 +857,33 @@ class RelayDescriptor(ServerDescriptor): return self._digest_for_content(b'router ', b'\nrouter-signature\n') + def make_router_status_entry(self): + """ + Provides a RouterStatusEntryV3 for this descriptor content. + + .. versionadded:: 1.6.0 + + :returns: :class:`~stem.descriptor.router_status_entry.RouterStatusEntryV3` + that would be in the consensus + """ + + attr = { + 'r': ' '.join([ + self.nickname, + base64.b64encode(binascii.unhexlify(self.fingerprint)).rstrip('='), + base64.b64encode(binascii.unhexlify(self.digest())).rstrip('='), + self.published.strftime('%Y-%m-%d %H:%M:%S'), + self.address, + str(self.or_port), + str(self.dir_port) if self.dir_port else '0', + ]), + } + + if self.tor_version: + attr['v'] = self.tor_version + + return RouterStatusEntryV3.create(attr) + @lru_cache() def onion_key_crosscert_digest(self): """

1 0

[stem/master] Revert "Validate signed descriptors when testing"
by atagar＠torproject.org 30 Jun '17

30 Jun '17

commit e6c5b92ea32e82ae0c4d4e1a5504f170c35ff12f Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jun 30 10:12:19 2017 -0700 Revert "Validate signed descriptors when testing" Ok, I'm dumb. This is the default. This reverts commit 0f35be160fe11e082ee9482a774136aa72fd3477. --- test/unit/descriptor/extrainfo_descriptor.py | 2 +- test/unit/descriptor/server_descriptor.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unit/descriptor/extrainfo_descriptor.py b/test/unit/descriptor/extrainfo_descriptor.py index 822f6d1..1f91e72 100644 --- a/test/unit/descriptor/extrainfo_descriptor.py +++ b/test/unit/descriptor/extrainfo_descriptor.py @@ -136,7 +136,7 @@ k0d2aofcVbHr4fPQOSST0LXDrhFl5Fqo5um296zpJGvRUeO6S44U/EfJAGShtqWw @test.require.cryptography def test_descriptor_signing(self): - RelayExtraInfoDescriptor.create(sign = True, validate = True) + RelayExtraInfoDescriptor.create(sign = True) self.assertRaisesRegexp(NotImplementedError, 'Signing of BridgeExtraInfoDescriptor not implemented', BridgeExtraInfoDescriptor.create, sign = True) def test_multiple_metrics_bridge_descriptors(self): diff --git a/test/unit/descriptor/server_descriptor.py b/test/unit/descriptor/server_descriptor.py index b015474..8578726 100644 --- a/test/unit/descriptor/server_descriptor.py +++ b/test/unit/descriptor/server_descriptor.py @@ -257,7 +257,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= @test.require.cryptography def test_descriptor_signing(self): - RelayDescriptor.create(sign = True, validate = True) + RelayDescriptor.create(sign = True) self.assertRaisesRegexp(NotImplementedError, 'Signing of BridgeDescriptor not implemented', BridgeDescriptor.create, sign = True) @patch('time.time', Mock(return_value = time.mktime(datetime.date(2010, 1, 1).timetuple())))

1 0

[stem/master] Populate fingerprints for created server descriptors
by atagar＠torproject.org 30 Jun '17

30 Jun '17

commit 02eb36058f91f88952378df76bc8584fa8a95ee0 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jun 30 10:25:57 2017 -0700 Populate fingerprints for created server descriptors Not technically a mandatory field but lets be honest, if you provide a signing key you want this. ;P --- stem/descriptor/server_descriptor.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 42cbca6..643831b 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -826,6 +826,10 @@ class RelayDescriptor(ServerDescriptor): if signing_key is None: signing_key = create_signing_key() + if 'fingerprint' not in attr: + fingerprint = hashlib.sha1(_bytes_for_block(signing_key.public_digest.strip())).hexdigest().upper() + attr['fingerprint'] = ' '.join(stem.util.str_tools._split_by_length(fingerprint, 4)) + attr['signing-key'] = signing_key.public_digest content = _descriptor_content(attr, exclude, sign, base_header) + b'\nrouter-signature\n'

1 0

[stem/master] Cite metrics-lib's shiny new site
by atagar＠torproject.org 30 Jun '17

30 Jun '17

commit ee8921e348828228d0cc0364e9d7433dfd7b9885 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jun 30 09:38:40 2017 -0700 Cite metrics-lib's shiny new site --- docs/tutorials/mirror_mirror_on_the_wall.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/mirror_mirror_on_the_wall.rst b/docs/tutorials/mirror_mirror_on_the_wall.rst index d215722..8ff63f5 100644 --- a/docs/tutorials/mirror_mirror_on_the_wall.rst +++ b/docs/tutorials/mirror_mirror_on_the_wall.rst @@ -307,7 +307,7 @@ Are there any other parsing libraries? -------------------------------------- Yup! Stem isn't the only game in town when it comes to parsing. `Metrics-lib -<https://gitweb.torproject.org/metrics-lib.git/>`_ is a highly mature parsing +<https://metrics.torproject.org/metrics-lib.html>`_ is a highly mature parsing library for Java, and `Zoossh <https://gitweb.torproject.org/user/phw/zoossh.git/>`_ is available for Go. Each library has its own capabilities...

1 0

[stem/master] Validate signed descriptors when testing
by atagar＠torproject.org 30 Jun '17

30 Jun '17

commit 0f35be160fe11e082ee9482a774136aa72fd3477 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jun 30 09:35:23 2017 -0700 Validate signed descriptors when testing Well, this is dumb. We sign the descriptors but didn't run validation checks. Oh well, trivial to fix. --- test/unit/descriptor/extrainfo_descriptor.py | 2 +- test/unit/descriptor/server_descriptor.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unit/descriptor/extrainfo_descriptor.py b/test/unit/descriptor/extrainfo_descriptor.py index 1f91e72..822f6d1 100644 --- a/test/unit/descriptor/extrainfo_descriptor.py +++ b/test/unit/descriptor/extrainfo_descriptor.py @@ -136,7 +136,7 @@ k0d2aofcVbHr4fPQOSST0LXDrhFl5Fqo5um296zpJGvRUeO6S44U/EfJAGShtqWw @test.require.cryptography def test_descriptor_signing(self): - RelayExtraInfoDescriptor.create(sign = True) + RelayExtraInfoDescriptor.create(sign = True, validate = True) self.assertRaisesRegexp(NotImplementedError, 'Signing of BridgeExtraInfoDescriptor not implemented', BridgeExtraInfoDescriptor.create, sign = True) def test_multiple_metrics_bridge_descriptors(self): diff --git a/test/unit/descriptor/server_descriptor.py b/test/unit/descriptor/server_descriptor.py index 8578726..b015474 100644 --- a/test/unit/descriptor/server_descriptor.py +++ b/test/unit/descriptor/server_descriptor.py @@ -257,7 +257,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= @test.require.cryptography def test_descriptor_signing(self): - RelayDescriptor.create(sign = True) + RelayDescriptor.create(sign = True, validate = True) self.assertRaisesRegexp(NotImplementedError, 'Signing of BridgeDescriptor not implemented', BridgeDescriptor.create, sign = True) @patch('time.time', Mock(return_value = time.mktime(datetime.date(2010, 1, 1).timetuple())))

1 0

[stem/master] Providing a signing key should imply we're signing
by atagar＠torproject.org 30 Jun '17

30 Jun '17

commit d1f7e4fe872e04cbec27b59f97c1fd240f288b87 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jun 30 09:30:58 2017 -0700 Providing a signing key should imply we're signing When the caller explicitly provides a signing key we should infer 'sign' to be true. Otherwise callers need to call... RelayServerDescriptor.create(sign = True, signing_key = key) ... and that's stupid. Callers that provide a signing key *always* want the descriptor to be signed. Otherwise... well, their key won't be used. :P --- stem/descriptor/extrainfo_descriptor.py | 3 +++ stem/descriptor/server_descriptor.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 27a75db..3a4a0f8 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -976,6 +976,9 @@ class RelayExtraInfoDescriptor(ExtraInfoDescriptor): @classmethod def content(cls, attr = None, exclude = (), sign = False, signing_key = None): + if signing_key: + sign = True + if sign: if attr and 'router-signature' in attr: raise ValueError('Cannot sign the descriptor if a router-signature has been provided') diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 1a1687f..42cbca6 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -802,6 +802,9 @@ class RelayDescriptor(ServerDescriptor): @classmethod def content(cls, attr = None, exclude = (), sign = False, signing_key = None): + if signing_key: + sign = True + if attr is None: attr = {}

1 0

[stem/master] Make create_signing_key() a public method
by atagar＠torproject.org 30 Jun '17

30 Jun '17

commit 9c1bad1b9c7e51a5d0e449573cb57531cc81edd3 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jun 30 09:26:12 2017 -0700 Make create_signing_key() a public method While swapping BridgeDB to use us for descriptor creation realized it would be handy for this method to be public. --- stem/descriptor/__init__.py | 5 ++++- stem/descriptor/extrainfo_descriptor.py | 12 +++++++----- stem/descriptor/server_descriptor.py | 12 +++++++----- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 5f8d1ca..ac9367b 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -10,6 +10,7 @@ Package for parsing and processing descriptor data. parse_file - Parses the descriptors in a file. create - Creates a new custom descriptor. + create_signing_key - Cretes a signing key that can be used for creating descriptors. Descriptor - Common parent for all descriptor file types. |- get_path - location of the descriptor on disk if it came from a file @@ -967,11 +968,13 @@ def _get_pseudo_pgp_block(remaining_contents): return None -def _signing_key(private_key = None): +def create_signing_key(private_key = None): """ Serializes a signing key if we have one. Otherwise this creates a new signing key we can use to create descriptors. + .. versionadded:: 1.6.0 + :param cryptography.hazmat.backends.openssl.rsa._RSAPrivateKey private_key: private key :returns: :class:`~stem.descriptor.__init__.SigningKey` that can be used to diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 3b4e9a4..27a75db 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -80,6 +80,7 @@ from stem.descriptor import ( CRYPTO_BLOB, PGP_BLOCK_END, Descriptor, + create_signing_key, _descriptor_content, _read_until_keywords, _descriptor_components, @@ -89,7 +90,6 @@ from stem.descriptor import ( _parse_timestamp_line, _parse_forty_character_hex, _parse_key_block, - _signing_key, _append_router_signature, ) @@ -975,20 +975,22 @@ class RelayExtraInfoDescriptor(ExtraInfoDescriptor): }) @classmethod - def content(cls, attr = None, exclude = (), sign = False, private_signing_key = None): + def content(cls, attr = None, exclude = (), sign = False, signing_key = None): if sign: if attr and 'router-signature' in attr: raise ValueError('Cannot sign the descriptor if a router-signature has been provided') - signing_key = _signing_key(private_signing_key) + if signing_key is None: + signing_key = create_signing_key() + content = _descriptor_content(attr, exclude, sign, RELAY_EXTRAINFO_HEADER) + b'\nrouter-signature\n' return _append_router_signature(content, signing_key.private) else: return _descriptor_content(attr, exclude, sign, RELAY_EXTRAINFO_HEADER, RELAY_EXTRAINFO_FOOTER) @classmethod - def create(cls, attr = None, exclude = (), validate = True, sign = False, private_signing_key = None): - return cls(cls.content(attr, exclude, sign, private_signing_key), validate = validate) + def create(cls, attr = None, exclude = (), validate = True, sign = False, signing_key = None): + return cls(cls.content(attr, exclude, sign, signing_key), validate = validate) @lru_cache() def digest(self): diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 5826bb6..1a1687f 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -51,6 +51,7 @@ from stem.util import str_type from stem.descriptor import ( PGP_BLOCK_END, Descriptor, + create_signing_key, _descriptor_content, _descriptor_components, _read_until_keywords, @@ -64,7 +65,6 @@ from stem.descriptor import ( _parse_forty_character_hex, _parse_protocol_line, _parse_key_block, - _signing_key, _append_router_signature, _random_ipv4_address, _random_date, @@ -801,7 +801,7 @@ class RelayDescriptor(ServerDescriptor): self.certificate.validate(self) @classmethod - def content(cls, attr = None, exclude = (), sign = False, private_signing_key = None): + def content(cls, attr = None, exclude = (), sign = False, signing_key = None): if attr is None: attr = {} @@ -820,7 +820,9 @@ class RelayDescriptor(ServerDescriptor): elif attr and 'router-signature' in attr: raise ValueError('Cannot sign the descriptor if a router-signature has been provided') - signing_key = _signing_key(private_signing_key) + if signing_key is None: + signing_key = create_signing_key() + attr['signing-key'] = signing_key.public_digest content = _descriptor_content(attr, exclude, sign, base_header) + b'\nrouter-signature\n' @@ -831,8 +833,8 @@ class RelayDescriptor(ServerDescriptor): )) @classmethod - def create(cls, attr = None, exclude = (), validate = True, sign = False, private_signing_key = None): - return cls(cls.content(attr, exclude, sign, private_signing_key), validate = validate, skip_crypto_validation = not sign) + def create(cls, attr = None, exclude = (), validate = True, sign = False, signing_key = None): + return cls(cls.content(attr, exclude, sign, signing_key), validate = validate, skip_crypto_validation = not sign) @lru_cache() def digest(self):

1 0

[metrics-web/master] Fix a regression in the legacy module.
by karsten＠torproject.org 30 Jun '17

30 Jun '17

commit 96f5fbe520e052ac62f2f99c2bbce7e92e03ac0f Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Jun 29 21:30:42 2017 +0200 Fix a regression in the legacy module. --- .../main/java/org/torproject/ernie/cron/Configuration.java | 12 ++++++------ .../ernie/cron/RelayDescriptorDatabaseImporter.java | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/legacy/src/main/java/org/torproject/ernie/cron/Configuration.java b/modules/legacy/src/main/java/org/torproject/ernie/cron/Configuration.java index e0d753f..12409cd 100644 --- a/modules/legacy/src/main/java/org/torproject/ernie/cron/Configuration.java +++ b/modules/legacy/src/main/java/org/torproject/ernie/cron/Configuration.java @@ -23,7 +23,7 @@ public class Configuration { private boolean importDirectoryArchives = false; - private List<String> directoryArchivesDirectories = new ArrayList<>(); + private List<File> directoryArchivesDirectories = new ArrayList<>(); private boolean keepDirectoryArchiveImportHistory = false; @@ -75,7 +75,7 @@ public class Configuration { this.importDirectoryArchives = Integer.parseInt( line.split(" ")[1]) != 0; } else if (line.startsWith("DirectoryArchivesDirectory")) { - this.directoryArchivesDirectories.add(line.split(" ")[1]); + this.directoryArchivesDirectories.add(new File(line.split(" ")[1])); } else if (line.startsWith("KeepDirectoryArchiveImportHistory")) { this.keepDirectoryArchiveImportHistory = Integer.parseInt( line.split(" ")[1]) != 0; @@ -140,12 +140,12 @@ public class Configuration { } /** Returns directories containing archived descriptors. */ - public List<String> getDirectoryArchivesDirectories() { + public List<File> getDirectoryArchivesDirectories() { if (this.directoryArchivesDirectories.isEmpty()) { String prefix = "../../shared/in/recent/relay-descriptors/"; - return Arrays.asList( - (prefix + "consensuses/," + prefix + "server-descriptors/," - + prefix + "extra-infos/").split(",")); + return Arrays.asList(new File(prefix + "consensuses/"), + new File(prefix + "server-descriptors/"), + new File(prefix + "extra-infos/")); } else { return this.directoryArchivesDirectories; } diff --git a/modules/legacy/src/main/java/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java b/modules/legacy/src/main/java/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java index 82fb70f..0e69389 100644 --- a/modules/legacy/src/main/java/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java +++ b/modules/legacy/src/main/java/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java @@ -181,7 +181,7 @@ public final class RelayDescriptorDatabaseImporter { private boolean writeRawImportFiles; - private List<String> archivesDirectories; + private List<File> archivesDirectories; private File statsDirectory; @@ -192,7 +192,7 @@ public final class RelayDescriptorDatabaseImporter { * preparing statements. */ public RelayDescriptorDatabaseImporter(String connectionUrl, - String rawFilesDirectory, List<String> archivesDirectories, + String rawFilesDirectory, List<File> archivesDirectories, File statsDirectory, boolean keepImportHistory) { if (archivesDirectories == null || statsDirectory == null) {

1 0

[metrics-web/master] Remove code to produce the deprecated torperf.csv.
by karsten＠torproject.org 30 Jun '17

30 Jun '17

commit ace813557e36a24635dfe28847dea631147487e0 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jun 30 10:11:16 2017 +0200 Remove code to produce the deprecated torperf.csv. Implements #22780. --- modules/legacy/config.template | 6 - .../org/torproject/ernie/cron/Configuration.java | 17 -- .../main/java/org/torproject/ernie/cron/Main.java | 7 - .../ernie/cron/performance/TorperfProcessor.java | 296 --------------------- website/src/main/resources/web/WEB-INF/stats.jsp | 2 +- 5 files changed, 1 insertion(+), 327 deletions(-) diff --git a/modules/legacy/config.template b/modules/legacy/config.template index 743fe1e..afa8f2d 100644 --- a/modules/legacy/config.template +++ b/modules/legacy/config.template @@ -41,9 +41,3 @@ ## Write bridge stats to disk #WriteBridgeStats 0 # -## Import torperf data, if available, and write stats to disk -#ImportWriteTorperfStats 0 -# -## Relative path to directory to import torperf results from -#TorperfDirectory /srv/metrics.torproject.org/metrics/shared/in/recent/torperf/ -# diff --git a/modules/legacy/src/main/java/org/torproject/ernie/cron/Configuration.java b/modules/legacy/src/main/java/org/torproject/ernie/cron/Configuration.java index 12409cd..35d52b9 100644 --- a/modules/legacy/src/main/java/org/torproject/ernie/cron/Configuration.java +++ b/modules/legacy/src/main/java/org/torproject/ernie/cron/Configuration.java @@ -44,10 +44,6 @@ public class Configuration { private boolean writeBridgeStats = false; - private boolean importWriteTorperfStats = false; - - private String torperfDirectory = "in/torperf/"; - private String exoneraTorDatabaseJdbc = "jdbc:postgresql:" + "//localhost/exonerator?user=metrics&password=password"; @@ -100,11 +96,6 @@ public class Configuration { } else if (line.startsWith("WriteBridgeStats")) { this.writeBridgeStats = Integer.parseInt( line.split(" ")[1]) != 0; - } else if (line.startsWith("ImportWriteTorperfStats")) { - this.importWriteTorperfStats = Integer.parseInt( - line.split(" ")[1]) != 0; - } else if (line.startsWith("TorperfDirectory")) { - this.torperfDirectory = line.split(" ")[1]; } else if (line.startsWith("ExoneraTorDatabaseJdbc")) { this.exoneraTorDatabaseJdbc = line.split(" ")[1]; } else if (line.startsWith("ExoneraTorImportDirectory")) { @@ -187,14 +178,6 @@ public class Configuration { return this.writeBridgeStats; } - public boolean getImportWriteTorperfStats() { - return this.importWriteTorperfStats; - } - - public String getTorperfDirectory() { - return this.torperfDirectory; - } - public String getExoneraTorDatabaseJdbc() { return this.exoneraTorDatabaseJdbc; } diff --git a/modules/legacy/src/main/java/org/torproject/ernie/cron/Main.java b/modules/legacy/src/main/java/org/torproject/ernie/cron/Main.java index 0eab86f..180031a 100644 --- a/modules/legacy/src/main/java/org/torproject/ernie/cron/Main.java +++ b/modules/legacy/src/main/java/org/torproject/ernie/cron/Main.java @@ -4,7 +4,6 @@ package org.torproject.ernie.cron; import org.torproject.ernie.cron.network.ConsensusStatsFileHandler; -import org.torproject.ernie.cron.performance.TorperfProcessor; import java.io.File; import java.util.logging.Logger; @@ -75,12 +74,6 @@ public class Main { csfh = null; } - // Import and process torperf stats - if (config.getImportWriteTorperfStats()) { - new TorperfProcessor(new File(config.getTorperfDirectory()), - statsDirectory); - } - // Remove lock file lf.releaseLock(); diff --git a/modules/legacy/src/main/java/org/torproject/ernie/cron/performance/TorperfProcessor.java b/modules/legacy/src/main/java/org/torproject/ernie/cron/performance/TorperfProcessor.java deleted file mode 100644 index b5dae12..0000000 --- a/modules/legacy/src/main/java/org/torproject/ernie/cron/performance/TorperfProcessor.java +++ /dev/null @@ -1,296 +0,0 @@ -/* Copyright 2011--2017 The Tor Project - * See LICENSE for licensing information */ - -package org.torproject.ernie.cron.performance; - -import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.DescriptorReader; -import org.torproject.descriptor.DescriptorSourceFactory; -import org.torproject.descriptor.TorperfResult; -import org.torproject.descriptor.UnparseableDescriptor; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.SortedMap; -import java.util.TimeZone; -import java.util.TreeMap; -import java.util.logging.Level; -import java.util.logging.Logger; - -public class TorperfProcessor { - - /** Processes Torperf data from the given directory and writes - * aggregates statistics to the given stats directory. */ - public TorperfProcessor(File torperfDirectory, File statsDirectory) { - - if (torperfDirectory == null || statsDirectory == null) { - throw new IllegalArgumentException(); - } - - Logger logger = Logger.getLogger(TorperfProcessor.class.getName()); - File rawFile = new File(statsDirectory, "torperf-raw"); - File statsFile = new File(statsDirectory, "torperf.csv"); - SortedMap<String, String> rawObs = new TreeMap<>(); - SortedMap<String, String> stats = new TreeMap<>(); - int addedRawObs = 0; - SimpleDateFormat formatter = - new SimpleDateFormat("yyyy-MM-dd,HH:mm:ss"); - formatter.setTimeZone(TimeZone.getTimeZone("UTC")); - try { - if (rawFile.exists()) { - logger.fine("Reading file " + rawFile.getAbsolutePath() + "..."); - BufferedReader br = new BufferedReader(new FileReader(rawFile)); - String line = br.readLine(); // ignore header - while ((line = br.readLine()) != null) { - if (line.split(",").length != 4) { - logger.warning("Corrupt line in " + rawFile.getAbsolutePath() - + "!"); - break; - } - String key = line.substring(0, line.lastIndexOf(",")); - rawObs.put(key, line); - } - br.close(); - logger.fine("Finished reading file " + rawFile.getAbsolutePath() - + "."); - } - if (statsFile.exists()) { - logger.fine("Reading file " + statsFile.getAbsolutePath() - + "..."); - BufferedReader br = new BufferedReader(new FileReader(statsFile)); - String line = br.readLine(); // ignore header - while ((line = br.readLine()) != null) { - String[] parts = line.split(","); - String key = String.format("%s,%s,%s", parts[0], parts[1], - parts[2]); - stats.put(key, line); - } - br.close(); - logger.fine("Finished reading file " + statsFile.getAbsolutePath() - + "."); - } - if (torperfDirectory.exists()) { - logger.fine("Importing files in " + torperfDirectory + "/..."); - DescriptorReader descriptorReader = - DescriptorSourceFactory.createDescriptorReader(); - File historyFile = new File(statsDirectory, "torperf-history"); - descriptorReader.setHistoryFile(historyFile); - for (Descriptor descriptor : descriptorReader.readDescriptors( - torperfDirectory)) { - if (descriptor instanceof UnparseableDescriptor) { - logger.log(Level.FINE, "Error parsing descriptor.", - ((UnparseableDescriptor) descriptor) - .getDescriptorParseException()); - continue; - } else if (!(descriptor instanceof TorperfResult)) { - continue; - } - TorperfResult result = (TorperfResult) descriptor; - if (null != result.getUnrecognizedKeys() - && result.getUnrecognizedKeys().containsKey("ENDPOINTREMOTE") - && result.getUnrecognizedKeys().get("ENDPOINTREMOTE") - .contains(".onion")) { - continue; - } - String source = result.getSource(); - long fileSize = result.getFileSize(); - if (fileSize == 51200) { - source += "-50kb"; - } else if (fileSize == 1048576) { - source += "-1mb"; - } else if (fileSize == 5242880) { - source += "-5mb"; - } else { - logger.fine("Unexpected file size '" + fileSize - + "'. Skipping."); - continue; - } - String dateTime = formatter.format(result.getStartMillis()); - long completeMillis = result.getDataCompleteMillis() - - result.getStartMillis(); - String key = source + "," + dateTime; - String value = key; - if ((result.didTimeout() == null - && result.getDataCompleteMillis() < 1) - || (result.didTimeout() != null && result.didTimeout())) { - value += ",-2"; // -2 for timeout - } else if (result.getReadBytes() < fileSize) { - value += ",-1"; // -1 for failure - } else { - value += "," + completeMillis; - } - if (!rawObs.containsKey(key)) { - rawObs.put(key, value); - addedRawObs++; - } - } - descriptorReader.saveHistoryFile(historyFile); - logger.fine("Finished importing files in " + torperfDirectory - + "/."); - } - if (rawObs.size() > 0) { - logger.fine("Writing file " + rawFile.getAbsolutePath() + "..."); - rawFile.getParentFile().mkdirs(); - BufferedWriter bw = new BufferedWriter(new FileWriter(rawFile)); - bw.append("source,date,start,completemillis\n"); - String tempSourceDate = null; - Iterator<Map.Entry<String, String>> it = - rawObs.entrySet().iterator(); - List<Long> dlTimes = new ArrayList<>(); - boolean haveWrittenFinalLine = false; - SortedMap<String, List<Long>> dlTimesAllSources = new TreeMap<>(); - SortedMap<String, long[]> statusesAllSources = new TreeMap<>(); - long failures = 0; - long timeouts = 0; - long requests = 0; - while (it.hasNext() || !haveWrittenFinalLine) { - Map.Entry<String, String> next = - it.hasNext() ? it.next() : null; - if (tempSourceDate != null - && (next == null || !(next.getValue().split(",")[0] + "," - + next.getValue().split(",")[1]).equals(tempSourceDate))) { - if (dlTimes.size() > 4) { - Collections.sort(dlTimes); - long q1 = dlTimes.get(dlTimes.size() / 4 - 1); - long md = dlTimes.get(dlTimes.size() / 2 - 1); - long q3 = dlTimes.get(dlTimes.size() * 3 / 4 - 1); - String[] tempParts = tempSourceDate.split("[,]", 3); - String tempDate = tempParts[1]; - int tempSize = Integer.parseInt(tempParts[0].substring( - tempParts[0].lastIndexOf('-') + 1, - tempParts[0].length() - 2)) - * 1024 * (tempParts[0].endsWith("mb") ? 1024 : 1); - String tempSource = tempParts[0].substring(0, - tempParts[0].lastIndexOf('-')); - String tempDateSizeSource = String.format("%s,%d,%s", - tempDate, tempSize, tempSource); - stats.put(tempDateSizeSource, - String.format("%s,%s,%s,%s,%s,%s,%s", - tempDateSizeSource, q1, md, q3, timeouts, failures, - requests)); - String allDateSizeSource = String.format("%s,%d,", - tempDate, tempSize); - if (dlTimesAllSources.containsKey(allDateSizeSource)) { - dlTimesAllSources.get(allDateSizeSource).addAll(dlTimes); - } else { - dlTimesAllSources.put(allDateSizeSource, dlTimes); - } - if (statusesAllSources.containsKey(allDateSizeSource)) { - long[] status = statusesAllSources.get(allDateSizeSource); - status[0] += timeouts; - status[1] += failures; - status[2] += requests; - } else { - long[] status = new long[3]; - status[0] = timeouts; - status[1] = failures; - status[2] = requests; - statusesAllSources.put(allDateSizeSource, status); - } - } - dlTimes = new ArrayList<>(); - failures = timeouts = requests = 0; - if (next == null) { - haveWrittenFinalLine = true; - } - } - if (next != null) { - bw.append(next.getValue() + "\n"); - String[] parts = next.getValue().split(","); - tempSourceDate = parts[0] + "," + parts[1]; - long completeMillis = Long.parseLong(parts[3]); - if (completeMillis == -2L) { - timeouts++; - } else if (completeMillis == -1L) { - failures++; - } else { - dlTimes.add(Long.parseLong(parts[3])); - } - requests++; - } - } - bw.close(); - for (Map.Entry<String, List<Long>> e - : dlTimesAllSources.entrySet()) { - String allDateSizeSource = e.getKey(); - dlTimes = e.getValue(); - Collections.sort(dlTimes); - long[] status = statusesAllSources.get(allDateSizeSource); - timeouts = status[0]; - failures = status[1]; - requests = status[2]; - long q1 = dlTimes.get(dlTimes.size() / 4 - 1); - long md = dlTimes.get(dlTimes.size() / 2 - 1); - long q3 = dlTimes.get(dlTimes.size() * 3 / 4 - 1); - stats.put(allDateSizeSource, - String.format("%s,%s,%s,%s,%s,%s,%s", - allDateSizeSource, q1, md, q3, timeouts, failures, - requests)); - } - logger.fine("Finished writing file " + rawFile.getAbsolutePath() - + "."); - } - if (stats.size() > 0) { - logger.fine("Writing file " + statsFile.getAbsolutePath() - + "..."); - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - String yesterday = dateFormat.format(System.currentTimeMillis() - - 86400000L); - statsFile.getParentFile().mkdirs(); - BufferedWriter bw = new BufferedWriter(new FileWriter(statsFile)); - bw.append("date,size,source,q1,md,q3,timeouts,failures," - + "requests\n"); - for (String s : stats.values()) { - if (s.compareTo(yesterday) < 0) { - bw.append(s + "\n"); - } - } - bw.close(); - logger.fine("Finished writing file " + statsFile.getAbsolutePath() - + "."); - } - } catch (IOException e) { - logger.log(Level.WARNING, "Failed writing " - + rawFile.getAbsolutePath() + " or " - + statsFile.getAbsolutePath() + "!", e); - } - - /* Write stats. */ - StringBuilder dumpStats = new StringBuilder("Finished writing " - + "statistics on torperf results.\nAdded " + addedRawObs - + " new observations in this execution.\n" - + "Last known obserations by source and file size are:"); - String lastSource = null; - String lastLine = null; - for (String s : rawObs.keySet()) { - String[] parts = s.split(","); - if (lastSource == null) { - lastSource = parts[0]; - } else if (!parts[0].equals(lastSource)) { - String lastKnownObservation = lastLine.split(",")[1] + " " - + lastLine.split(",")[2]; - dumpStats.append("\n" + lastSource + " " + lastKnownObservation); - lastSource = parts[0]; - } - lastLine = s; - } - if (lastSource != null) { - String lastKnownObservation = lastLine.split(",")[1] + " " - + lastLine.split(",")[2]; - dumpStats.append("\n" + lastSource + " " + lastKnownObservation); - } - logger.info(dumpStats.toString()); - } -} - diff --git a/website/src/main/resources/web/WEB-INF/stats.jsp b/website/src/main/resources/web/WEB-INF/stats.jsp index 90b9c3b..2d5a69d 100644 --- a/website/src/main/resources/web/WEB-INF/stats.jsp +++ b/website/src/main/resources/web/WEB-INF/stats.jsp @@ -310,7 +310,7 @@ there are users by either of the two numbers.</li> <div class="bs-callout bs-callout-warning"> <h3>Deprecated</h3> -<p>As of May 12, 2017, the linked data file has been replaced by <a href="#torperf-1.1">this data file</a>. Starting on June 1, 2017, the linked data file will not be updated anymore. This section and the linked data file will be removed in the future.</p> +<p>As of May 12, 2017, the linked data file has been replaced by <a href="#torperf-1.1">this data file</a>. As of June 30, 2017, the linked data file is not updated anymore. This section and the linked data file will be removed in the future.</p> </div> <p>The following data file contains aggregate statistics on performance when

1 0

[metrics-web/master] Remove unused ExoneraTor config lines.
by karsten＠torproject.org 30 Jun '17

30 Jun '17

commit 71d2eb30c909ec46f2ec79c99c06ae3141b10ebe Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jun 30 10:18:24 2017 +0200 Remove unused ExoneraTor config lines. --- .../java/org/torproject/ernie/cron/Configuration.java | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/modules/legacy/src/main/java/org/torproject/ernie/cron/Configuration.java b/modules/legacy/src/main/java/org/torproject/ernie/cron/Configuration.java index 35d52b9..d1bc63b 100644 --- a/modules/legacy/src/main/java/org/torproject/ernie/cron/Configuration.java +++ b/modules/legacy/src/main/java/org/torproject/ernie/cron/Configuration.java @@ -44,11 +44,6 @@ public class Configuration { private boolean writeBridgeStats = false; - private String exoneraTorDatabaseJdbc = "jdbc:postgresql:" - + "//localhost/exonerator?user=metrics&password=password"; - - private String exoneraTorImportDirectory = "exonerator-import/"; - /** Initializes this configuration class. */ public Configuration() { @@ -96,10 +91,6 @@ public class Configuration { } else if (line.startsWith("WriteBridgeStats")) { this.writeBridgeStats = Integer.parseInt( line.split(" ")[1]) != 0; - } else if (line.startsWith("ExoneraTorDatabaseJdbc")) { - this.exoneraTorDatabaseJdbc = line.split(" ")[1]; - } else if (line.startsWith("ExoneraTorImportDirectory")) { - this.exoneraTorImportDirectory = line.split(" ")[1]; } else { logger.severe("Configuration file contains unrecognized " + "configuration key in line '" + line + "'! Exiting!"); @@ -177,13 +168,5 @@ public class Configuration { public boolean getWriteBridgeStats() { return this.writeBridgeStats; } - - public String getExoneraTorDatabaseJdbc() { - return this.exoneraTorDatabaseJdbc; - } - - public String getExoneraTorImportDirectory() { - return this.exoneraTorImportDirectory; - } }

1 0

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits June 2017