August 2017 - tor-commits - lists.torproject.org

[stem/master] Avoid infinite recursion from new is_geoip_unavailable check
by atagar＠torproject.org 14 Aug '17

14 Aug '17

commit f2305304084c62016b030064b97e43388d5af57a Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Aug 14 10:36:40 2017 -0700 Avoid infinite recursion from new is_geoip_unavailable check Um... oops. /nyx/stem/control.py", line 469, in wrapped return func(self, *args, **kwargs) File "/home/ubuntu/nyx/stem/control.py", line 1147, in get_info if param.startswith('ip-to-country/') and self.is_geoip_unavailable(): File "/home/ubuntu/nyx/stem/control.py", line 3661, in is_geoip_unavailable self.get_info('ip-to-country/0.0.0.0') File "/home/ubuntu/nyx/stem/control.py", line 469, in wrapped return func(self, *args, **kwargs) File "/home/ubuntu/nyx/stem/control.py", line 1147, in get_info if param.startswith('ip-to-country/') and self.is_geoip_unavailable(): File "/home/ubuntu/nyx/stem/control.py", line 3661, in is_geoip_unavailable self.get_info('ip-to-country/0.0.0.0') File "/home/ubuntu/nyx/stem/control.py", line 469, in wrapped return func(self, *args, **kwargs) File "/home/ubuntu/nyx/stem/control.py", line 1139, in get_info cached_results = self._get_cache_map(from_cache, 'getinfo') File "/home/ubuntu/nyx/stem/control.py", line 3064, in _get_cache_map with self._cache_lock: File "/usr/lib/python2.7/threading.py", line 169, in acquire if self.__owner == me: RuntimeError: maximum recursion depth exceeded in cmp --- stem/control.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stem/control.py b/stem/control.py index 760f9433..7dc632bb 100644 --- a/stem/control.py +++ b/stem/control.py @@ -1144,7 +1144,7 @@ class Controller(BaseController): params.remove(user_expected_key) for param in params: - if param.startswith('ip-to-country/') and self.is_geoip_unavailable(): + if param.startswith('ip-to-country/') and param != 'ip-to-country/0.0.0.0' and self.is_geoip_unavailable(): raise stem.ProtocolError('Tor geoip database is unavailable') # if everything was cached then short circuit making the query

1 0

[stem/master] Fixing trace truncation for received messages
by atagar＠torproject.org 14 Aug '17

14 Aug '17

commit 2d25cf9f103f9a876dcfee9d3e2b233a97f3e0c3 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Aug 14 10:12:26 2017 -0700 Fixing trace truncation for received messages Oops, missed that these were the lines themselves rather than the count. --- stem/socket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stem/socket.py b/stem/socket.py index 8c3e4205..d10b40d2 100644 --- a/stem/socket.py +++ b/stem/socket.py @@ -599,7 +599,7 @@ def recv_message(control_file): if TRUNCATE_LOGS and len(log_message_lines) > TRUNCATE_LOGS: log_message = '\n'.join(log_message_lines[:TRUNCATE_LOGS] + ['... %i more lines...' % (len(log_message_lines) - TRUNCATE_LOGS)]) - if log_message_lines > 2: + if len(log_message_lines) > 2: log.trace('Received from tor:\n%s' % log_message) else: log.trace('Received from tor: %s' % log_message.replace('\n', '\\n'))

1 0

[stem/master] More succinct trace level logging
by atagar＠torproject.org 14 Aug '17

14 Aug '17

commit 245b11bb111161c5105150d4f250ddfbce6de7c0 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Aug 14 10:00:09 2017 -0700 More succinct trace level logging Reading nyx's trace level logs is made quite a bit more annoying due to short request/responses being placed on their own lines. Rather, keeping everything on one line if the message is short. --- docs/change_log.rst | 2 ++ stem/socket.py | 16 +++++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/change_log.rst b/docs/change_log.rst index ea353a23..59b546ad 100644 --- a/docs/change_log.rst +++ b/docs/change_log.rst @@ -54,8 +54,10 @@ The following are only available within Stem's `git repository * Unable to use cookie auth when path includes wide characters (chinese, japanese, etc) * Tor change caused :func:`~stem.control.Controller.list_ephemeral_hidden_services` to provide empty strings if unset (:trac:`21329`) * Better error message when :func:`~stem.control.Controller.set_conf` fails due to an option being immutable + * :func:`~stem.control.Controller.is_geoip_unavailable` now determines if database is available right away * Failed to parse torrcs without a port on ipv6 exit policy entries * Resilient to 'Tor' prefix in 'GETINFO version' result (:spec:`c5ff1b1`) + * More succinct trace level logging * **Descriptors** diff --git a/stem/socket.py b/stem/socket.py index 011f35ef..8c3e4205 100644 --- a/stem/socket.py +++ b/stem/socket.py @@ -485,7 +485,8 @@ def send_message(control_file, message, raw = False): control_file.flush() log_message = message.replace('\r\n', '\n').rstrip() - log.trace('Sent to tor:\n' + log_message) + msg_div = '\n' if '\n' in log_message else ' ' + log.trace('Sent to tor:%s%s' % (msg_div, log_message)) except socket.error as exc: log.info('Failed to send message: %s' % exc) @@ -593,14 +594,15 @@ def recv_message(control_file): raw_content_str = b''.join(raw_content) log_message = stem.util.str_tools._to_unicode(raw_content_str.replace(b'\r\n', b'\n').rstrip()) + log_message_lines = log_message.split('\n') - if TRUNCATE_LOGS: - log_message_lines = log_message.split('\n') + if TRUNCATE_LOGS and len(log_message_lines) > TRUNCATE_LOGS: + log_message = '\n'.join(log_message_lines[:TRUNCATE_LOGS] + ['... %i more lines...' % (len(log_message_lines) - TRUNCATE_LOGS)]) - if len(log_message_lines) > TRUNCATE_LOGS: - log_message = '\n'.join(log_message_lines[:TRUNCATE_LOGS] + ['... %i more lines...' % (len(log_message_lines) - TRUNCATE_LOGS)]) - - log.trace('Received from tor:\n' + log_message) + if log_message_lines > 2: + log.trace('Received from tor:\n%s' % log_message) + else: + log.trace('Received from tor: %s' % log_message.replace('\n', '\\n')) return stem.response.ControlMessage(parsed_content, raw_content_str) elif divider == '+':

1 0

[stem/master] Determine if geoip as unavailable right away
by atagar＠torproject.org 14 Aug '17

14 Aug '17

commit 75adf029b63423782b0375ab273560c1d597fde6 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Aug 14 08:50:46 2017 -0700 Determine if geoip as unavailable right away Since at least 2012 tor's been providing an error response that explicitly says the geoip database is unavailable (commit 817ff96). As such we no longer need the hack where we count the number of GETINFO failures. --- stem/control.py | 64 +++++++++++++++++++++++---------------------------------- 1 file changed, 26 insertions(+), 38 deletions(-) diff --git a/stem/control.py b/stem/control.py index a88f6e08..760f9433 100644 --- a/stem/control.py +++ b/stem/control.py @@ -395,11 +395,6 @@ UNCACHEABLE_GETCONF_PARAMS = ( 'hiddenserviceauthorizeclient', ) -# number of sequential attempts before we decide that the Tor geoip database -# is unavailable - -GEOIP_FAILURE_THRESHOLD = 5 - SERVER_DESCRIPTORS_UNSUPPORTED = "Tor is currently not configured to retrieve \ server descriptors. As of Tor version 0.2.3.25 it downloads microdescriptors \ instead unless you set 'UseMicrodescriptors 0' in your torrc." @@ -1035,11 +1030,8 @@ class Controller(BaseController): self._event_listeners = {} self._event_listeners_lock = threading.RLock() - - # number of sequential 'GETINFO ip-to-country/*' lookups that have failed - - self._geoip_failure_count = 0 self._enabled_features = [] + self._is_geoip_unavailable = None super(Controller, self).__init__(control_socket, is_authenticated) @@ -1125,8 +1117,7 @@ class Controller(BaseController): provided a default response * :class:`stem.InvalidArguments` if the 'params' requested was invalid - * :class:`stem.ProtocolError` if the geoip database is known to be - unavailable + * :class:`stem.ProtocolError` if the geoip database is unavailable """ start_time = time.time() @@ -1154,8 +1145,6 @@ class Controller(BaseController): for param in params: if param.startswith('ip-to-country/') and self.is_geoip_unavailable(): - # the geoip database already looks to be unavailable - abort the request - raise stem.ProtocolError('Tor geoip database is unavailable') # if everything was cached then short circuit making the query @@ -1189,9 +1178,7 @@ class Controller(BaseController): if key in CACHEABLE_GETINFO_PARAMS or key in CACHEABLE_GETINFO_PARAMS_UNTIL_SETCONF: to_cache[key] = value elif key.startswith('ip-to-country/'): - # both cache-able and means that we should reset the geoip failure count to_cache[key] = value - self._geoip_failure_count = -1 self._set_cache(to_cache, 'getinfo') @@ -1202,21 +1189,7 @@ class Controller(BaseController): else: return list(reply.values())[0] except stem.ControllerError as exc: - # bump geoip failure count if... - # * we're caching results - # * this was soley a geoip lookup - # * we've never had a successful geoip lookup (failure count isn't -1) - - is_geoip_request = len(params) == 1 and list(params)[0].startswith('ip-to-country/') - - if is_geoip_request and self.is_caching_enabled() and self._geoip_failure_count != -1: - self._geoip_failure_count += 1 - - if self.is_geoip_unavailable(): - log.warn("Tor's geoip database is unavailable.") - log.debug('GETINFO %s (failed: %s)' % (' '.join(params), exc)) - raise @with_default() @@ -3157,7 +3130,7 @@ class Controller(BaseController): with self._cache_lock: self._request_cache = {} self._last_newnym = 0.0 - self._geoip_failure_count = 0 + self._is_geoip_unavailable = None def load_conf(self, configtext): """ @@ -3667,18 +3640,33 @@ class Controller(BaseController): def is_geoip_unavailable(self): """ - Provides **True** if we've concluded hat our geoip database is unavailable, - **False** otherwise. This is determined by having our 'GETINFO - ip-to-country/\*' lookups fail so this will default to **False** if we - aren't making those queries. + Provides **True** if tor's geoip database is unavailable, **False** + otherwise. + + .. versionchanged:: 1.6.0 + No longer requires previously failed GETINFO requests to determine this. + + .. deprecated:: 1.6.0 + If this becomes available in tor we'll be deprecating this in our 2.x + release (:trac:`23237`). If not this will be renamed to + is_geoip_available. - Geoip failures will be untracked if caching is disabled. - :returns: **bool** to indicate if we've concluded our geoip database to be - unavailable or not + :returns: **bool** indicating if we've determined tor's geoip database to + be unavailable or not """ - return self._geoip_failure_count >= GEOIP_FAILURE_THRESHOLD + if self._is_geoip_unavailable is None: + try: + self.get_info('ip-to-country/0.0.0.0') + self._is_geoip_unavailable = False + except stem.ControllerError as exc: + if 'GeoIP data not loaded' in str(exc): + self._is_geoip_unavailable = True + else: + return False # unexpected issue, fail open and don't cache + + return self._is_geoip_unavailable def map_address(self, mapping): """

1 0

[metrics-lib/master] Avoid compiler warnings in test classes.
by karsten＠torproject.org 14 Aug '17

14 Aug '17

commit e0c5774529fb2597e80690b5cc625e54a2eeb245 Author: iwakeh <iwakeh(a)torproject.org> Date: Fri Jul 28 07:53:28 2017 +0000 Avoid compiler warnings in test classes. --- .../descriptor/index/DescriptorIndexCollectorTest.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/test/java/org/torproject/descriptor/index/DescriptorIndexCollectorTest.java b/src/test/java/org/torproject/descriptor/index/DescriptorIndexCollectorTest.java index d7f602f..15ca5b5 100644 --- a/src/test/java/org/torproject/descriptor/index/DescriptorIndexCollectorTest.java +++ b/src/test/java/org/torproject/descriptor/index/DescriptorIndexCollectorTest.java @@ -43,7 +43,7 @@ public class DescriptorIndexCollectorTest { makeStructure(remoteDirectory, "2"); File indexFile = newIndexFile("testindex.json", - remoteDirectory.toURL().toString()); + remoteDirectory.toURI().toURL().toString()); // verify precondition for test. checkContains(true, @@ -54,7 +54,7 @@ public class DescriptorIndexCollectorTest { "a/b/y2","a/b/x2"); DescriptorCollector dc = new DescriptorIndexCollector(); - dc.collectDescriptors(indexFile.toURL().toString(), + dc.collectDescriptors(indexFile.toURI().toURL().toString(), new String[]{"a/b", "a"}, 1451606400_000L, localFolder, false); // verify that files in 'a/b' were fetched @@ -146,7 +146,7 @@ public class DescriptorIndexCollectorTest { makeStructure(remoteDirectory, "2"); File indexFile = newIndexFile("testindexDelete.json", - remoteDirectory.toURL().toString()); + remoteDirectory.toURI().toURL().toString()); // verify precondition for test. checkContains(true, @@ -157,7 +157,7 @@ public class DescriptorIndexCollectorTest { "a/b/y2","a/b/x2"); new DescriptorIndexCollector() - .collectDescriptors(indexFile.toURL().toString(), + .collectDescriptors(indexFile.toURI().toURL().toString(), new String[]{"a/b/", "a/b/c"}, 1451606400_000L, localFolder, true); // verify file addition, including that the non-synced dir is not touched. @@ -187,7 +187,7 @@ public class DescriptorIndexCollectorTest { makeStructure(remoteDirectory, "2"); File indexFile = newIndexFile("testindexDelete.json", - remoteDirectory.toURL().toString()); + remoteDirectory.toURI().toURL().toString()); // verify precondition for test. checkContains(true, @@ -198,7 +198,7 @@ public class DescriptorIndexCollectorTest { "a/b/y2","a/b/x2"); new DescriptorIndexCollector() - .collectDescriptors(indexFile.toURL().toString(), + .collectDescriptors(indexFile.toURI().toURL().toString(), new String[]{"/a/b", "/a/b/c/"}, 1451606400_000L, localFolder, true);

1 0

[metrics-lib/master] Removed javadoc links to removed interfaces.
by karsten＠torproject.org 14 Aug '17

14 Aug '17

commit e2246804e366920c9c334d774575217be0c644f8 Author: iwakeh <iwakeh(a)torproject.org> Date: Fri Jul 28 07:53:29 2017 +0000 Removed javadoc links to removed interfaces. --- src/main/java/org/torproject/descriptor/package-info.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/torproject/descriptor/package-info.java b/src/main/java/org/torproject/descriptor/package-info.java index e5f36d2..5827115 100644 --- a/src/main/java/org/torproject/descriptor/package-info.java +++ b/src/main/java/org/torproject/descriptor/package-info.java @@ -20,15 +20,11 @@ * {@link org.torproject.descriptor.DescriptorSourceFactory} which is the * main entry point into using this library. This factory is used to * create the descriptor sources for obtaining remote descriptor data - * ({@link org.torproject.descriptor.DescriptorDownloader} and - * {@link org.torproject.descriptor.DescriptorCollector}) and descriptor + * ({@link org.torproject.descriptor.DescriptorCollector}) and descriptor * sources for processing local descriptor data * ({@link org.torproject.descriptor.DescriptorReader} and * {@link org.torproject.descriptor.DescriptorParser}). General-purpose - * types also include descriptor containers - * ({@link org.torproject.descriptor.DescriptorRequest} and - * {@link org.torproject.descriptor.DescriptorFile}) and the - * superinterface for all provided descriptors + * types also include the superinterface for all provided descriptors * ({@link org.torproject.descriptor.Descriptor}).</li> * * <li>The first group of descriptors is published by relays and servers

1 0

[nyx/master] Fix installation and tests
by atagar＠torproject.org 13 Aug '17

13 Aug '17

commit 9f2881293f50e9fca60f767a36d52ee237464737 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Aug 13 14:59:23 2017 -0700 Fix installation and tests Oops! Could've sworn I ran our tests after our earlier changes but evidently not. Moving our nyxrc.sample broke installation, and swapping to bytes broke some test assertions. --- nyx/starter.py | 2 +- setup.py | 2 +- test/panel/graph.py | 14 +++++++------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/nyx/starter.py b/nyx/starter.py index 48885a6..6aa98b7 100644 --- a/nyx/starter.py +++ b/nyx/starter.py @@ -23,7 +23,7 @@ import stem import stem.util.log import stem.util.system -from nyx import log, init_controller, uses_settings, nyx_interface +from nyx import init_controller, uses_settings, nyx_interface DEBUG_HEADER = """ Nyx {nyx_version} Debug Dump diff --git a/setup.py b/setup.py index f47f915..d463c2a 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ class NyxInstaller(install): self.install_bin_script('run_nyx', os.path.join(self.install_scripts, 'nyx')) self.install_file('man page', 'nyx.1', self.man_page) - self.install_file('web/nyxrc sample', 'nyxrc.sample', self.sample_path) + self.install_file('nyxrc sample', os.path.join('web', 'nyxrc.sample'), self.sample_path) def install_bin_script(self, source, dest): # Install our bin script. We do this ourselves rather than with the setup() diff --git a/test/panel/graph.py b/test/panel/graph.py index 1420a00..99139e4 100644 --- a/test/panel/graph.py +++ b/test/panel/graph.py @@ -16,26 +16,26 @@ from mock import patch EXPECTED_BLANK_GRAPH = """ Download: -0 b +0 B -0 b +0 B 5s 10 15 """.rstrip() EXPECTED_ACCOUNTING = """ Accounting (awake) Time to reset: 01:02 - 37.7 Kb / 842.0 Kb 16.0 Kb / 74.1 Kb + 4.7 KB / 105.2 KB 2.0 KB / 9.2 KB """.strip() EXPECTED_GRAPH = """ Download: -5 Kb * +6 KB * * -2 Kb ** * +3 KB ** * * **** -0 b ********* +0 B ********* 5s 10 15 """.rstrip() @@ -103,7 +103,7 @@ class TestGraphPanel(unittest.TestCase): @require_curses @patch('nyx.panel.graph.tor_controller') def test_draw_subgraph(self, tor_controller_mock): - tor_controller_mock().get_info.return_value = '543,543 421,421 551,551 710,710 200,200 175,175 188,188 250,250 377,377' + tor_controller_mock().get_info.return_value = '5430,5430 4210,4210 5510,5510 7100,7100 2000,2000 1750,1750 1880,1880 2500,2500 3770,3770' data = nyx.panel.graph.BandwidthStats() rendered = test.render(nyx.panel.graph._draw_subgraph, data.primary, 0, 30, 7, nyx.panel.graph.Bounds.LOCAL_MAX, nyx.panel.graph.Interval.EACH_SECOND, nyx.curses.Color.CYAN, '*')

1 0

[nyx/master] Deduplicating additional messages
by atagar＠torproject.org 13 Aug '17

13 Aug '17

commit 252632e4d41306a127a4a4cb0107c75dd1c96b20 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Aug 10 10:41:40 2017 -0700 Deduplicating additional messages No doubt there's more we should deduplicate, but went ahead and deduplicating things that came up while off-network at debug runlevel. --- nyx/settings/dedup.cfg | 10 ++++++++++ web/changelog/index.html | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/nyx/settings/dedup.cfg b/nyx/settings/dedup.cfg index 42d8102..fa44cfa 100644 --- a/nyx/settings/dedup.cfg +++ b/nyx/settings/dedup.cfg @@ -20,10 +20,17 @@ # tls object). at_most 12800. # [DEBUG] connection_read_to_buf(): TLS connection closed on read. Closing. # (Nickname moria1, address 128.31.0.34) +# [DEBUG] channel_change_state_(): Changing state of channel 0x55fda0f26550 +# (global ID 13) from "opening" to "closing" +# [DEBUG] directory_send_command(): Sent request to directory server +# '194.109.206.212:443': (purpose: 14, request size: 188, payload size: 0) # [INFO] run_connection_housekeeping(): Expiring non-open OR connection to fd # 16 (79.193.61.171:443). # [INFO] rep_hist_downrate_old_runs(): Discounting all old stability info by a # factor of 0.950000 +# [INFO] connection_or_set_identity_digest(): Set identity digest for +# 0x55fda0f26840 ([scrubbed]): 7EA6EAD6FD83083C538F44038BBFA077587DD755 +# <unset>. # [NOTICE] Circuit build timeout of 96803ms is beyond the maximum build time we # have ever observed. Capping it to 96107ms. # The above NOTICE changes to an INFO message in maint-0.2.2 @@ -72,8 +79,11 @@ dedup.DEBUG connection_remove(): removing socket dedup.DEBUG connection_or_process_cells_from_inbuf(): dedup.DEBUG *pending in tls object). at_most dedup.DEBUG connection_read_to_buf(): TLS connection closed on read. Closing. +dedup.DEBUG channel_change_state_(): Changing state of channel +dedup.DEBUG directory_send_command(): Sent request to directory server dedup.INFO run_connection_housekeeping(): Expiring dedup.INFO rep_hist_downrate_old_runs(): Discounting all old stability info by a factor of +dedup.INFO connection_or_set_identity_digest(): Set identity digest for dedup.INFO *build time we have ever observed. Capping it to dedup.NOTICE *build time we have ever observed. Capping it to dedup.NOTICE *We will now assume a circuit is too slow to use after waiting diff --git a/web/changelog/index.html b/web/changelog/index.html index c95f522..3bc4641 100644 --- a/web/changelog/index.html +++ b/web/changelog/index.html @@ -79,7 +79,7 @@ <li><span class="component">Logging</span> <ul> - <li>Order of magnitude faster log deduplication</li> + <li>Order of magnitude more performant log deduplication</li> </ul> </li>

1 0

[nyx/master] Cache our router status entry
by atagar＠torproject.org 13 Aug '17

13 Aug '17

commit 769ab42c72fb30fe2a7713604bc7663262d41379 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Aug 13 15:12:17 2017 -0700 Cache our router status entry Both our header and graph frequenly refer to our router status entry. In practice this only changes once per hour so caching it for five minutes. At debug runlevel this causes the vast majority of the noise in our logs. --- nyx/panel/graph.py | 2 +- nyx/panel/header.py | 3 ++- nyx/tracker.py | 24 ++++++++++++++++++++++++ test/panel/header.py | 6 ++++-- 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/nyx/panel/graph.py b/nyx/panel/graph.py index fc1e7cb..e7114cb 100644 --- a/nyx/panel/graph.py +++ b/nyx/panel/graph.py @@ -338,7 +338,7 @@ class BandwidthStats(GraphCategory): stats.append('limit: %s/s' % bw_rate_label) stats.append('burst: %s/s' % bw_burst_label) - my_router_status_entry = controller.get_network_status(default = None) + my_router_status_entry = nyx.tracker.get_consensus_tracker().my_router_status_entry() measured_bw = getattr(my_router_status_entry, 'bandwidth', None) if measured_bw: diff --git a/nyx/panel/header.py b/nyx/panel/header.py index 9d90d75..cf11b67 100644 --- a/nyx/panel/header.py +++ b/nyx/panel/header.py @@ -232,6 +232,7 @@ class Sampling(object): or_listeners = controller.get_listeners(stem.control.Listener.OR, []) control_listeners = controller.get_listeners(stem.control.Listener.CONTROL, []) + my_router_status_entry = nyx.tracker.get_consensus_tracker().my_router_status_entry() if controller.get_conf('HashedControlPassword', None): auth_type = 'password' @@ -262,7 +263,7 @@ class Sampling(object): 'nickname': controller.get_conf('Nickname', ''), 'newnym_wait': controller.get_newnym_wait(), 'exit_policy': controller.get_exit_policy(None), - 'flags': getattr(controller.get_network_status(default = None), 'flags', []), + 'flags': getattr(my_router_status_entry, 'flags', []), 'version': str(controller.get_version('Unknown')).split()[0], 'version_status': controller.get_info('status/version/current', 'Unknown'), diff --git a/nyx/tracker.py b/nyx/tracker.py index 0e97560..21d5b36 100644 --- a/nyx/tracker.py +++ b/nyx/tracker.py @@ -33,6 +33,7 @@ Background tasks for gathering information about the tor process. ConsensusTracker - performant lookups for consensus related information |- update - updates the consensus information we're based on + |- my_router_status_entry - provides the router status entry for ourselves |- get_relay_nickname - provides the nickname for a given relay |- get_relay_fingerprints - provides relays running at a location +- get_relay_address - provides the address a relay is running at @@ -810,6 +811,9 @@ class ConsensusTracker(object): self._nickname_cache = {} # fingerprint => nickname lookup cache self._address_cache = {} + self._my_router_status_entry = None + self._my_router_status_entry_time = 0 + # Stem's get_network_statuses() is slow, and overkill for what we need # here. Just parsing the raw GETINFO response to cut startup time down. @@ -842,16 +846,36 @@ class ConsensusTracker(object): new_fingerprint_cache = {} new_address_cache = {} new_nickname_cache = {} + our_fingerprint = tor_controller().get_info('fingerprint', None) for desc in router_status_entries: new_fingerprint_cache.setdefault(desc.address, []).append((desc.or_port, desc.fingerprint)) new_address_cache[desc.fingerprint] = (desc.address, desc.or_port) new_nickname_cache[desc.fingerprint] = desc.nickname + if desc.fingerprint == our_fingerprint: + self._my_router_status_entry = desc + self._my_router_status_entry_time = time.time() + self._fingerprint_cache = new_fingerprint_cache self._address_cache = new_address_cache self._nickname_cache = new_nickname_cache + def my_router_status_entry(self): + """ + Provides the router status entry of ourselves. Descriptors are published + hourly, and results are cached for five minutes. + + :returns: :class:`~stem.descriptor.router_status_entry.RouterStatusEntryV3` + for ourselves, **None** if it cannot be retrieved + """ + + if self._my_router_status_entry is None or (time.time() - self._my_router_status_entry_time) > 300: + self._my_router_status_entry = tor_controller().get_network_status(default = None) + self._my_router_status_entry_time = time.time() + + return self._my_router_status_entry + def get_relay_nickname(self, fingerprint): """ Provides the nickname associated with the given relay. diff --git a/test/panel/header.py b/test/panel/header.py index 6a9ba3f..805c8c7 100644 --- a/test/panel/header.py +++ b/test/panel/header.py @@ -83,18 +83,18 @@ class TestHeaderPanel(unittest.TestCase): @patch('nyx.panel.header.tor_controller') @patch('nyx.tracker.get_resource_tracker') + @patch('nyx.tracker.get_consensus_tracker') @patch('time.time', Mock(return_value = 1234.5)) @patch('os.times', Mock(return_value = (0.08, 0.03, 0.0, 0.0, 18759021.31))) @patch('os.uname', Mock(return_value = ('Linux', 'odin', '3.5.0-54-generic', '#81~precise1-Ubuntu SMP Tue Jul 15 04:05:58 UTC 2014', 'i686'))) @patch('stem.util.system.start_time', Mock(return_value = 5678)) @patch('stem.util.proc.file_descriptors_used', Mock(return_value = 89)) - def test_sample(self, resource_tracker_mock, tor_controller_mock): + def test_sample(self, consensus_tracker_mock, resource_tracker_mock, tor_controller_mock): tor_controller_mock().is_alive.return_value = True tor_controller_mock().connection_time.return_value = 567.8 tor_controller_mock().get_latest_heartbeat.return_value = 89.0 tor_controller_mock().get_newnym_wait.return_value = 0 tor_controller_mock().get_exit_policy.return_value = stem.exit_policy.ExitPolicy('reject *:*') - tor_controller_mock().get_network_status.return_value = None tor_controller_mock().get_version.return_value = stem.version.Version('0.1.2.3-tag') tor_controller_mock().get_pid.return_value = '123' @@ -126,6 +126,8 @@ class TestHeaderPanel(unittest.TestCase): resource_tracker_mock().get_value.return_value = resources stem.util.system.SYSTEM_CALL_TIME = 0.0 + consensus_tracker_mock().my_router_status_entry.return_value = None + vals = nyx.panel.header.Sampling.create() self.assertEqual(1234.5, vals.retrieved)

1 0

[stem/master] Don't load vim swap files as configurations
by atagar＠torproject.org 13 Aug '17

13 Aug '17

commit a55d9ba5a5410d67db10796868cae997ff07ac36 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Aug 10 10:39:50 2017 -0700 Don't load vim swap files as configurations When loading a directory of config files we blindly read everything as configs. We should probably skip quite a few things but starting with vim swap files since this is just bit me. --- docs/change_log.rst | 1 + stem/util/conf.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/docs/change_log.rst b/docs/change_log.rst index 7159288d..ea353a23 100644 --- a/docs/change_log.rst +++ b/docs/change_log.rst @@ -77,6 +77,7 @@ The following are only available within Stem's `git repository * Added :class:`~stem.util.test_tools.TimedTestRunner` and :func:`~stem.util.test_tools.test_runtimes` * Supporing pid arguments in :func:`~stem.util.system.is_running` * Normalized :func:`~stem.util.term.format` to return unicode + * Don't load vim swap files as configurations * **Interpreter** diff --git a/stem/util/conf.py b/stem/util/conf.py index 5691b2a4..d4af467d 100644 --- a/stem/util/conf.py +++ b/stem/util/conf.py @@ -475,6 +475,9 @@ class Config(object): .. versionchanged:: 1.3.0 Added the **commenting** argument. + .. versionchanged:: 1.6.0 + Avoid loading vim swap files. + :param str path: file or directory path to be loaded, this uses the last loaded path if not provided :param bool commenting: ignore line content after a '#' if **True**, read @@ -494,6 +497,9 @@ class Config(object): if os.path.isdir(self._path): for root, dirnames, filenames in os.walk(self._path): for filename in filenames: + if filename.endswith('.swp'): + continue # vim swap file + self.load(os.path.join(root, filename)) return

1 0