February 2016 - tor-commits - lists.torproject.org

[nyx/master] Retrieve connection information despite DisableDebuggerAttachment
by atagar＠torproject.org 01 Feb '16

01 Feb '16

commit 53fbe024898e96d9564ba982448f5c048acd0be0 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jan 31 20:08:40 2016 -0800 Retrieve connection information despite DisableDebuggerAttachment For years tor's DisableDebuggerAttachment has been the bane of nyx. The feature wasn't intended to effect us, but screws with proc permissions breaking every connection resolver we have... https://trac.torproject.org/projects/tor/ticket/15259 Currently we read /proc/<pid>/fd to get connection inodes, then use that determine what from /proc/net/tcp belongs to our process. Tor's DisableDebuggerAttachment breaks that by making /proc/<pid>/fd only readable by root. However, even without knowing the inodes we can identify tor related connections by if they go to a relay or our ORPort/DirPort/ControlPort. This is exactly what nyx already does to identify a connection's type. TL;DR. Connection resolution works all the time now. Only drawbacks are... * Connection resolution can't work until we have consensus information. This can take a few seconds so we don't show connections right away. * When resolving this way we can't show client or exit connections. Nyx already scrubbed these so it's not a big loss, but means we now don't even show that they exist. If the user sets 'DisableDebuggerAttachment 0' in their torrc we still do connection resolution via the normal method. --- nyx/controller.py | 33 +++------------------------------ nyx/util/__init__.py | 4 ++++ nyx/util/tracker.py | 37 ++++++++++++++++++++++++++++++++++--- 3 files changed, 41 insertions(+), 33 deletions(-) diff --git a/nyx/controller.py b/nyx/controller.py index 7ff6e53..ee647a7 100644 --- a/nyx/controller.py +++ b/nyx/controller.py @@ -115,37 +115,10 @@ def init_controller(stdscr, start_time): if CONFIG['features.panels.show.connection']: page_panels.append([nyx.connection_panel.ConnectionPanel(stdscr)]) - # The DisableDebuggerAttachment will prevent our connection panel from really - # functioning. It'll have circuits, but little else. If this is the case then - # notify the user and tell them what they can do to fix it. - controller = tor_controller() - - if controller.get_conf('DisableDebuggerAttachment', None) == '1': - log.notice("Tor is preventing system utilities like netstat and lsof from working. This means that nyx can't provide you with connection information. You can change this by adding 'DisableDebuggerAttachment 0' to your torrc and restarting tor. For more information see...\nhttps://trac.torproject.org/3313") - nyx.util.tracker.get_connection_tracker().set_paused(True) - else: - # Configures connection resoultions. This is paused/unpaused according to - # if Tor's connected or not. - - controller.add_status_listener(conn_reset_listener) - - tor_pid = controller.get_pid(None) - - if tor_pid: - # use the tor pid to help narrow connection results - tor_cmd = system.name_by_pid(tor_pid) - - if tor_cmd is None: - tor_cmd = 'tor' - - resolver = nyx.util.tracker.get_connection_tracker() - log.info('Operating System: %s, Connection Resolvers: %s' % (os.uname()[0], ', '.join(resolver._resolvers))) - else: - # constructs singleton resolver and, if tor isn't connected, initizes - # it to be paused - - nyx.util.tracker.get_connection_tracker().set_paused(not controller.is_alive()) + controller.add_status_listener(conn_reset_listener) + resolver = nyx.util.tracker.get_connection_tracker() + log.info('Operating System: %s, Connection Resolvers: %s' % (os.uname()[0], ', '.join(resolver._resolvers))) # third page: config diff --git a/nyx/util/__init__.py b/nyx/util/__init__.py index 0f53ab6..a7b2e8d 100644 --- a/nyx/util/__init__.py +++ b/nyx/util/__init__.py @@ -31,6 +31,10 @@ TESTING = False stem.control.CACHEABLE_GETINFO_PARAMS = list(stem.control.CACHEABLE_GETINFO_PARAMS) + ['address'] +# disable trace level messages about cache hits + +stem.control.LOG_CACHE_FETCHES = False + try: uses_settings = stem.util.conf.uses_settings('nyx', os.path.join(BASE_DIR, 'config'), lazy_load = False) except IOError as exc: diff --git a/nyx/util/tracker.py b/nyx/util/tracker.py index ffd24c9..f218630 100644 --- a/nyx/util/tracker.py +++ b/nyx/util/tracker.py @@ -52,7 +52,7 @@ import threading import stem.control -from stem.util import conf, connection, proc, str_tools, system +from stem.util import conf, connection, enum, proc, str_tools, system from nyx.util import log, tor_controller @@ -67,6 +67,10 @@ RESOURCE_TRACKER = None PORT_USAGE_TRACKER = None CONSENSUS_TRACKER = None +CustomResolver = enum.Enum( + ('INFERENCE', 'by inference'), +) + # Extending stem's Connection tuple with attributes for the uptime of the # connection. @@ -476,7 +480,6 @@ class ConnectionTracker(Daemon): self._connections = [] self._start_times = {} # connection => (unix_timestamp, is_legacy) - self._resolvers = connection.system_resolvers() self._custom_resolver = None self._is_first_run = True @@ -486,6 +489,15 @@ class ConnectionTracker(Daemon): self._failure_count = 0 self._rate_too_low_count = 0 + # If 'DisableDebuggerAttachment 0' is set we can do normal connection + # resolution. Otherwise connection resolution by inference is the only game + # in town. + + if tor_controller().get_conf('DisableDebuggerAttachment', None) == '0': + self._resolvers = connection.system_resolvers() + else: + self._resolvers = [CustomResolver.INFERENCE] + def _task(self, process_pid, process_name): if self._custom_resolver: resolver = self._custom_resolver @@ -500,7 +512,26 @@ class ConnectionTracker(Daemon): start_time = time.time() new_connections, new_start_times = [], {} - for conn in connection.get_connections(resolver, process_pid = process_pid, process_name = process_name): + if resolver == CustomResolver.INFERENCE: + # provide connections going to a relay or one of our tor ports + + connections = [] + controller = tor_controller() + consensus_tracker = get_consensus_tracker() + + for conn in proc.connections(user = controller.get_user(None)): + if conn.remote_port in consensus_tracker.get_relay_fingerprints(conn.remote_address): + connections.append(conn) # outbound to another relay + elif conn.local_port in controller.get_ports(stem.control.Listener.OR, []): + connections.append(conn) # inbound to our ORPort + elif conn.local_port in controller.get_ports(stem.control.Listener.DIR, []): + connections.append(conn) # inbound to our DirPort + elif conn.local_port in controller.get_ports(stem.control.Listener.CONTROL, []): + connections.append(conn) # controller connection + else: + connections = connection.get_connections(resolver, process_pid = process_pid, process_name = process_name) + + for conn in connections: conn_start_time, is_legacy = self._start_times.get(conn, (start_time, self._is_first_run)) new_start_times[conn] = (conn_start_time, is_legacy) new_connections.append(Connection(conn_start_time, is_legacy, *conn))

1 0

[nyx/master] Mappings for new event types
by atagar＠torproject.org 01 Feb '16

01 Feb '16

commit 047ec0b90080ac487390ff80d604f8908ad438f1 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jan 31 13:17:55 2016 -0800 Mappings for new event types Nyx uses a shorthand for events where you can list them with one character each. For instance, 'nyx -e Db' would log BW events along with DEBUG and above. Naturally tor's added quite a few event types in the last few years, causing the following warning... 01/31/2016 13:00:07 [INFO] nyx doesn't recognize the following event types: CIRC_MINOR, SIGNAL, CONF_CHANGED, CONN_BW, CELL_STATS, TB_EMPTY, CIRC_BW, TRANSPORT_LAUNCHED, HS_DESC, HS_DESC_CONTENT, NETWORK_LIVENESS (log 'UNKNOWN' events to see them) We now have so many that we need to use capital letters for many of these. Honestly this shorthand approach kinda sucks. It's handy for common event types like runlevels, but for anything else you need to check the mappings. I might change this in a future release but for now just adding the missing mappings. --- nyx/arguments.py | 46 ++++++++++++++++++++++++++++++++-------------- nyx/config/strings.cfg | 21 ++++++++++++++------- nyx/log_panel.py | 2 +- 3 files changed, 47 insertions(+), 22 deletions(-) diff --git a/nyx/arguments.py b/nyx/arguments.py index 7510057..01c2095 100644 --- a/nyx/arguments.py +++ b/nyx/arguments.py @@ -38,29 +38,47 @@ OPT_EXPANDED = [ ] TOR_EVENT_TYPES = { + # runlevels + 'd': 'DEBUG', 'i': 'INFO', 'n': 'NOTICE', 'w': 'WARN', 'e': 'ERR', - 'a': 'ADDRMAP', - 'f': 'AUTHDIR_NEWDESCS', - 'h': 'BUILDTIMEOUT_SET', + # important events + 'b': 'BW', 'c': 'CIRC', - 'j': 'CLIENTS_SEEN', - 'k': 'DESCCHANGED', - 'g': 'GUARD', - 'l': 'NEWCONSENSUS', - 'm': 'NEWDESC', - 'p': 'NS', - 'q': 'ORCONN', 's': 'STREAM', - 'r': 'STREAM_BW', - 't': 'STATUS_CLIENT', - 'u': 'STATUS_GENERAL', - 'v': 'STATUS_SERVER', + + # everything else + + 'a': 'ADDRMAP', + 'f': 'AUTHDIR_NEWDESCS', + 'j': 'BUILDTIMEOUT_SET', + 'k': 'CELL_STATS', + 'l': 'CIRC_BW', + 'm': 'CIRC_MINOR', + 'p': 'CONF_CHANGED', + 'q': 'CONN_BW', + 'r': 'CLIENTS_SEEN', + 'u': 'DESCCHANGED', + 'g': 'GUARD', + 'h': 'HS_DESC', + 'v': 'HS_DESC_CONTENT', + 'x': 'NETWORK_LIVENESS', + 'y': 'NEWCONSENSUS', + 'z': 'NEWDESC', + 'B': 'NS', + 'o': 'ORCONN', + 'C': 'SIGNAL', + 'F': 'STREAM_BW', + 'G': 'STATUS_CLIENT', + 'H': 'STATUS_GENERAL', + 'I': 'STATUS_SERVER', + 'J': 'TB_EMPTY', + 't': 'TRANSPORT_LAUNCHED', } diff --git a/nyx/config/strings.cfg b/nyx/config/strings.cfg index 9cd217f..85b4f8d 100644 --- a/nyx/config/strings.cfg +++ b/nyx/config/strings.cfg @@ -60,14 +60,19 @@ msg.debug.header |-------------------------------------------------------------------------------- msg.misc.event_types -| d DEBUG a ADDRMAP k DESCCHANGED s STREAM -| i INFO f AUTHDIR_NEWDESCS g GUARD r STREAM_BW -| n NOTICE h BUILDTIMEOUT_SET l NEWCONSENSUS t STATUS_CLIENT -| w WARN b BW m NEWDESC u STATUS_GENERAL -| e ERR c CIRC p NS v STATUS_SERVER -| j CLIENTS_SEEN q ORCONN +| d DEBUG a ADDRMAP r CLIENTS_SEEN C SIGNAL +| i INFO f AUTHDIR_NEWDESCS u DESCCHANGED F STREAM_BW +| n NOTICE j BUILDTIMEOUT_SET g GUARD G STATUS_CLIENT +| w WARN b BW h HS_DESC H STATUS_GENERAL +| e ERR k CELL_STATS v HS_DESC_CONTENT I STATUS_SERVER +| c CIRC x NETWORK_LIVENESS s STREAM +| l CIRC_BW y NEWCONSENSUS J TB_EMPTY +| m CIRC_MINOR z NEWDESC t TRANSPORT_LAUNCHED +| p CONF_CHANGED B NS +| q CONN_BW o ORCONN +| | DINWE tor runlevel+ A All Events -| 12345 nyx runlevel+ X No Events +| 12345 nyx runlevel+ X No Events | U Unknown Events msg.setup.unknown_term @@ -86,7 +91,9 @@ msg.usage.help_output | defaults to: {config_path} | -d, --debug LOG_PATH writes all nyx logs to the given location | -l, --log EVENT_FLAGS event types to be logged (default: {events}) +| |{event_flags} +| | -v, --version provides version information | -h, --help presents this help | diff --git a/nyx/log_panel.py b/nyx/log_panel.py index dedfc99..e97d74b 100644 --- a/nyx/log_panel.py +++ b/nyx/log_panel.py @@ -140,7 +140,7 @@ class LogPanel(panel.Panel, threading.Thread): # allow user to enter new types of events to log - unchanged if left blank - with nyx.popups.popup_window(11, 80) as (popup, width, height): + with nyx.popups.popup_window(16, 80) as (popup, width, height): if popup: # displays the available flags

1 0

[nyx/master] Better deduplication for nyx debug messages
by atagar＠torproject.org 01 Feb '16

01 Feb '16

commit 12b7c7c1420837b1f1e18e21c588a41bd7bd24c6 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Jan 30 18:24:48 2016 -0800 Better deduplication for nyx debug messages At the debug runlevel nyx mostly prints messages to indicate how long queries take. These are obviously dynamic. Rather than enumerating all log messages including a little hack to ignore that bit. --- nyx/config/dedup.cfg | 20 -------------------- nyx/util/log.py | 6 ++++++ 2 files changed, 6 insertions(+), 20 deletions(-) diff --git a/nyx/config/dedup.cfg b/nyx/config/dedup.cfg index 954e588..42d8102 100644 --- a/nyx/config/dedup.cfg +++ b/nyx/config/dedup.cfg @@ -88,24 +88,4 @@ dedup.WARN I have no descriptor for the router named dedup.WARN Controller gave us config lines that didn't validate dedup.WARN Problem bootstrapping. Stuck at dedup.WARN *missing key, -dedup.NYX_DEBUG refresh rate: -dedup.NYX_DEBUG proc call (cwd): -dedup.NYX_DEBUG proc call (memory usage): -dedup.NYX_DEBUG proc call (process command -dedup.NYX_DEBUG proc call (process utime -dedup.NYX_DEBUG proc call (process stime -dedup.NYX_DEBUG proc call (process start time -dedup.NYX_DEBUG proc call (process connections): -dedup.NYX_DEBUG system call: ps -dedup.NYX_DEBUG system call: netstat -dedup.NYX_DEBUG recreating panel ' -dedup.NYX_DEBUG redrawing the log panel with the corrected content height ( -dedup.NYX_DEBUG GETINFO accounting/bytes -dedup.NYX_DEBUG GETINFO accounting/bytes-left -dedup.NYX_DEBUG GETINFO accounting/interval-end -dedup.NYX_DEBUG GETINFO accounting/hibernating -dedup.NYX_DEBUG GETINFO traffic/read -dedup.NYX_DEBUG GETINFO traffic/written -dedup.NYX_DEBUG GETCONF -dedup.NYX_DEBUG Unable to query process resource usage from ps diff --git a/nyx/util/log.py b/nyx/util/log.py index 3bd0d40..f9c95cc 100644 --- a/nyx/util/log.py +++ b/nyx/util/log.py @@ -281,6 +281,12 @@ class LogEntry(object): elif self.message == entry.message: return True + if self.type == 'NYX_DEBUG' and 'runtime:' in self.message and 'runtime:' in entry.message: + # most nyx debug messages show runtimes so try matching without that + + if self.message[:self.message.find('runtime:')] == entry.message[:self.message.find('runtime:')]: + return True + for common_msg in _common_log_messages().get(self.type, []): # if it starts with an asterisk then check the whole message rather # than just the start

1 0