- tor-commits - lists.torproject.org

[translation/tor-launcher-network-settings_completed] Update translations for tor-launcher-network-settings_completed
by translation＠torproject.org 14 Mar '16

14 Mar '16

commit be741f531a6be975b7042fafbc9ddf00916994b5 Author: Translation commit bot <translation(a)torproject.org> Date: Mon Mar 14 06:46:00 2016 +0000 Update translations for tor-launcher-network-settings_completed --- hr_HR/network-settings.dtd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hr_HR/network-settings.dtd b/hr_HR/network-settings.dtd index 510c73a..038a4ac 100644 --- a/hr_HR/network-settings.dtd +++ b/hr_HR/network-settings.dtd @@ -24,7 +24,7 @@ internetskoj vezi ovog računala."> <!ENTITY torSettings.proxyQuestion "Treba li ovo računalo koristiti lokalni proxy da pristupi Internetu?">  <!ENTITY torSettings.proxyExplanation1 "U većini slučaja lokalni proxy nije potreban, ali može biti kad se spajate kroz mržu tvrtke, škole ili fakulteta."> -<!ENTITY torSettings.proxyExplanation2 "Ako niste sigurni kako odgovoriti na ovo pitanje, pogledajte Internet postavke u drugom pregledniku i provjerite mrežne postavke Vašeg sustava kako bi vidjeli da li je potreban lokalni proxy."> +<!ENTITY torSettings.proxyExplanation2 "Ako niste sigurni kako odgovoriti na ovo pitanje, pogledajte Internet postavke u drugom pregledniku ili provjerite mrežne postavke Vašeg sustava kako bi vidjeli da li je potreban lokalni proxy."> <!ENTITY torSettings.enterProxy "Unesite postavke proxyj-a."> <!ENTITY torSettings.bridgePageTitle "Konfiguracija Tor mostova"> <!ENTITY torSettings.bridgeQuestion "Blokira li Vaš davetelj Internet usluge (ISP) ili drugačije cenzurira veze

1 0

[translation/tor-launcher-network-settings] Update translations for tor-launcher-network-settings
by translation＠torproject.org 14 Mar '16

14 Mar '16

commit 3a8f786468ba672023b95a002c4744386d984a0d Author: Translation commit bot <translation(a)torproject.org> Date: Mon Mar 14 06:45:56 2016 +0000 Update translations for tor-launcher-network-settings --- hr_HR/network-settings.dtd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hr_HR/network-settings.dtd b/hr_HR/network-settings.dtd index 510c73a..038a4ac 100644 --- a/hr_HR/network-settings.dtd +++ b/hr_HR/network-settings.dtd @@ -24,7 +24,7 @@ internetskoj vezi ovog računala."> <!ENTITY torSettings.proxyQuestion "Treba li ovo računalo koristiti lokalni proxy da pristupi Internetu?">  <!ENTITY torSettings.proxyExplanation1 "U većini slučaja lokalni proxy nije potreban, ali može biti kad se spajate kroz mržu tvrtke, škole ili fakulteta."> -<!ENTITY torSettings.proxyExplanation2 "Ako niste sigurni kako odgovoriti na ovo pitanje, pogledajte Internet postavke u drugom pregledniku i provjerite mrežne postavke Vašeg sustava kako bi vidjeli da li je potreban lokalni proxy."> +<!ENTITY torSettings.proxyExplanation2 "Ako niste sigurni kako odgovoriti na ovo pitanje, pogledajte Internet postavke u drugom pregledniku ili provjerite mrežne postavke Vašeg sustava kako bi vidjeli da li je potreban lokalni proxy."> <!ENTITY torSettings.enterProxy "Unesite postavke proxyj-a."> <!ENTITY torSettings.bridgePageTitle "Konfiguracija Tor mostova"> <!ENTITY torSettings.bridgeQuestion "Blokira li Vaš davetelj Internet usluge (ISP) ili drugačije cenzurira veze

1 0

[translation/tor-launcher-progress_completed] Update translations for tor-launcher-progress_completed
by translation＠torproject.org 14 Mar '16

14 Mar '16

commit 3e0eafa4b5fcdafdbb359e559e3f26e2ca116b7c Author: Translation commit bot <translation(a)torproject.org> Date: Mon Mar 14 06:45:52 2016 +0000 Update translations for tor-launcher-progress_completed --- hr_HR/progress.dtd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hr_HR/progress.dtd b/hr_HR/progress.dtd index f3a11aa..da9a846 100644 --- a/hr_HR/progress.dtd +++ b/hr_HR/progress.dtd @@ -1,4 +1,4 @@ <!ENTITY torprogress.dialog.title "Tor Status"> <!ENTITY torprogress.openSettings "Otvori Postavke"> <!ENTITY torprogress.heading "Spajanje na Tor mrežu"> -<!ENTITY torprogress.pleaseWait "Molimo pričekajte dok ne ustanovimo vezu s Tor mrežom.  Ovo bi moglo potrajati nekoliko minuta."> +<!ENTITY torprogress.pleaseWait "Molimo pričekajte dok ne uspostavimo vezu s Tor mrežom.  To bi moglo potrajati nekoliko minuta.">

1 0

[translation/tor-launcher-progress] Update translations for tor-launcher-progress
by translation＠torproject.org 14 Mar '16

14 Mar '16

commit 756c4c83d1cfb6dfa3f12009639d1c27e08c8ca3 Author: Translation commit bot <translation(a)torproject.org> Date: Mon Mar 14 06:45:48 2016 +0000 Update translations for tor-launcher-progress --- hr_HR/progress.dtd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hr_HR/progress.dtd b/hr_HR/progress.dtd index f3a11aa..da9a846 100644 --- a/hr_HR/progress.dtd +++ b/hr_HR/progress.dtd @@ -1,4 +1,4 @@ <!ENTITY torprogress.dialog.title "Tor Status"> <!ENTITY torprogress.openSettings "Otvori Postavke"> <!ENTITY torprogress.heading "Spajanje na Tor mrežu"> -<!ENTITY torprogress.pleaseWait "Molimo pričekajte dok ne ustanovimo vezu s Tor mrežom.  Ovo bi moglo potrajati nekoliko minuta."> +<!ENTITY torprogress.pleaseWait "Molimo pričekajte dok ne uspostavimo vezu s Tor mrežom.  To bi moglo potrajati nekoliko minuta.">

1 0

[translation/mat-gui_completed] Update translations for mat-gui_completed
by translation＠torproject.org 14 Mar '16

14 Mar '16

commit 7241ec9a549fde5026b07ff222fb53de1215e7a5 Author: Translation commit bot <translation(a)torproject.org> Date: Mon Mar 14 06:45:41 2016 +0000 Update translations for mat-gui_completed --- hr_HR.po | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hr_HR.po b/hr_HR.po index 0b0c0e2..a3539c9 100644 --- a/hr_HR.po +++ b/hr_HR.po @@ -12,7 +12,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2016-02-10 23:06+0100\n" -"PO-Revision-Date: 2016-03-05 09:59+0000\n" +"PO-Revision-Date: 2016-03-14 06:44+0000\n" "Last-Translator: Igor <lyricaltumor(a)gmail.com>\n" "Language-Team: Croatian (Croatia) (http://www.transifex.com/otf/torproject/language/hr_HR/)\n" "MIME-Version: 1.0\n"

1 0

[translation/mat-gui] Update translations for mat-gui
by translation＠torproject.org 14 Mar '16

14 Mar '16

commit 9b06fc2d8d01b812815a7cf9c6bb87b4bf9c82a8 Author: Translation commit bot <translation(a)torproject.org> Date: Mon Mar 14 06:45:37 2016 +0000 Update translations for mat-gui --- hr_HR.po | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hr_HR.po b/hr_HR.po index 0b0c0e2..a3539c9 100644 --- a/hr_HR.po +++ b/hr_HR.po @@ -12,7 +12,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2016-02-10 23:06+0100\n" -"PO-Revision-Date: 2016-03-05 09:59+0000\n" +"PO-Revision-Date: 2016-03-14 06:44+0000\n" "Last-Translator: Igor <lyricaltumor(a)gmail.com>\n" "Language-Team: Croatian (Croatia) (http://www.transifex.com/otf/torproject/language/hr_HR/)\n" "MIME-Version: 1.0\n"

1 0

[translation/https_everywhere_completed] Update translations for https_everywhere_completed
by translation＠torproject.org 14 Mar '16

14 Mar '16

commit 0f93c2dc49c05117ebf7770f587da9f4df5bc345 Author: Translation commit bot <translation(a)torproject.org> Date: Mon Mar 14 06:45:21 2016 +0000 Update translations for https_everywhere_completed --- hr_HR/https-everywhere.dtd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hr_HR/https-everywhere.dtd b/hr_HR/https-everywhere.dtd index cd77471..62dc1d4 100644 --- a/hr_HR/https-everywhere.dtd +++ b/hr_HR/https-everywhere.dtd @@ -7,7 +7,7 @@ <!ENTITY https-everywhere.about.librarians "Autori biblioteke skupova pravila"> <!ENTITY https-everywhere.about.thanks "Zahvala"> <!ENTITY https-everywhere.about.many_contributors "Puno, puno kontributora, uključujući"> -<!ENTITY https-everywhere.about.noscript "Također, dijelovi HTTPS Everywhere su zasnovani na kodu iz NoScript-a, autora Giorgio Maone i ostalih. Zahvalni smo im za odličan rad!"> +<!ENTITY https-everywhere.about.noscript "Također, dijelovi HTTPS Everywhere su zasnovani na kodu iz NoScript-a, autora Giorgioa Maonea i ostalih. Zahvalni smo im za odličan rad!"> <!ENTITY https-everywhere.about.contribute "Ako Vam se sviđa HTTPS Everywhere, mogli biste razmotriti"> <!ENTITY https-everywhere.about.donate_tor "doniranje Tor-u"> <!ENTITY https-everywhere.about.tor_lang_code "en">

1 0

[translation/https_everywhere] Update translations for https_everywhere
by translation＠torproject.org 14 Mar '16

14 Mar '16

commit eb004c519eaf45cac6e8096225eae826da4317fd Author: Translation commit bot <translation(a)torproject.org> Date: Mon Mar 14 06:45:15 2016 +0000 Update translations for https_everywhere --- hr_HR/https-everywhere.dtd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hr_HR/https-everywhere.dtd b/hr_HR/https-everywhere.dtd index cd77471..62dc1d4 100644 --- a/hr_HR/https-everywhere.dtd +++ b/hr_HR/https-everywhere.dtd @@ -7,7 +7,7 @@ <!ENTITY https-everywhere.about.librarians "Autori biblioteke skupova pravila"> <!ENTITY https-everywhere.about.thanks "Zahvala"> <!ENTITY https-everywhere.about.many_contributors "Puno, puno kontributora, uključujući"> -<!ENTITY https-everywhere.about.noscript "Također, dijelovi HTTPS Everywhere su zasnovani na kodu iz NoScript-a, autora Giorgio Maone i ostalih. Zahvalni smo im za odličan rad!"> +<!ENTITY https-everywhere.about.noscript "Također, dijelovi HTTPS Everywhere su zasnovani na kodu iz NoScript-a, autora Giorgioa Maonea i ostalih. Zahvalni smo im za odličan rad!"> <!ENTITY https-everywhere.about.contribute "Ako Vam se sviđa HTTPS Everywhere, mogli biste razmotriti"> <!ENTITY https-everywhere.about.donate_tor "doniranje Tor-u"> <!ENTITY https-everywhere.about.tor_lang_code "en">

1 0

[nyx/master] Merge ui_tools.py into curses.py
by atagar＠torproject.org 14 Mar '16

14 Mar '16

commit aa58b0d3e2b032e85c1295028ee5d3865bd74508 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Mar 12 16:32:15 2016 -0800 Merge ui_tools.py into curses.py Merging our old util in, revising the functions (especially the Scroller) along the way. The draw_box() method better fits with the panel for now. --- nyx/controller.py | 5 +- nyx/curses.py | 214 ++++++++++++++++++++++++++++++++++++++ nyx/panel/config.py | 46 ++++----- nyx/panel/connection.py | 36 +++---- nyx/panel/log.py | 17 ++- nyx/panel/torrc.py | 29 ++++-- nyx/popups.py | 14 ++- nyx/starter.py | 4 +- nyx/util/__init__.py | 1 - nyx/util/panel.py | 29 ++++++ nyx/util/ui_tools.py | 268 ------------------------------------------------ 11 files changed, 320 insertions(+), 343 deletions(-) diff --git a/nyx/controller.py b/nyx/controller.py index 1478779..f188bb1 100644 --- a/nyx/controller.py +++ b/nyx/controller.py @@ -9,6 +9,7 @@ import time import curses import threading +import nyx.curses import nyx.menu.menu import nyx.popups import nyx.util.tracker @@ -25,7 +26,7 @@ import stem from stem.util import conf, log from nyx.curses import NORMAL, BOLD, HIGHLIGHT -from nyx.util import panel, tor_controller, ui_tools +from nyx.util import panel, tor_controller NYX_CONTROLLER = None @@ -389,7 +390,7 @@ def start_nyx(stdscr): control = get_controller() if not CONFIG['features.acsSupport']: - ui_tools.disable_acs() + nyx.curses.disable_acs() # provides notice about any unused config keys diff --git a/nyx/curses.py b/nyx/curses.py index 1280fc8..af3484b 100644 --- a/nyx/curses.py +++ b/nyx/curses.py @@ -14,6 +14,17 @@ if we want Windows support in the future too. get_color_override - provides color we override requests with set_color_override - sets color we override requests with + disable_acs - renders replacements for ACS characters + is_wide_characters_supported - checks if curses supports wide character + + Scroller - scrolls content with keyboard navigation + |- location - present scroll location + +- handle_key - moves scroll based on user input + + CursorScroller - scrolls content with a cursor for selecting items + |- selection - present selection and scroll location + +- handle_key - moves cursor based on user input + .. data:: Color (enum) Terminal colors. @@ -51,6 +62,7 @@ import curses import stem.util.conf import stem.util.enum +import stem.util.system from nyx.util import msg, log @@ -189,3 +201,205 @@ def _color_attr(): COLOR_ATTR = DEFAULT_COLOR_ATTR return COLOR_ATTR + + +def disable_acs(): + """ + Replaces ACS characters used for showing borders. This can be preferable if + curses is `unable to render them + <https://www.atagar.com/arm/images/acs_display_failure.png>`_. + """ + + for item in curses.__dict__: + if item.startswith('ACS_'): + curses.__dict__[item] = ord('+') + + # replace common border pipe cahracters + + curses.ACS_SBSB = ord('|') + curses.ACS_VLINE = ord('|') + curses.ACS_BSBS = ord('-') + curses.ACS_HLINE = ord('-') + + +def is_wide_characters_supported(): + """ + Checks if our version of curses has wide character support. This is required + to print unicode. + + :returns: **bool** that's **True** if curses supports wide characters, and + **False** if it either can't or this can't be determined + """ + + try: + # Gets the dynamic library used by the interpretor for curses. This uses + # 'ldd' on Linux or 'otool -L' on OSX. + # + # atagar@fenrir:~/Desktop$ ldd /usr/lib/python2.6/lib-dynload/_curses.so + # linux-gate.so.1 => (0x00a51000) + # libncursesw.so.5 => /lib/libncursesw.so.5 (0x00faa000) + # libpthread.so.0 => /lib/tls/i686/cmov/libpthread.so.0 (0x002f1000) + # libc.so.6 => /lib/tls/i686/cmov/libc.so.6 (0x00158000) + # libdl.so.2 => /lib/tls/i686/cmov/libdl.so.2 (0x00398000) + # /lib/ld-linux.so.2 (0x00ca8000) + # + # atagar$ otool -L /System/Library/Frameworks/Python.framework/Versions/2.5/lib/python2.5/lib-dynload/_curses.so + # /System/Library/Frameworks/Python.framework/Versions/2.5/lib/python2.5/lib-dynload/_curses.so: + # /usr/lib/libncurses.5.4.dylib (compatibility version 5.4.0, current version 5.4.0) + # /usr/lib/libgcc_s.1.dylib (compatibility version 1.0.0, current version 1.0.0) + # /usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 111.1.6) + + import _curses + + if stem.util.system.is_available('ldd'): + return 'libncursesw' in '\n'.join(lib_dependency_lines = stem.util.system.call('ldd %s' % _curses.__file__)) + elif stem.util.system.is_available('otool'): + return 'libncursesw' in '\n'.join(lib_dependency_lines = stem.util.system.call('otool -L %s' % _curses.__file__)) + except: + pass + + return False + + +class Scroller(object): + """ + Simple scroller that provides keyboard navigation of content. + """ + + def __init__(self): + self._location = 0 + + def location(self, content_height = None, page_height = None): + """ + Provides the position we've scrolled to. + + If a **content_height** and **page_height** are provided this ensures our + scroll position falls within a valid range. This should be done when the + content changes or panel resized. + + :param int content_height: height of the content being renered + :param int page_height: height visible on the page + + :returns: **int** position we've scrolled to + """ + + if content_height is not None and page_height is not None: + self._location = max(0, min(self._location, content_height - page_height)) + + return self._location + + def handle_key(self, key, content_height, page_height): + """ + Moves scrolling location according to the given input... + + * up / down - scrolls one position up or down + * page up / page down - scrolls by the page_height + * home / end - moves to the top or bottom + + :param nyx.util.panel.KeyInput key: pressed key + :param int content_height: height of the content being renered + :param int page_height: height visible on the page + + :returns: **bool** that's **True** if the scrolling position changed and + **False** otherwise + """ + + new_location = _scroll_position(self._location, key, content_height, page_height, False) + + if new_location != self._location: + self._location = new_location + return True + else: + return False + + +class CursorScroller(object): + """ + Scroller that tracks a cursor's position. + """ + + def __init__(self): + self._location = 0 + + # We track the cursor location by the item we have selected, so it stays + # selected as the content changes. We also keep track of its last location + # so we can fall back to that if it disappears. + + self._cursor_location = 0 + self._cursor_selection = None + + def selection(self, content, page_height = None): + """ + Provides the item from the content that's presently selected. If provided + the height of our page this provides the scroll position as well... + + :: + + selected, scroll = my_scroller.selection(content, page_height) + + :param list content: content the scroller is tracking + :param int page_height: height visible on the page + + :returns: **tuple** of the form **(cursor, scroll)**, the cursor is + **None** if content is empty + """ + + content = list(content) # shallow copy for thread safety + + if not content: + self._cursor_location = 0 + self._cursor_selection = None + return None if page_height is None else None, 0 + + if self._cursor_selection in content: + # moves cursor location to track the selection + self._cursor_location = content.index(self._cursor_selection) + else: + # select the next closest entry + self._cursor_location = max(0, min(self._cursor_location, len(content) - 1)) + self._cursor_selection = content[self._cursor_location] + + # ensure our cursor is visible + + if page_height: + if self._cursor_location < self._location: + self._location = self._cursor_location + elif self._cursor_location > self._location + page_height - 1: + self._location = self._cursor_location - page_height + 1 + + if page_height is None: + return self._cursor_selection + else: + return self._cursor_selection, self._location + + def handle_key(self, key, content, page_height): + self.selection(content, page_height) # reset cursor position + new_location = _scroll_position(self._cursor_location, key, len(content), page_height, True) + + if new_location != self._cursor_location: + self._cursor_location = new_location + self._cursor_selection = content[new_location] + + return True + else: + return False + + +def _scroll_position(location, key, content_height, page_height, is_cursor): + if key.match('up'): + shift = -1 + elif key.match('down'): + shift = 1 + elif key.match('page_up'): + shift = -page_height + 1 if is_cursor else -page_height + elif key.match('page_down'): + shift = page_height - 1 if is_cursor else page_height + elif key.match('home'): + shift = -content_height + elif key.match('end'): + shift = content_height + else: + return location + + max_position = content_height - 1 if is_cursor else content_height - page_height + return max(0, min(location + shift, max_position)) diff --git a/nyx/panel/config.py b/nyx/panel/config.py index fd529ca..2c09a35 100644 --- a/nyx/panel/config.py +++ b/nyx/panel/config.py @@ -7,13 +7,14 @@ import curses import os import nyx.controller +import nyx.curses import nyx.popups import stem.control import stem.manual from nyx.curses import GREEN, CYAN, WHITE, NORMAL, BOLD, HIGHLIGHT -from nyx.util import DATA_DIR, panel, tor_controller, ui_tools +from nyx.util import DATA_DIR, panel, tor_controller from stem.util import conf, enum, log, str_tools @@ -120,7 +121,7 @@ class ConfigPanel(panel.Panel): panel.Panel.__init__(self, stdscr, 'configuration', 0) self._contents = [] - self._scroller = ui_tools.Scroller(True) + self._scroller = nyx.curses.CursorScroller() self._sort_order = CONFIG['features.config.order'] self._show_all = False # show all options, or just the important ones @@ -237,23 +238,23 @@ class ConfigPanel(panel.Panel): if is_changed: self.redraw(True) elif key.is_selection(): - selection = self._scroller.get_cursor_selection(self._get_config_options()) - initial_value = selection.value() if selection.is_set() else '' - new_value = nyx.popups.input_prompt('%s Value (esc to cancel): ' % selection.name, initial_value) + selected = self._scroller.selection(self._get_config_options()) + initial_value = selected.value() if selected.is_set() else '' + new_value = nyx.popups.input_prompt('%s Value (esc to cancel): ' % selected.name, initial_value) if new_value != initial_value: try: - if selection.value_type == 'Boolean': + if selected.value_type == 'Boolean': # if the value's a boolean then allow for 'true' and 'false' inputs if new_value.lower() == 'true': new_value = '1' elif new_value.lower() == 'false': new_value = '0' - elif selection.value_type == 'LineList': + elif selected.value_type == 'LineList': new_value = new_value.split(',') # set_conf accepts list inputs - tor_controller().set_conf(selection.name, new_value) + tor_controller().set_conf(selected.name, new_value) self.redraw(True) except Exception as exc: nyx.popups.show_msg('%s (press any key)' % exc) @@ -283,12 +284,11 @@ class ConfigPanel(panel.Panel): def draw(self, width, height): contents = self._get_config_options() - selection = self._scroller.get_cursor_selection(contents) - scroll_location = self._scroller.get_scroll_location(contents, height - DETAILS_HEIGHT) + selected, scroll = self._scroller.selection(contents, height - DETAILS_HEIGHT) is_scrollbar_visible = len(contents) > height - DETAILS_HEIGHT - if selection is not None: - self._draw_selection_details(selection, width) + if selected is not None: + self._draw_selection_details(selected, width) if self.is_title_visible(): hidden_msg = "press 'a' to hide most options" if self._show_all else "press 'a' to show all options" @@ -298,9 +298,9 @@ class ConfigPanel(panel.Panel): if is_scrollbar_visible: scroll_offset = 3 - self.add_scroll_bar(scroll_location, scroll_location + height - DETAILS_HEIGHT, len(contents), DETAILS_HEIGHT) + self.add_scroll_bar(scroll, scroll + height - DETAILS_HEIGHT, len(contents), DETAILS_HEIGHT) - if selection is not None: + if selected is not None: self.addch(DETAILS_HEIGHT - 1, 1, curses.ACS_TTEE) # Description column can grow up to eighty characters. After that any extra @@ -314,10 +314,10 @@ class ConfigPanel(panel.Panel): else: value_width = VALUE_WIDTH - for i, entry in enumerate(contents[scroll_location:]): + for i, entry in enumerate(contents[scroll:]): attr = [CONFIG['attr.config.category_color'].get(entry.manual.category, WHITE)] attr.append(BOLD if entry.is_set() else NORMAL) - attr.append(HIGHLIGHT if entry == selection else NORMAL) + attr.append(HIGHLIGHT if entry == selected else NORMAL) option_label = str_tools.crop(entry.name, NAME_WIDTH).ljust(NAME_WIDTH + 1) value_label = str_tools.crop(entry.value(), value_width).ljust(value_width + 1) @@ -331,18 +331,18 @@ class ConfigPanel(panel.Panel): def _get_config_options(self): return self._contents if self._show_all else filter(lambda entry: stem.manual.is_important(entry.name) or entry.is_set(), self._contents) - def _draw_selection_details(self, selection, width): + def _draw_selection_details(self, selected, width): """ Shows details of the currently selected option. """ - description = 'Description: %s' % (selection.manual.description) - attr = ', '.join(('custom' if selection.is_set() else 'default', selection.value_type, 'usage: %s' % selection.manual.usage)) - selected_color = CONFIG['attr.config.category_color'].get(selection.manual.category, WHITE) - ui_tools.draw_box(self, 0, 0, width, DETAILS_HEIGHT) + description = 'Description: %s' % (selected.manual.description) + attr = ', '.join(('custom' if selected.is_set() else 'default', selected.value_type, 'usage: %s' % selected.manual.usage)) + selected_color = CONFIG['attr.config.category_color'].get(selected.manual.category, WHITE) + self.draw_box(0, 0, width, DETAILS_HEIGHT) - self.addstr(1, 2, '%s (%s Option)' % (selection.name, selection.manual.category), selected_color, BOLD) - self.addstr(2, 2, 'Value: %s (%s)' % (selection.value(), str_tools.crop(attr, width - len(selection.value()) - 13)), selected_color, BOLD) + self.addstr(1, 2, '%s (%s Option)' % (selected.name, selected.manual.category), selected_color, BOLD) + self.addstr(2, 2, 'Value: %s (%s)' % (selected.value(), str_tools.crop(attr, width - len(selected.value()) - 13)), selected_color, BOLD) for i in range(DETAILS_HEIGHT - 4): if not description: diff --git a/nyx/panel/connection.py b/nyx/panel/connection.py index 2c8fce9..7e01674 100644 --- a/nyx/panel/connection.py +++ b/nyx/panel/connection.py @@ -9,12 +9,12 @@ import curses import itertools import threading +import nyx.curses import nyx.popups import nyx.util.tracker -import nyx.util.ui_tools from nyx.curses import WHITE, NORMAL, BOLD, HIGHLIGHT -from nyx.util import panel, tor_controller, ui_tools +from nyx.util import panel, tor_controller from stem.control import Listener from stem.util import datetime_to_unix, conf, connection, enum, str_tools @@ -263,7 +263,7 @@ class ConnectionPanel(panel.Panel, threading.Thread): threading.Thread.__init__(self) self.setDaemon(True) - self._scroller = ui_tools.Scroller(True) + self._scroller = nyx.curses.CursorScroller() self._entries = [] # last fetched display entries self._show_details = False # presents the details panel if true self._sort_order = CONFIG['features.connection.order'] @@ -338,10 +338,10 @@ class ConnectionPanel(panel.Panel, threading.Thread): resolver = connection_tracker.get_custom_resolver() selected_index = 0 if resolver is None else options.index(resolver) - selection = nyx.popups.show_menu('Connection Resolver:', options, selected_index) + selected = nyx.popups.show_menu('Connection Resolver:', options, selected_index) - if selection != -1: - connection_tracker.set_custom_resolver(None if selection == 0 else options[selection]) + if selected != -1: + connection_tracker.set_custom_resolver(None if selected == 0 else options[selected]) elif key.match('d'): self.set_title_visible(False) self.redraw(True) @@ -349,16 +349,16 @@ class ConnectionPanel(panel.Panel, threading.Thread): while True: lines = list(itertools.chain.from_iterable([entry.get_lines() for entry in entries])) - selection = self._scroller.get_cursor_selection(lines) + selected = self._scroller.selection(lines) - if not selection: + if not selected: break def is_close_key(key): return key.is_selection() or key.match('d') or key.match('left') or key.match('right') - color = CONFIG['attr.connection.category_color'].get(selection.entry.get_type(), WHITE) - key = nyx.popups.show_descriptor_popup(selection.fingerprint, color, self.max_x, is_close_key) + color = CONFIG['attr.connection.category_color'].get(selected.entry.get_type(), WHITE) + key = nyx.popups.show_descriptor_popup(selected.fingerprint, color, self.max_x, is_close_key) if not key or key.is_selection() or key.match('d'): break # closes popup @@ -445,7 +445,9 @@ class ConnectionPanel(panel.Panel, threading.Thread): entries = self._entries lines = list(itertools.chain.from_iterable([entry.get_lines() for entry in entries])) - selected = self._scroller.get_cursor_selection(lines) + is_showing_details = self._show_details and lines + details_offset = DETAILS_HEIGHT + 1 if is_showing_details else 0 + selected, scroll = self._scroller.selection(lines, height - details_offset - 1) if self.is_paused(): current_time = self.get_pause_time() @@ -454,12 +456,8 @@ class ConnectionPanel(panel.Panel, threading.Thread): else: current_time = time.time() - is_showing_details = self._show_details and selected - details_offset = DETAILS_HEIGHT + 1 if is_showing_details else 0 - is_scrollbar_visible = len(lines) > height - details_offset - 1 scroll_offset = 2 if is_scrollbar_visible else 0 - scroll_location = self._scroller.get_scroll_location(lines, height - details_offset - 1) if self.is_title_visible(): self._draw_title(entries, self._show_details) @@ -468,10 +466,10 @@ class ConnectionPanel(panel.Panel, threading.Thread): self._draw_details(selected, width, is_scrollbar_visible) if is_scrollbar_visible: - self.add_scroll_bar(scroll_location, scroll_location + height - details_offset - 1, len(lines), 1 + details_offset) + self.add_scroll_bar(scroll, scroll + height - details_offset - 1, len(lines), 1 + details_offset) - for line_number in range(scroll_location, len(lines)): - y = line_number + details_offset + 1 - scroll_location + for line_number in range(scroll, len(lines)): + y = line_number + details_offset + 1 - scroll self._draw_line(scroll_offset, y, lines[line_number], lines[line_number] == selected, width - scroll_offset, current_time) if y >= height: @@ -547,7 +545,7 @@ class ConnectionPanel(panel.Panel, threading.Thread): # draw the border, with a 'T' pipe if connecting with the scrollbar - ui_tools.draw_box(self, 0, 0, width, DETAILS_HEIGHT + 2) + self.draw_box(0, 0, width, DETAILS_HEIGHT + 2) if is_scrollbar_visible: self.addch(DETAILS_HEIGHT + 1, 1, curses.ACS_TTEE) diff --git a/nyx/panel/log.py b/nyx/panel/log.py index e674ac1..ba22d3c 100644 --- a/nyx/panel/log.py +++ b/nyx/panel/log.py @@ -11,11 +11,12 @@ import threading import stem.response.events import nyx.arguments +import nyx.curses import nyx.popups import nyx.util.log from nyx.curses import GREEN, YELLOW, WHITE, NORMAL, BOLD, HIGHLIGHT -from nyx.util import join, panel, tor_controller, ui_tools +from nyx.util import join, panel, tor_controller from stem.util import conf, log @@ -76,6 +77,7 @@ class LogPanel(panel.Panel, threading.Thread): self.set_pause_attr('_event_log') + self._scroller = nyx.curses.Scroller() self._halt = False # terminates thread if true self._pause_condition = threading.Condition() self._has_new_event = False @@ -96,7 +98,6 @@ class LogPanel(panel.Panel, threading.Thread): log.info(str(exc)) self._last_content_height = len(self._event_log) # height of the rendered content when last drawn - self._scroll = 0 # merge NYX_LOGGER into us, and listen for its future events @@ -223,10 +224,9 @@ class LogPanel(panel.Panel, threading.Thread): def handle_key(self, key): if key.is_scroll(): page_height = self.get_preferred_size()[0] - 1 - new_scroll = ui_tools.get_scroll_position(key, self._scroll, page_height, self._last_content_height) + is_changed = self._scroller.handle_key(key, self._last_content_height, page_height) - if self._scroll != new_scroll: - self._scroll = new_scroll + if is_changed: self.redraw(True) elif key.match('u'): self.set_duplicate_visability(not self._show_duplicates) @@ -271,12 +271,11 @@ class LogPanel(panel.Panel, threading.Thread): ] def draw(self, width, height): - self._scroll = max(0, min(self._scroll, self._last_content_height - height + 1)) + scroll = self._scroller.location(self._last_content_height, height) event_log = list(self.get_attr('_event_log')) event_filter = self._filter.clone() event_types = list(self._event_types) - scroll = int(self._scroll) last_content_height = self._last_content_height show_duplicates = self._show_duplicates @@ -309,7 +308,7 @@ class LogPanel(panel.Panel, threading.Thread): for entry in day_to_entries[day]: y = self._draw_entry(x, y, width, entry, show_duplicates) - ui_tools.draw_box(self, original_y, x - 1, width - x + 1, y - original_y + 1, YELLOW, BOLD) + self.draw_box(original_y, x - 1, width - x + 1, y - original_y + 1, YELLOW, BOLD) time_label = time.strftime(' %B %d, %Y ', time.localtime(day_to_entries[day][0].timestamp)) self.addstr(original_y, x + 1, time_label, YELLOW, BOLD) @@ -330,7 +329,7 @@ class LogPanel(panel.Panel, threading.Thread): if content_height_delta >= CONTENT_HEIGHT_REDRAW_THRESHOLD: force_redraw_reason = 'estimate was off by %i' % content_height_delta - elif new_content_height > height and self._scroll + height - 1 > new_content_height: + elif new_content_height > height and scroll + height - 1 > new_content_height: force_redraw_reason = 'scrolled off the bottom of the page' elif not is_scrollbar_visible and new_content_height > height - 1: force_redraw_reason = "scroll bar wasn't previously visible" diff --git a/nyx/panel/torrc.py b/nyx/panel/torrc.py index ae26c83..bf10efb 100644 --- a/nyx/panel/torrc.py +++ b/nyx/panel/torrc.py @@ -3,9 +3,12 @@ Panel displaying the torrc or nyxrc with the validation done against it. """ import math +import string + +import nyx.curses from nyx.curses import RED, GREEN, YELLOW, CYAN, WHITE, BOLD, HIGHLIGHT -from nyx.util import expand_path, msg, panel, tor_controller, ui_tools +from nyx.util import expand_path, msg, panel, tor_controller from stem import ControllerError from stem.control import State @@ -20,7 +23,7 @@ class TorrcPanel(panel.Panel): def __init__(self, stdscr): panel.Panel.__init__(self, stdscr, 'torrc', 0) - self._scroll = 0 + self._scroller = nyx.curses.Scroller() self._show_line_numbers = True # shows left aligned line numbers self._show_comments = True # shows comments and extra whitespace self._last_content_height = 0 @@ -41,9 +44,14 @@ class TorrcPanel(panel.Panel): if event_type == State.RESET: try: self._torrc_location = expand_path(controller.get_info('config-file')) + contents = [] with open(self._torrc_location) as torrc_file: - self._torrc_content = [ui_tools.get_printable(line.replace('\t', ' ')).rstrip() for line in torrc_file.readlines()] + for line in torrc_file.readlines(): + line = line.replace('\t', ' ').replace('\xc2', "'").rstrip() + contents.append(filter(lambda char: char in string.printable, line)) + + self._torrc_content = contents except ControllerError as exc: self._torrc_load_error = msg('panel.torrc.unable_to_find_torrc', error = exc) self._torrc_location = None @@ -75,10 +83,9 @@ class TorrcPanel(panel.Panel): def handle_key(self, key): if key.is_scroll(): page_height = self.get_preferred_size()[0] - 1 - new_scroll = ui_tools.get_scroll_position(key, self._scroll, page_height, self._last_content_height) + is_changed = self._scroller.handle_key(key, self._last_content_height, page_height) - if self._scroll != new_scroll: - self._scroll = new_scroll + if is_changed: self.redraw(True) elif key.match('l'): self.set_line_number_visible(not self._show_line_numbers) @@ -101,12 +108,12 @@ class TorrcPanel(panel.Panel): ] def draw(self, width, height): + scroll = self._scroller.location(self._last_content_height, height) + if self._torrc_content is None: self.addstr(1, 0, self._torrc_load_error, RED, BOLD) new_content_height = 1 else: - self._scroll = max(0, min(self._scroll, self._last_content_height - height + 1)) - if not self._show_line_numbers: line_number_offset = 0 elif len(self._torrc_content) == 0: @@ -118,9 +125,9 @@ class TorrcPanel(panel.Panel): if self._last_content_height > height - 1: scroll_offset = 3 - self.add_scroll_bar(self._scroll, self._scroll + height - 1, self._last_content_height, 1) + self.add_scroll_bar(scroll, scroll + height - 1, self._last_content_height, 1) - y = 1 - self._scroll + y = 1 - scroll is_multiline = False # true if we're in the middle of a multiline torrc entry for line_number, line in enumerate(self._torrc_content): @@ -159,7 +166,7 @@ class TorrcPanel(panel.Panel): y += 1 - new_content_height = y + self._scroll - 1 + new_content_height = y + scroll - 1 if self.is_title_visible(): self.addstr(0, 0, ' ' * width) # clear line diff --git a/nyx/popups.py b/nyx/popups.py index 1a711c7..c05db05 100644 --- a/nyx/popups.py +++ b/nyx/popups.py @@ -9,10 +9,11 @@ import curses import operator import nyx.controller +import nyx.curses from nyx import __version__, __release_date__ from nyx.curses import RED, GREEN, YELLOW, CYAN, WHITE, NORMAL, BOLD, HIGHLIGHT -from nyx.util import tor_controller, panel, ui_tools +from nyx.util import tor_controller, panel NO_STATS_MSG = "Usage stats aren't available yet, press any key..." @@ -463,20 +464,17 @@ def show_descriptor_popup(fingerprint, color, max_width, is_close_key): if not popup: return None - scroll, redraw = 0, True + scroller, redraw = nyx.curses.Scroller(), True while True: if redraw: - _draw(popup, title, lines, color, scroll, show_line_numbers) + _draw(popup, title, lines, color, scroller.location(), show_line_numbers) redraw = False key = nyx.controller.get_controller().key_input() if key.is_scroll(): - new_scroll = ui_tools.get_scroll_position(key, scroll, height - 2, len(lines)) - - if scroll != new_scroll: - scroll, redraw = new_scroll, True + redraw = scroller.handle_key(key, len(lines), height - 2) elif is_close_key(key): return key @@ -560,7 +558,7 @@ def _draw(popup, title, lines, entry_color, scroll, show_line_numbers): if show_line_numbers: popup.addstr(y, 2, str(i + 1).rjust(line_number_width), LINE_NUMBER_COLOR, BOLD) - x, y = popup.addstr_wrap(y, width, keyword, width, offset, color, BOLD) + x, y = popup.addstr_wrap(y, 3 + line_number_width, keyword, width, offset, color, BOLD) x, y = popup.addstr_wrap(y, x + 1, value, width, offset, color) y += 1 diff --git a/nyx/starter.py b/nyx/starter.py index e46d494..83825e0 100644 --- a/nyx/starter.py +++ b/nyx/starter.py @@ -17,9 +17,9 @@ import time import nyx import nyx.arguments import nyx.controller +import nyx.curses import nyx.util.panel import nyx.util.tracker -import nyx.util.ui_tools import stem import stem.util.log @@ -237,7 +237,7 @@ def _use_unicode(config): is_lang_unicode = 'utf-' in os.getenv('LANG', '').lower() - if is_lang_unicode and nyx.util.ui_tools.is_wide_characters_supported(): + if is_lang_unicode and nyx.curses.is_wide_characters_supported(): locale.setlocale(locale.LC_ALL, '') diff --git a/nyx/util/__init__.py b/nyx/util/__init__.py index 1bc3d71..176753b 100644 --- a/nyx/util/__init__.py +++ b/nyx/util/__init__.py @@ -18,7 +18,6 @@ __all__ = [ 'log', 'panel', 'tracker', - 'ui_tools', ] TOR_CONTROLLER = None diff --git a/nyx/util/panel.py b/nyx/util/panel.py index 26b0af1..43e0236 100644 --- a/nyx/util/panel.py +++ b/nyx/util/panel.py @@ -769,6 +769,35 @@ class Panel(object): return recreate + def draw_box(self, top, left, width, height, *attributes): + """ + Draws a box in the panel with the given bounds. + + Arguments: + top - vertical position of the box's top + left - horizontal position of the box's left side + width - width of the drawn box + height - height of the drawn box + attr - text attributes + """ + + # draws the top and bottom + + self.hline(top, left + 1, width - 2, *attributes) + self.hline(top + height - 1, left + 1, width - 2, *attributes) + + # draws the left and right sides + + self.vline(top + 1, left, height - 2, *attributes) + self.vline(top + 1, left + width - 1, height - 2, *attributes) + + # draws the corners + + self.addch(top, left, curses.ACS_ULCORNER, *attributes) + self.addch(top, left + width - 1, curses.ACS_URCORNER, *attributes) + self.addch(top + height - 1, left, curses.ACS_LLCORNER, *attributes) + self.addch(top + height - 1, left + width - 1, curses.ACS_LRCORNER, *attributes) + class KeyInput(object): """ diff --git a/nyx/util/ui_tools.py b/nyx/util/ui_tools.py deleted file mode 100644 index ccea602..0000000 --- a/nyx/util/ui_tools.py +++ /dev/null @@ -1,268 +0,0 @@ -""" -Toolkit for working with curses. -""" - -import curses - -from curses.ascii import isprint - -from stem.util import system - - -def disable_acs(): - """ - Replaces the curses ACS characters. This can be preferable if curses is - unable to render them... - - http://www.atagar.com/nyx/images/acs_display_failure.png - """ - - for item in curses.__dict__: - if item.startswith('ACS_'): - curses.__dict__[item] = ord('+') - - # replace a few common border pipes that are better rendered as '|' or - # '-' instead - - curses.ACS_SBSB = ord('|') - curses.ACS_VLINE = ord('|') - curses.ACS_BSBS = ord('-') - curses.ACS_HLINE = ord('-') - - -def get_printable(line, keep_newlines = True): - """ - Provides the line back with non-printable characters stripped. - - :param str line: string to be processed - :param str keep_newlines: retains newlines if **True**, stripped otherwise - - :returns: **str** of the line with only printable content - """ - - line = line.replace('\xc2', "'") - line = filter(lambda char: isprint(char) or (keep_newlines and char == '\n'), line) - - return line - - -def draw_box(panel, top, left, width, height, *attributes): - """ - Draws a box in the panel with the given bounds. - - Arguments: - panel - panel in which to draw - top - vertical position of the box's top - left - horizontal position of the box's left side - width - width of the drawn box - height - height of the drawn box - attr - text attributes - """ - - # draws the top and bottom - - panel.hline(top, left + 1, width - 2, *attributes) - panel.hline(top + height - 1, left + 1, width - 2, *attributes) - - # draws the left and right sides - - panel.vline(top + 1, left, height - 2, *attributes) - panel.vline(top + 1, left + width - 1, height - 2, *attributes) - - # draws the corners - - panel.addch(top, left, curses.ACS_ULCORNER, *attributes) - panel.addch(top, left + width - 1, curses.ACS_URCORNER, *attributes) - panel.addch(top + height - 1, left, curses.ACS_LLCORNER, *attributes) - panel.addch(top + height - 1, left + width - 1, curses.ACS_LRCORNER, *attributes) - - -def get_scroll_position(key, position, page_height, content_height, is_cursor = False): - """ - Parses navigation keys, providing the new scroll possition the panel should - use. Position is always between zero and (content_height - page_height). This - handles the following keys: - Up / Down - scrolls a position up or down - Page Up / Page Down - scrolls by the page_height - Home - top of the content - End - bottom of the content - - This provides the input position if the key doesn't correspond to the above. - - Arguments: - key - keycode for the user's input - position - starting position - page_height - size of a single screen's worth of content - content_height - total lines of content that can be scrolled - is_cursor - tracks a cursor position rather than scroll if true - """ - - if key.is_scroll(): - shift = 0 - - if key.match('up'): - shift = -1 - elif key.match('down'): - shift = 1 - elif key.match('page_up'): - shift = -page_height + 1 if is_cursor else -page_height - elif key.match('page_down'): - shift = page_height - 1 if is_cursor else page_height - elif key.match('home'): - shift = -content_height - elif key.match('end'): - shift = content_height - - # returns the shift, restricted to valid bounds - - max_location = content_height - 1 if is_cursor else content_height - page_height - return max(0, min(position + shift, max_location)) - else: - return position - - -class Scroller: - """ - Tracks the scrolling position when there might be a visible cursor. This - expects that there is a single line displayed per an entry in the contents. - """ - - def __init__(self, is_cursor_enabled): - self.scroll_location, self.cursor_location = 0, 0 - self.cursor_selection = None - self.is_cursor_enabled = is_cursor_enabled - - def get_scroll_location(self, content, page_height): - """ - Provides the scrolling location, taking into account its cursor's location - content size, and page height. - - Arguments: - content - displayed content - page_height - height of the display area for the content - """ - - if content and page_height: - self.scroll_location = max(0, min(self.scroll_location, len(content) - page_height + 1)) - - if self.is_cursor_enabled: - self.get_cursor_selection(content) # resets the cursor location - - # makes sure the cursor is visible - - if self.cursor_location < self.scroll_location: - self.scroll_location = self.cursor_location - elif self.cursor_location > self.scroll_location + page_height - 1: - self.scroll_location = self.cursor_location - page_height + 1 - - # checks if the bottom would run off the content (this could be the - # case when the content's size is dynamic and entries are removed) - - if len(content) > page_height: - self.scroll_location = min(self.scroll_location, len(content) - page_height) - - return self.scroll_location - - def get_cursor_selection(self, content): - """ - Provides the selected item in the content. This is the same entry until - the cursor moves or it's no longer available (in which case it moves on to - the next entry). - - Arguments: - content - displayed content - """ - - # TODO: needs to handle duplicate entries when using this for the - # connection panel - - if not self.is_cursor_enabled: - return None - elif not content: - self.cursor_location, self.cursor_selection = 0, None - return None - - self.cursor_location = min(self.cursor_location, len(content) - 1) - - if self.cursor_selection is not None and self.cursor_selection in content: - # moves cursor location to track the selection - self.cursor_location = content.index(self.cursor_selection) - else: - # select the next closest entry - self.cursor_selection = content[self.cursor_location] - - return self.cursor_selection - - def handle_key(self, key, content, page_height): - """ - Moves either the scroll or cursor according to the given input. - - Arguments: - key - key code of user input - content - displayed content - page_height - height of the display area for the content - """ - - if self.is_cursor_enabled: - self.get_cursor_selection(content) # resets the cursor location - start_location = self.cursor_location - else: - start_location = self.scroll_location - - new_location = get_scroll_position(key, start_location, page_height, len(content), self.is_cursor_enabled) - - if start_location != new_location: - if self.is_cursor_enabled: - self.cursor_selection = content[new_location] - else: - self.scroll_location = new_location - - return True - else: - return False - - -def is_wide_characters_supported(): - """ - Checks if our version of curses has wide character support. This is required - to print unicode. - - :returns: **bool** that's **True** if curses supports wide characters, and - **False** if it either can't or this can't be determined - """ - - try: - # Gets the dynamic library used by the interpretor for curses. This uses - # 'ldd' on Linux or 'otool -L' on OSX. - # - # atagar@fenrir:~/Desktop$ ldd /usr/lib/python2.6/lib-dynload/_curses.so - # linux-gate.so.1 => (0x00a51000) - # libncursesw.so.5 => /lib/libncursesw.so.5 (0x00faa000) - # libpthread.so.0 => /lib/tls/i686/cmov/libpthread.so.0 (0x002f1000) - # libc.so.6 => /lib/tls/i686/cmov/libc.so.6 (0x00158000) - # libdl.so.2 => /lib/tls/i686/cmov/libdl.so.2 (0x00398000) - # /lib/ld-linux.so.2 (0x00ca8000) - # - # atagar$ otool -L /System/Library/Frameworks/Python.framework/Versions/2.5/lib/python2.5/lib-dynload/_curses.so - # /System/Library/Frameworks/Python.framework/Versions/2.5/lib/python2.5/lib-dynload/_curses.so: - # /usr/lib/libncurses.5.4.dylib (compatibility version 5.4.0, current version 5.4.0) - # /usr/lib/libgcc_s.1.dylib (compatibility version 1.0.0, current version 1.0.0) - # /usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 111.1.6) - - import _curses - - lib_dependency_lines = None - - if system.is_available('ldd'): - lib_dependency_lines = system.call('ldd %s' % _curses.__file__) - elif system.is_available('otool'): - lib_dependency_lines = system.call('otool -L %s' % _curses.__file__) - - if lib_dependency_lines: - for line in lib_dependency_lines: - if 'libncursesw' in line: - return True - except: - pass - - return False

1 0

[nyx/master] Merge nyx.util into the module
by atagar＠torproject.org 14 Mar '16

14 Mar '16

commit 9df034a5cd01fbad4aa17e9059cab6567d3e91f4 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Mar 13 19:03:10 2016 -0700 Merge nyx.util into the module As the saying goes, "everything's miscellaneous". Now that we've moved the panels out and dropped most of the utils we can merge the three that remain in. The log and tracker are done being overhauled, but clearly the panel still needs a lot of love. --- nyx/__init__.py | 136 +++++ nyx/arguments.py | 2 +- nyx/controller.py | 12 +- nyx/curses.py | 2 +- nyx/log.py | 536 ++++++++++++++++++ nyx/menu/actions.py | 7 +- nyx/panel/__init__.py | 839 ++++++++++++++++++++++++++++ nyx/panel/config.py | 7 +- nyx/panel/connection.py | 45 +- nyx/panel/graph.py | 19 +- nyx/panel/header.py | 10 +- nyx/panel/log.py | 39 +- nyx/panel/torrc.py | 2 +- nyx/popups.py | 16 +- nyx/starter.py | 10 +- nyx/tracker.py | 895 +++++++++++++++++++++++++++++ nyx/util/__init__.py | 146 ----- nyx/util/log.py | 537 ------------------ nyx/util/panel.py | 841 ---------------------------- nyx/util/tracker.py | 896 ------------------------------ run_tests.py | 6 +- setup.py | 2 +- test/__init__.py | 4 + test/expand_path.py | 21 + test/log/__init__.py | 8 + test/log/condense_runlevels.py | 13 + test/log/data/daybreak_deduplication | 37 ++ test/log/data/empty_file | 0 test/log/data/malformed_date | 21 + test/log/data/malformed_line | 21 + test/log/data/malformed_runlevel | 21 + test/log/data/multiple_tor_instances | 33 ++ test/log/data/tor_log | 21 + test/log/log_entry.py | 27 + test/log/log_group.py | 146 +++++ test/log/read_tor_log.py | 59 ++ test/tracker/__init__.py | 10 + test/tracker/connection_tracker.py | 115 ++++ test/tracker/daemon.py | 76 +++ test/tracker/port_usage_tracker.py | 117 ++++ test/tracker/resource_tracker.py | 143 +++++ test/util/__init__.py | 5 - test/util/expand_path.py | 21 - test/util/log/__init__.py | 8 - test/util/log/condense_runlevels.py | 13 - test/util/log/data/daybreak_deduplication | 37 -- test/util/log/data/empty_file | 0 test/util/log/data/malformed_date | 21 - test/util/log/data/malformed_line | 21 - test/util/log/data/malformed_runlevel | 21 - test/util/log/data/multiple_tor_instances | 33 -- test/util/log/data/tor_log | 21 - test/util/log/log_entry.py | 27 - test/util/log/log_group.py | 146 ----- test/util/log/read_tor_log.py | 59 -- test/util/tracker/__init__.py | 10 - test/util/tracker/connection_tracker.py | 115 ---- test/util/tracker/daemon.py | 76 --- test/util/tracker/port_usage_tracker.py | 117 ---- test/util/tracker/resource_tracker.py | 143 ----- 60 files changed, 3391 insertions(+), 3401 deletions(-) diff --git a/nyx/__init__.py b/nyx/__init__.py index 25da9dd..abb8aea 100644 --- a/nyx/__init__.py +++ b/nyx/__init__.py @@ -3,8 +3,15 @@ Tor curses monitoring application. """ import distutils.spawn +import os import sys +import stem.connection +import stem.control +import stem.util.conf +import stem.util.log +import stem.util.system + __version__ = '1.4.6-dev' __release_date__ = 'April 28, 2011' __author__ = 'Damian Johnson' @@ -15,10 +22,33 @@ __license__ = 'GPLv3' __all__ = [ 'arguments', 'controller', + 'curses', + 'log', + 'panel', 'popups', 'starter', + 'tracker', ] +TOR_CONTROLLER = None +BASE_DIR = os.path.sep.join(__file__.split(os.path.sep)[:-1]) +DATA_DIR = os.path.expanduser('~/.nyx') +TESTING = False + +# technically can change but we use this query a *lot* so needs to be cached + +stem.control.CACHEABLE_GETINFO_PARAMS = list(stem.control.CACHEABLE_GETINFO_PARAMS) + ['address'] + +# disable trace level messages about cache hits + +stem.control.LOG_CACHE_FETCHES = False + +try: + uses_settings = stem.util.conf.uses_settings('nyx', os.path.join(BASE_DIR, 'settings'), lazy_load = False) +except IOError as exc: + print("Unable to load nyx's internal configurations: %s" % exc) + sys.exit(1) + def main(): try: @@ -45,3 +75,109 @@ def main(): print('Unable to start nyx: %s' % exc) sys.exit(1) + + +def tor_controller(): + """ + Singleton for getting our tor controller connection. + + :returns: :class:`~stem.control.Controller` nyx is using + """ + + return TOR_CONTROLLER + + +def init_controller(*args, **kwargs): + """ + Sets the Controller used by nyx. This is a passthrough for Stem's + :func:`~stem.connection.connect` function. + + :returns: :class:`~stem.control.Controller` nyx is using + """ + + global TOR_CONTROLLER + TOR_CONTROLLER = stem.connection.connect(*args, **kwargs) + return TOR_CONTROLLER + + +@uses_settings +def expand_path(path, config): + """ + Expands relative paths and include our chroot if one was set. + + :param str path: path to be expanded + + :returns: **str** with the expanded path + """ + + if path is None: + return None + + try: + chroot = config.get('tor.chroot', '') + tor_cwd = stem.util.system.cwd(tor_controller().get_pid(None)) + return chroot + stem.util.system.expand_path(path, tor_cwd) + except IOError as exc: + stem.util.log.info('Unable to expand a relative path (%s): %s' % (path, exc)) + return path + + +def join(entries, joiner = ' ', size = None): + """ + Joins a series of strings similar to str.join(), but only up to a given size. + This returns an empty string if none of the entries will fit. For example... + + >>> join(['This', 'is', 'a', 'looooong', 'message'], size = 18) + 'This is a looooong' + + >>> join(['This', 'is', 'a', 'looooong', 'message'], size = 17) + 'This is a' + + >>> join(['This', 'is', 'a', 'looooong', 'message'], size = 2) + '' + + :param list entries: strings to be joined + :param str joiner: strings to join the entries with + :param int size: maximum length the result can be, there's no length + limitation if **None** + + :returns: **str** of the joined entries up to the given length + """ + + if size is None: + return joiner.join(entries) + + result = '' + + for entry in entries: + new_result = joiner.join((result, entry)) if result else entry + + if len(new_result) > size: + break + else: + result = new_result + + return result + + +@uses_settings +def msg(message, config, **attr): + """ + Provides the given message. + + :param str message: message handle to log + :param dict attr: attributes to format the message with + + :returns: **str** that was requested + """ + + try: + return config.get('msg.%s' % message).format(**attr) + except: + msg = 'BUG: We attempted to use an undefined string resource (%s)' % message + + if TESTING: + raise ValueError(msg) + + stem.util.log.notice(msg) + return '' diff --git a/nyx/arguments.py b/nyx/arguments.py index 01c2095..b758640 100644 --- a/nyx/arguments.py +++ b/nyx/arguments.py @@ -10,7 +10,7 @@ import nyx import stem.util.connection -from nyx.util import DATA_DIR, tor_controller, msg +from nyx import DATA_DIR, tor_controller, msg DEFAULT_ARGS = { 'control_address': '127.0.0.1', diff --git a/nyx/controller.py b/nyx/controller.py index f188bb1..c3b9146 100644 --- a/nyx/controller.py +++ b/nyx/controller.py @@ -12,8 +12,8 @@ import threading import nyx.curses import nyx.menu.menu import nyx.popups -import nyx.util.tracker +import nyx.panel import nyx.panel.config import nyx.panel.connection import nyx.panel.graph @@ -26,7 +26,7 @@ import stem from stem.util import conf, log from nyx.curses import NORMAL, BOLD, HIGHLIGHT -from nyx.util import panel, tor_controller +from nyx import tor_controller NYX_CONTROLLER = None @@ -62,13 +62,13 @@ def get_controller(): return NYX_CONTROLLER -class LabelPanel(panel.Panel): +class LabelPanel(nyx.panel.Panel): """ Panel that just displays a single line of text. """ def __init__(self, stdscr): - panel.Panel.__init__(self, stdscr, 'msg', 0, height=1) + nyx.panel.Panel.__init__(self, stdscr, 'msg', 0, height=1) self.msg_text = '' self.msg_attr = NORMAL @@ -151,7 +151,7 @@ class Controller: Gets keystroke from the user. """ - return panel.KeyInput(self.get_screen().getch()) + return nyx.panel.KeyInput(self.get_screen().getch()) def get_page_count(self): """ @@ -444,7 +444,7 @@ def start_nyx(stdscr): key, override_key = override_key, None else: curses.halfdelay(CONFIG['features.redrawRate'] * 10) - key = panel.KeyInput(stdscr.getch()) + key = nyx.panel.KeyInput(stdscr.getch()) if key.match('right'): control.next_page() diff --git a/nyx/curses.py b/nyx/curses.py index af3484b..4e32fd9 100644 --- a/nyx/curses.py +++ b/nyx/curses.py @@ -64,7 +64,7 @@ import stem.util.conf import stem.util.enum import stem.util.system -from nyx.util import msg, log +from nyx import msg, log # Text colors and attributes. These are *very* commonly used so including # shorter aliases (so they can be referenced as just GREEN or BOLD). diff --git a/nyx/log.py b/nyx/log.py new file mode 100644 index 0000000..642e97c --- /dev/null +++ b/nyx/log.py @@ -0,0 +1,536 @@ +""" +Logging utilities, primiarily short aliases for logging a message at various +runlevels. + +:: + + trace - logs a message at the TRACE runlevel + debug - logs a message at the DEBUG runlevel + info - logs a message at the INFO runlevel + notice - logs a message at the NOTICE runlevel + warn - logs a message at the WARN runlevel + error - logs a message at the ERROR runlevel + + day_count - number of days since a given timestamp + log_file_path - path of tor's log file if one is present on disk + condense_runlevels - condensed displayable listing of log events + listen_for_events - notifies listener of tor events + read_tor_log - provides LogEntry from a tor log file + + LogGroup - thread safe, deduplicated grouping of events + |- add - adds an event to the group + +- pop - removes and returns an event + + LogEntry - individual log event + |- is_duplicate_of - checks if a duplicate message of another LogEntry + +- day_count - number of days since this even occured + + LogFileOutput - writes log events to a file + +- write - persist a given message + + LogFilters - regex filtering of log events + |- select - filters by this regex + |- selection - current regex filter + |- latest_selections - past regex selections + |- match - checks if a LogEntry matches this filter + +- clone - deep clone of this LogFilters +""" + +import collections +import datetime +import os +import re +import time +import threading + +import stem.util.conf +import stem.util.log +import stem.util.system + +import nyx + +try: + # added in python 3.2 + from functools import lru_cache +except ImportError: + from stem.util.lru_cache import lru_cache + +TOR_RUNLEVELS = ['DEBUG', 'INFO', 'NOTICE', 'WARN', 'ERR'] +TIMEZONE_OFFSET = time.altzone if time.localtime()[8] else time.timezone + + +def day_count(timestamp): + """ + Provoides a unique number for the day a given timestamp falls on, by local + time. Daybreaks are rolled over at midnight. + + :param int timestamp: unix timestamp to provide a count for + + :reutrns: **int** for the day it falls on + """ + + return int((timestamp - TIMEZONE_OFFSET) / 86400) + + +def log_file_path(controller): + """ + Provides the path where tor's log file resides, if one exists. + + :params stem.control.Controller controller: tor controller connection + + :returns: **str** with the absolute path of our log file, or **None** if one + doesn't exist + """ + + for log_entry in controller.get_conf('Log', [], True): + entry_comp = log_entry.split() # looking for an entry like: notice file /var/log/tor/notices.log + + if entry_comp[1] == 'file': + return nyx.expand_path(entry_comp[2]) + + +@lru_cache() +def condense_runlevels(*events): + """ + Provides runlevel events with condensed. For example... + + >>> condense_runlevels('DEBUG', 'NOTICE', 'WARN', 'ERR', 'NYX_NOTICE', 'NYX_WARN', 'NYX_ERR', 'BW') + ['TOR/NYX NOTICE-ERROR', 'DEBUG', 'BW'] + + :param list events: event types to be condensed + + :returns: **list** of the input events, with condensed runlevels + """ + + def ranges(runlevels): + ranges = [] + + while runlevels: + # provides the (start, end) for a contiguous range + start = end = runlevels[0] + + for r in TOR_RUNLEVELS[TOR_RUNLEVELS.index(start):]: + if r in runlevels: + runlevels.remove(r) + end = r + else: + break + + ranges.append((start, end)) + + return ranges + + events = list(events) + tor_runlevels, nyx_runlevels = [], [] + + for r in TOR_RUNLEVELS: + if r in events: + tor_runlevels.append(r) + events.remove(r) + + if 'NYX_%s' % r in events: + nyx_runlevels.append(r) + events.remove('NYX_%s' % r) + + tor_ranges = ranges(tor_runlevels) + nyx_ranges = ranges(nyx_runlevels) + + result = [] + + for runlevel_range in tor_ranges: + if runlevel_range[0] == runlevel_range[1]: + range_label = runlevel_range[0] + else: + range_label = '%s-%s' % (runlevel_range[0], runlevel_range[1]) + + if runlevel_range in nyx_ranges: + result.append('TOR/NYX %s' % range_label) + nyx_ranges.remove(runlevel_range) + else: + result.append(range_label) + + for runlevel_range in nyx_ranges: + if runlevel_range[0] == runlevel_range[1]: + result.append('NYX %s' % runlevel_range[0]) + else: + result.append('NYX %s-%s' % (runlevel_range[0], runlevel_range[1])) + + return result + events + + +def listen_for_events(listener, events): + """ + Configures tor to notify a function of these event types. If tor is + configured to notify this listener then the old listener is replaced. + + :param function listener: listener to be notified + :param list events: event types to attempt to set + + :returns: **list** of event types we're successfully now listening to + """ + + import nyx.arguments + events = set(events) # drops duplicates + + # accounts for runlevel naming difference + + tor_events = events.intersection(set(nyx.arguments.TOR_EVENT_TYPES.values())) + nyx_events = events.intersection(set(['NYX_%s' % runlevel for runlevel in TOR_RUNLEVELS])) + + # adds events unrecognized by nyx if we're listening to the 'UNKNOWN' type + + if 'UNKNOWN' in events: + tor_events.update(set(nyx.arguments.missing_event_types())) + + controller = nyx.tor_controller() + controller.remove_event_listener(listener) + + for event_type in list(tor_events): + try: + controller.add_event_listener(listener, event_type) + except stem.ProtocolError: + tor_events.remove(event_type) + + return sorted(tor_events.union(nyx_events)) + + +@lru_cache() +def _common_log_messages(): + """ + Provides a mapping of message types to its common log messages. These are + message prefixes unless it starts with an asterisk, in which case it can + appear anywhere in the message. + + :returns: **dict** of the form {event_type => [msg1, msg2...]} + """ + + nyx_config, messages = stem.util.conf.get_config('nyx'), {} + + for conf_key in nyx_config.keys(): + if conf_key.startswith('dedup.'): + event_type = conf_key[6:] + messages[event_type] = nyx_config.get(conf_key, []) + + return messages + + +class LogGroup(object): + """ + Thread safe collection of LogEntry instancs, which maintains a certain size + and supports deduplication. + """ + + def __init__(self, max_size, group_by_day = False): + self._max_size = max_size + self._group_by_day = group_by_day + self._entries = [] + self._lock = threading.RLock() + + def add(self, entry): + with self._lock: + duplicate = None + our_day = entry.day_count() + + for existing_entry in self._entries: + if self._group_by_day and our_day != existing_entry.day_count(): + break + elif entry.is_duplicate_of(existing_entry): + duplicate = existing_entry + break + + if duplicate: + if not duplicate.duplicates: + duplicate.duplicates = [duplicate] + + duplicate.is_duplicate = True + entry.duplicates = duplicate.duplicates + entry.duplicates.insert(0, entry) + + self._entries.insert(0, entry) + + while len(self._entries) > self._max_size: + self.pop() + + def pop(self): + with self._lock: + last_entry = self._entries.pop() + + # By design if the last entry is a duplicate it will also be the last + # item in its duplicate group. + + if last_entry.is_duplicate: + last_entry.duplicates.pop() + + def __len__(self): + with self._lock: + return len(self._entries) + + def __iter__(self): + with self._lock: + for entry in self._entries: + yield entry + + +class LogEntry(object): + """ + Individual tor or nyx log entry. + + **Note:** Tor doesn't include the date in its timestamps so the year + component may be inaccurate. (:trac:`15607`) + + :var int timestamp: unix timestamp for when the event occured + :var str type: event type + :var str message: event's message + :var str display_message: message annotated with our time and runlevel + + :var bool is_duplicate: true if this matches other messages in the group and + isn't the first + :var list duplicates: messages that are identical to thsi one + """ + + def __init__(self, timestamp, type, message): + self.timestamp = timestamp + self.type = type + self.message = message + + entry_time = time.localtime(self.timestamp) + self.display_message = '%02i:%02i:%02i [%s] %s' % (entry_time[3], entry_time[4], entry_time[5], self.type, self.message) + + self.is_duplicate = False + self.duplicates = None + + @lru_cache() + def is_duplicate_of(self, entry): + """ + Checks if we are a duplicate of the given message or not. + + :returns: **True** if the given log message is a duplicate of us and **False** otherwise + """ + + if self.type != entry.type: + return False + elif self.message == entry.message: + return True + + if self.type == 'NYX_DEBUG' and 'runtime:' in self.message and 'runtime:' in entry.message: + # most nyx debug messages show runtimes so try matching without that + + if self.message[:self.message.find('runtime:')] == entry.message[:self.message.find('runtime:')]: + return True + + for common_msg in _common_log_messages().get(self.type, []): + # if it starts with an asterisk then check the whole message rather + # than just the start + + if common_msg[0] == '*': + if common_msg[1:] in self.message and common_msg[1:] in entry.message: + return True + else: + if self.message.startswith(common_msg) and entry.message.startswith(common_msg): + return True + + return False + + def day_count(self): + """ + Provides the day this event occured on by local time. + + :reutrns: **int** with the day this occured on + """ + + return day_count(self.timestamp) + + def __eq__(self, other): + if isinstance(other, LogEntry): + return hash(self) == hash(other) + else: + return False + + def __hash__(self): + return hash(self.display_message) + + +class LogFileOutput(object): + """ + File where log messages we receive are written. If unable to do so then a + notification is logged and further write attempts are skipped. + """ + + def __init__(self, path): + self._file = None + + if path: + try: + path_dir = os.path.dirname(path) + + if not os.path.exists(path_dir): + os.makedirs(path_dir) + + self._file = open(path, 'a') + notice('nyx %s opening log file (%s)' % (nyx.__version__, path)) + except IOError as exc: + error('Unable to write to log file: %s' % exc.strerror) + except OSError as exc: + error('Unable to write to log file: %s' % exc) + + def write(self, msg): + if self._file: + try: + self._file.write(msg + '\n') + self._file.flush() + except IOError as exc: + error('Unable to write to log file: %s' % exc.strerror) + self._file = None + + +class LogFilters(object): + """ + Regular expression filtering for log output. This is thread safe and tracks + the latest selections. + """ + + def __init__(self, initial_filters = None, max_filters = 5): + self._max_filters = max_filters + self._selected = None + self._past_filters = collections.OrderedDict() + self._lock = threading.RLock() + + if initial_filters: + for regex in initial_filters: + self.select(regex) + + self.select(None) + + def select(self, regex): + with self._lock: + if regex is None: + self._selected = None + return + + if regex in self._past_filters: + del self._past_filters[regex] + + try: + self._past_filters[regex] = re.compile(regex) + self._selected = regex + + if len(self._past_filters) > self._max_filters: + self._past_filters.popitem(False) + except re.error as exc: + notice('Invalid regular expression pattern (%s): %s' % (exc, regex)) + + def selection(self): + return self._selected + + def latest_selections(self): + return list(reversed(self._past_filters.keys())) + + def match(self, message): + regex_filter = self._past_filters.get(self._selected) + return not regex_filter or bool(regex_filter.search(message)) + + def clone(self): + with self._lock: + clone = LogFilters(max_filters = self._max_filters) + clone._selected = self._selected + clone._past_filters = self._past_filters + return clone + + +def trace(msg, **attr): + _log(stem.util.log.TRACE, msg, **attr) + + +def debug(msg, **attr): + _log(stem.util.log.DEBUG, msg, **attr) + + +def info(msg, **attr): + _log(stem.util.log.INFO, msg, **attr) + + +def notice(msg, **attr): + _log(stem.util.log.NOTICE, msg, **attr) + + +def warn(msg, **attr): + _log(stem.util.log.WARN, msg, **attr) + + +def error(msg, **attr): + _log(stem.util.log.ERROR, msg, **attr) + + +def _log(runlevel, message, **attr): + """ + Logs the given message, formatted with optional attributes. + + :param stem.util.log.Runlevel runlevel: runlevel at which to log the message + :param str message: message handle to log + :param dict attr: attributes to format the message with + """ + + stem.util.log.log(runlevel, nyx.msg(message, **attr)) + + +def read_tor_log(path, read_limit = None): + """ + Provides logging messages from a tor log file, from newest to oldest. + + :param str path: logging location to read from + :param int read_limit: maximum number of lines to read from the file + + :returns: **iterator** for **LogEntry** for the file's contents + + :raises: + * **ValueError** if the log file has unrecognized content + * **IOError** if unable to read the file + """ + + start_time = time.time() + count, isdst = 0, time.localtime().tm_isdst + + for line in stem.util.system.tail(path, read_limit): + # entries look like: + # Jul 15 18:29:48.806 [notice] Parsing GEOIP file. + + line_comp = line.split() + + # Checks that we have all the components we expect. This could happen if + # we're either not parsing a tor log or in weird edge cases (like being + # out of disk space). + + if len(line_comp) < 4: + raise ValueError("Log located at %s has a line that doesn't match the format we expect: %s" % (path, line)) + elif len(line_comp[3]) < 3 or line_comp[3][1:-1].upper() not in TOR_RUNLEVELS: + raise ValueError('Log located at %s has an unrecognized runlevel: %s' % (path, line_comp[3])) + + runlevel = line_comp[3][1:-1].upper() + msg = ' '.join(line_comp[4:]) + current_year = str(datetime.datetime.now().year) + + # Pretending it's the current year. We don't know the actual year (#15607) + # and this may fail due to leap years when picking Feb 29th (#5265). + + try: + timestamp_str = current_year + ' ' + ' '.join(line_comp[:3]) + timestamp_str = timestamp_str.split('.', 1)[0] # drop fractional seconds + timestamp_comp = list(time.strptime(timestamp_str, '%Y %b %d %H:%M:%S')) + timestamp_comp[8] = isdst + + timestamp = int(time.mktime(tuple(timestamp_comp))) # converts local to unix time + + if timestamp > time.time(): + # log entry is from before a year boundary + timestamp_comp[0] -= 1 + timestamp = int(time.mktime(timestamp_comp)) + except ValueError: + raise ValueError("Log located at %s has a timestamp we don't recognize: %s" % (path, ' '.join(line_comp[:3]))) + + count += 1 + yield LogEntry(timestamp, runlevel, msg) + + if 'opening log file' in msg: + break # this entry marks the start of this tor instance + + info('panel.log.read_from_log_file', count = count, path = path, read_limit = read_limit if read_limit else 'none', runtime = '%0.3f' % (time.time() - start_time)) diff --git a/nyx/menu/actions.py b/nyx/menu/actions.py index 6be1766..38e8629 100644 --- a/nyx/menu/actions.py +++ b/nyx/menu/actions.py @@ -9,13 +9,12 @@ import nyx.curses import nyx.panel.graph import nyx.popups import nyx.menu.item -import nyx.util.tracker - -from nyx.util import tor_controller +import nyx.tracker import stem import stem.util.connection +from nyx import tor_controller from stem.util import conf, str_tools CONFIG = conf.config_dict('nyx', { @@ -244,7 +243,7 @@ def make_connections_menu(conn_panel): # resolver submenu - conn_resolver = nyx.util.tracker.get_connection_tracker() + conn_resolver = nyx.tracker.get_connection_tracker() resolver_menu = nyx.menu.item.Submenu('Resolver') resolver_group = nyx.menu.item.SelectionGroup(conn_resolver.set_custom_resolver, conn_resolver.get_custom_resolver()) diff --git a/nyx/panel/__init__.py b/nyx/panel/__init__.py index 0f25fac..0bbde97 100644 --- a/nyx/panel/__init__.py +++ b/nyx/panel/__init__.py @@ -2,6 +2,40 @@ Panels consisting the nyx interface. """ +import copy +import time +import curses +import curses.ascii +import curses.textpad +from threading import RLock + +import nyx.curses +import stem.util.log + +from nyx.curses import HIGHLIGHT +from stem.util import conf, str_tools + +# global ui lock governing all panel instances (curses isn't thread save and +# concurrency bugs produce especially sinister glitches) + +CURSES_LOCK = RLock() + +SCROLL_KEYS = (curses.KEY_UP, curses.KEY_DOWN, curses.KEY_PPAGE, curses.KEY_NPAGE, curses.KEY_HOME, curses.KEY_END) + +SPECIAL_KEYS = { + 'up': curses.KEY_UP, + 'down': curses.KEY_DOWN, + 'left': curses.KEY_LEFT, + 'right': curses.KEY_RIGHT, + 'home': curses.KEY_HOME, + 'end': curses.KEY_END, + 'page_up': curses.KEY_PPAGE, + 'page_down': curses.KEY_NPAGE, + 'esc': 27, +} + +PASS = -1 + __all__ = [ 'config', 'connection', @@ -9,3 +43,808 @@ __all__ = [ 'log', 'torrc', ] + + +def conf_handler(key, value): + if key == 'features.torrc.maxLineWrap': + return max(1, value) + + +CONFIG = conf.config_dict('nyx', { + 'features.maxLineWrap': 8, +}, conf_handler) + +# prevents curses redraws if set +HALT_ACTIVITY = False + + +class BasicValidator(object): + """ + Interceptor for keystrokes given to a textbox, doing the following: + - quits by setting the input to curses.ascii.BEL when escape is pressed + - stops the cursor at the end of the box's content when pressing the right + arrow + - home and end keys move to the start/end of the line + """ + + def validate(self, key, textbox): + """ + Processes the given key input for the textbox. This may modify the + textbox's content, cursor position, etc depending on the functionality + of the validator. This returns the key that the textbox should interpret, + PASS if this validator doesn't want to take any action. + + Arguments: + key - key code input from the user + textbox - curses Textbox instance the input came from + """ + + result = self.handle_key(key, textbox) + return key if result == PASS else result + + def handle_key(self, key, textbox): + y, x = textbox.win.getyx() + + if curses.ascii.isprint(key) and x < textbox.maxx: + # Shifts the existing text forward so input is an insert method rather + # than replacement. The curses.textpad accepts an insert mode flag but + # this has a couple issues... + # - The flag is only available for Python 2.6+, before that the + # constructor only accepted a subwindow argument as per: + # https://trac.torproject.org/projects/tor/ticket/2354 + # - The textpad doesn't shift text that has text attributes. This is + # because keycodes read by textbox.win.inch() includes formatting, + # causing the curses.ascii.isprint() check it does to fail. + + current_input = textbox.gather() + textbox.win.addstr(y, x + 1, current_input[x:textbox.maxx - 1]) + textbox.win.move(y, x) # reverts cursor movement during gather call + elif key == 27: + # curses.ascii.BEL is a character codes that causes textpad to terminate + + return curses.ascii.BEL + elif key == curses.KEY_HOME: + textbox.win.move(y, 0) + return None + elif key in (curses.KEY_END, curses.KEY_RIGHT): + msg_length = len(textbox.gather()) + textbox.win.move(y, x) # reverts cursor movement during gather call + + if key == curses.KEY_END and msg_length > 0 and x < msg_length - 1: + # if we're in the content then move to the end + + textbox.win.move(y, msg_length - 1) + return None + elif key == curses.KEY_RIGHT and x >= msg_length - 1: + # don't move the cursor if there's no content after it + + return None + elif key == 410: + # if we're resizing the display during text entry then cancel it + # (otherwise the input field is filled with nonprintable characters) + + return curses.ascii.BEL + + return PASS + + +class Panel(object): + """ + Wrapper for curses subwindows. This hides most of the ugliness in common + curses operations including: + - locking when concurrently drawing to multiple windows + - gracefully handle terminal resizing + - clip text that falls outside the panel + - convenience methods for word wrap, in-line formatting, etc + + This uses a design akin to Swing where panel instances provide their display + implementation by overwriting the draw() method, and are redrawn with + redraw(). + """ + + def __init__(self, parent, name, top, left = 0, height = -1, width = -1): + """ + Creates a durable wrapper for a curses subwindow in the given parent. + + Arguments: + parent - parent curses window + name - identifier for the panel + top - positioning of top within parent + left - positioning of the left edge within the parent + height - maximum height of panel (uses all available space if -1) + width - maximum width of panel (uses all available space if -1) + """ + + # The not-so-pythonic getters for these parameters are because some + # implementations aren't entirely deterministic (for instance panels + # might chose their height based on its parent's current width). + + self.panel_name = name + self.parent = parent + self.visible = False + self.title_visible = True + + # Attributes for pausing. The pause_attr contains variables our get_attr + # method is tracking, and the pause buffer has copies of the values from + # when we were last unpaused (unused unless we're paused). + + self.paused = False + self.pause_attr = [] + self.pause_buffer = {} + self.pause_time = -1 + + self.top = top + self.left = left + self.height = height + self.width = width + + # The panel's subwindow instance. This is made available to implementors + # via their draw method and shouldn't be accessed directly. + # + # This is None if either the subwindow failed to be created or needs to be + # remade before it's used. The later could be for a couple reasons: + # - The subwindow was never initialized. + # - Any of the parameters used for subwindow initialization have changed. + + self.win = None + + self.max_y, self.max_x = -1, -1 # subwindow dimensions when last redrawn + + def get_name(self): + """ + Provides panel's identifier. + """ + + return self.panel_name + + def is_title_visible(self): + """ + True if the title is configured to be visible, False otherwise. + """ + + return self.title_visible + + def set_title_visible(self, is_visible): + """ + Configures the panel's title to be visible or not when it's next redrawn. + This is not guarenteed to be respected (not all panels have a title). + """ + + self.title_visible = is_visible + + def get_parent(self): + """ + Provides the parent used to create subwindows. + """ + + return self.parent + + def set_visible(self, is_visible): + """ + Toggles if the panel is visible or not. + + Arguments: + is_visible - panel is redrawn when requested if true, skipped otherwise + """ + + self.visible = is_visible + + def is_paused(self): + """ + Provides if the panel's configured to be paused or not. + """ + + return self.paused + + def set_pause_attr(self, attr): + """ + Configures the panel to track the given attribute so that get_attr provides + the value when it was last unpaused (or its current value if we're + currently unpaused). For instance... + + > self.set_pause_attr('myVar') + > self.myVar = 5 + > self.myVar = 6 # self.get_attr('myVar') -> 6 + > self.set_paused(True) + > self.myVar = 7 # self.get_attr('myVar') -> 6 + > self.set_paused(False) + > self.myVar = 7 # self.get_attr('myVar') -> 7 + + Arguments: + attr - parameter to be tracked for get_attr + """ + + self.pause_attr.append(attr) + self.pause_buffer[attr] = self.copy_attr(attr) + + def get_attr(self, attr): + """ + Provides the value of the given attribute when we were last unpaused. If + we're currently unpaused then this is the current value. If untracked this + returns None. + + Arguments: + attr - local variable to be returned + """ + + if attr not in self.pause_attr: + return None + elif self.paused: + return self.pause_buffer[attr] + else: + return self.__dict__.get(attr) + + def copy_attr(self, attr): + """ + Provides a duplicate of the given configuration value, suitable for the + pause buffer. + + Arguments: + attr - parameter to be provided back + """ + + current_value = self.__dict__.get(attr) + return copy.copy(current_value) + + def set_paused(self, is_pause, suppress_redraw = False): + """ + Toggles if the panel is paused or not. This causes the panel to be redrawn + when toggling is pause state unless told to do otherwise. This is + important when pausing since otherwise the panel's display could change + when redrawn for other reasons. + + This returns True if the panel's pause state was changed, False otherwise. + + Arguments: + is_pause - freezes the state of the pause attributes if true, makes + them editable otherwise + suppress_redraw - if true then this will never redraw the panel + """ + + if is_pause != self.paused: + if is_pause: + self.pause_time = time.time() + + self.paused = is_pause + + if is_pause: + # copies tracked attributes so we know what they were before pausing + + for attr in self.pause_attr: + self.pause_buffer[attr] = self.copy_attr(attr) + + if not suppress_redraw: + self.redraw(True) + + return True + else: + return False + + def get_pause_time(self): + """ + Provides the time that we were last paused, returning -1 if we've never + been paused. + """ + + return self.pause_time + + def set_top(self, top): + """ + Changes the position where subwindows are placed within its parent. + + Arguments: + top - positioning of top within parent + """ + + if self.top != top: + self.top = top + self.win = None + + def get_height(self): + """ + Provides the height used for subwindows (-1 if it isn't limited). + """ + + return self.height + + def set_height(self, height): + """ + Changes the height used for subwindows. This uses all available space if -1. + + Arguments: + height - maximum height of panel (uses all available space if -1) + """ + + if self.height != height: + self.height = height + self.win = None + + def get_width(self): + """ + Provides the width used for subwindows (-1 if it isn't limited). + """ + + return self.width + + def set_width(self, width): + """ + Changes the width used for subwindows. This uses all available space if -1. + + Arguments: + width - maximum width of panel (uses all available space if -1) + """ + + if self.width != width: + self.width = width + self.win = None + + def get_preferred_size(self): + """ + Provides the dimensions the subwindow would use when next redrawn, given + that none of the properties of the panel or parent change before then. This + returns a tuple of (height, width). + """ + + new_height, new_width = self.parent.getmaxyx() + set_height, set_width = self.get_height(), self.get_width() + new_height = max(0, new_height - self.top) + new_width = max(0, new_width - self.left) + + if set_height != -1: + new_height = min(new_height, set_height) + + if set_width != -1: + new_width = min(new_width, set_width) + + return (new_height, new_width) + + def handle_key(self, key): + """ + Handler for user input. This returns true if the key press was consumed, + false otherwise. + + Arguments: + key - keycode for the key pressed + """ + + return False + + def get_help(self): + """ + Provides help information for the controls this page provides. This is a + list of tuples of the form... + (control, description, status) + """ + + return [] + + def draw(self, width, height): + """ + Draws display's content. This is meant to be overwritten by + implementations and not called directly (use redraw() instead). The + dimensions provided are the drawable dimensions, which in terms of width is + a column less than the actual space. + + Arguments: + width - horizontal space available for content + height - vertical space available for content + """ + + pass + + def redraw(self, force_redraw=False): + """ + Clears display and redraws its content. This can skip redrawing content if + able (ie, the subwindow's unchanged), instead just refreshing the display. + + Arguments: + force_redraw - forces the content to be cleared and redrawn if true + """ + + # skipped if not currently visible or activity has been halted + + if not self.visible or HALT_ACTIVITY: + return + + # if the panel's completely outside its parent then this is a no-op + + new_height, new_width = self.get_preferred_size() + + if new_height == 0 or new_width == 0: + self.win = None + return + + # recreates the subwindow if necessary + + is_new_window = self._reset_subwindow() + + # The reset argument is disregarded in a couple of situations: + # - The subwindow's been recreated (obviously it then doesn't have the old + # content to refresh). + # - The subwindow's dimensions have changed since last drawn (this will + # likely change the content's layout) + + subwin_max_y, subwin_max_x = self.win.getmaxyx() + + if is_new_window or subwin_max_y != self.max_y or subwin_max_x != self.max_x: + force_redraw = True + + self.max_y, self.max_x = subwin_max_y, subwin_max_x + + if not CURSES_LOCK.acquire(False): + return + + try: + if force_redraw: + self.win.erase() # clears any old contents + self.draw(self.max_x, self.max_y) + self.win.refresh() + finally: + CURSES_LOCK.release() + + def hline(self, y, x, length, *attributes): + """ + Draws a horizontal line. This should only be called from the context of a + panel's draw method. + + Arguments: + y - vertical location + x - horizontal location + length - length the line spans + attr - text attributes + """ + + format_attr = nyx.curses.curses_attr(*attributes) + + if self.win and self.max_x > x and self.max_y > y: + try: + draw_length = min(length, self.max_x - x) + self.win.hline(y, x, curses.ACS_HLINE | format_attr, draw_length) + except: + # in edge cases drawing could cause a _curses.error + pass + + def vline(self, y, x, length, *attributes): + """ + Draws a vertical line. This should only be called from the context of a + panel's draw method. + + Arguments: + y - vertical location + x - horizontal location + length - length the line spans + attr - text attributes + """ + + format_attr = nyx.curses.curses_attr(*attributes) + + if self.win and self.max_x > x and self.max_y > y: + try: + draw_length = min(length, self.max_y - y) + self.win.vline(y, x, curses.ACS_VLINE | format_attr, draw_length) + except: + # in edge cases drawing could cause a _curses.error + pass + + def addch(self, y, x, char, *attributes): + """ + Draws a single character. This should only be called from the context of a + panel's draw method. + + Arguments: + y - vertical location + x - horizontal location + char - character to be drawn + attr - text attributes + """ + + format_attr = nyx.curses.curses_attr(*attributes) + + if self.win and self.max_x > x and self.max_y > y: + try: + self.win.addch(y, x, char, format_attr) + return x + 1 + except: + # in edge cases drawing could cause a _curses.error + pass + + return x + + def addstr(self, y, x, msg, *attributes): + """ + Writes string to subwindow if able. This takes into account screen bounds + to avoid making curses upset. This should only be called from the context + of a panel's draw method. + + Arguments: + y - vertical location + x - horizontal location + msg - text to be added + attr - text attributes + """ + + format_attr = nyx.curses.curses_attr(*attributes) + + # subwindows need a single character buffer (either in the x or y + # direction) from actual content to prevent crash when shrank + + if self.win and self.max_x > x and self.max_y > y: + try: + drawn_msg = msg[:self.max_x - x] + self.win.addstr(y, x, drawn_msg, format_attr) + return x + len(drawn_msg) + except: + # this might produce a _curses.error during edge cases, for instance + # when resizing with visible popups + + pass + + return x + + def addstr_wrap(self, y, x, msg, width, min_x = 0, *attr): + orig_y = y + + while msg: + draw_msg, msg = str_tools.crop(msg, width - x, None, ending = None, get_remainder = True) + + if not draw_msg: + draw_msg, msg = str_tools.crop(msg, width - x), '' # first word is longer than the line + + x = self.addstr(y, x, draw_msg, *attr) + + if (y - orig_y + 1) >= CONFIG['features.maxLineWrap']: + break # maximum number we'll wrap + + if msg: + x, y = min_x, y + 1 + + return x, y + + def getstr(self, y, x, initial_text = ''): + """ + Provides a text field where the user can input a string, blocking until + they've done so and returning the result. If the user presses escape then + this terminates and provides back None. This should only be called from + the context of a panel's draw method. + + This blanks any content within the space that the input field is rendered + (otherwise stray characters would be interpreted as part of the initial + input). + + Arguments: + y - vertical location + x - horizontal location + initial_text - starting text in this field + """ + + # makes cursor visible + + try: + previous_cursor_state = curses.curs_set(1) + except curses.error: + previous_cursor_state = 0 + + # temporary subwindow for user input + + display_width = self.get_preferred_size()[1] + + input_subwindow = self.parent.subwin(1, display_width - x, self.top + y, self.left + x) + + # blanks the field's area, filling it with the font in case it's hilighting + + input_subwindow.clear() + input_subwindow.bkgd(' ', curses.A_NORMAL) + + # prepopulates the initial text + + if initial_text: + input_subwindow.addstr(0, 0, initial_text[:display_width - x - 1], curses.A_NORMAL) + + # Displays the text field, blocking until the user's done. This closes the + # text panel and returns user_input to the initial text if the user presses + # escape. + + textbox = curses.textpad.Textbox(input_subwindow) + + validator = BasicValidator() + + textbox.win.attron(curses.A_NORMAL) + user_input = textbox.edit(lambda key: validator.validate(key, textbox)).strip() + textbox.win.attroff(curses.A_NORMAL) + + if textbox.lastcmd == curses.ascii.BEL: + user_input = None + + # reverts visability settings + + try: + curses.curs_set(previous_cursor_state) + except curses.error: + pass + + return user_input + + def add_scroll_bar(self, top, bottom, size, draw_top = 0, draw_bottom = -1, draw_left = 0): + """ + Draws a left justified scroll bar reflecting position within a vertical + listing. This is shorted if necessary, and left undrawn if no space is + available. The bottom is squared off, having a layout like: + | + *| + *| + *| + | + -+ + + This should only be called from the context of a panel's draw method. + + Arguments: + top - list index for the top-most visible element + bottom - list index for the bottom-most visible element + size - size of the list in which the listed elements are contained + draw_top - starting row where the scroll bar should be drawn + draw_bottom - ending row where the scroll bar should end, -1 if it should + span to the bottom of the panel + draw_left - left offset at which to draw the scroll bar + """ + + if (self.max_y - draw_top) < 2: + return # not enough room + + # sets draw_bottom to be the actual row on which the scrollbar should end + + if draw_bottom == -1: + draw_bottom = self.max_y - 1 + else: + draw_bottom = min(draw_bottom, self.max_y - 1) + + # determines scrollbar dimensions + + scrollbar_height = draw_bottom - draw_top + slider_top = scrollbar_height * top / size + slider_size = scrollbar_height * (bottom - top) / size + + # ensures slider isn't at top or bottom unless really at those extreme bounds + + if top > 0: + slider_top = max(slider_top, 1) + + if bottom != size: + slider_top = min(slider_top, scrollbar_height - slider_size - 2) + + # avoids a rounding error that causes the scrollbar to be too low when at + # the bottom + + if bottom == size: + slider_top = scrollbar_height - slider_size - 1 + + # draws scrollbar slider + + for i in range(scrollbar_height): + if i >= slider_top and i <= slider_top + slider_size: + self.addstr(i + draw_top, draw_left, ' ', HIGHLIGHT) + else: + self.addstr(i + draw_top, draw_left, ' ') + + # draws box around the scroll bar + + self.vline(draw_top, draw_left + 1, draw_bottom - 1) + self.addch(draw_bottom, draw_left + 1, curses.ACS_LRCORNER) + self.addch(draw_bottom, draw_left, curses.ACS_HLINE) + + def _reset_subwindow(self): + """ + Create a new subwindow instance for the panel if: + - Panel currently doesn't have a subwindow (was uninitialized or + invalidated). + - There's room for the panel to grow vertically (curses automatically + lets subwindows regrow horizontally, but not vertically). + - The subwindow has been displaced. This is a curses display bug that + manifests if the terminal's shrank then re-expanded. Displaced + subwindows are never restored to their proper position, resulting in + graphical glitches if we draw to them. + - The preferred size is smaller than the actual size (should shrink). + + This returns True if a new subwindow instance was created, False otherwise. + """ + + new_height, new_width = self.get_preferred_size() + + if new_height == 0: + return False # subwindow would be outside its parent + + # determines if a new subwindow should be recreated + + recreate = self.win is None + + if self.win: + subwin_max_y, subwin_max_x = self.win.getmaxyx() + recreate |= subwin_max_y < new_height # check for vertical growth + recreate |= self.top > self.win.getparyx()[0] # check for displacement + recreate |= subwin_max_x > new_width or subwin_max_y > new_height # shrinking + + # I'm not sure if recreating subwindows is some sort of memory leak but the + # Python curses bindings seem to lack all of the following: + # - subwindow deletion (to tell curses to free the memory) + # - subwindow moving/resizing (to restore the displaced windows) + # so this is the only option (besides removing subwindows entirely which + # would mean far more complicated code and no more selective refreshing) + + if recreate: + self.win = self.parent.subwin(new_height, new_width, self.top, self.left) + + # note: doing this log before setting win produces an infinite loop + stem.util.log.debug("recreating panel '%s' with the dimensions of %i/%i" % (self.get_name(), new_height, new_width)) + + return recreate + + def draw_box(self, top, left, width, height, *attributes): + """ + Draws a box in the panel with the given bounds. + + Arguments: + top - vertical position of the box's top + left - horizontal position of the box's left side + width - width of the drawn box + height - height of the drawn box + attr - text attributes + """ + + # draws the top and bottom + + self.hline(top, left + 1, width - 2, *attributes) + self.hline(top + height - 1, left + 1, width - 2, *attributes) + + # draws the left and right sides + + self.vline(top + 1, left, height - 2, *attributes) + self.vline(top + 1, left + width - 1, height - 2, *attributes) + + # draws the corners + + self.addch(top, left, curses.ACS_ULCORNER, *attributes) + self.addch(top, left + width - 1, curses.ACS_URCORNER, *attributes) + self.addch(top + height - 1, left, curses.ACS_LLCORNER, *attributes) + self.addch(top + height - 1, left + width - 1, curses.ACS_LRCORNER, *attributes) + + +class KeyInput(object): + """ + Keyboard input by the user. + """ + + def __init__(self, key): + self._key = key # pressed key as an integer + + def match(self, *keys): + """ + Checks if we have a case insensitive match with the given key. Beside + characters, this also recognizes: up, down, left, right, home, end, + page_up, page_down, and esc. + """ + + for key in keys: + if key in SPECIAL_KEYS: + if self._key == SPECIAL_KEYS[key]: + return True + elif len(key) == 1: + if self._key in (ord(key.lower()), ord(key.upper())): + return True + else: + raise ValueError("%s wasn't among our recognized key codes" % key) + + return False + + def is_scroll(self): + """ + True if the key is used for scrolling, false otherwise. + """ + + return self._key in SCROLL_KEYS + + def is_selection(self): + """ + True if the key matches the enter or space keys. + """ + + return self._key in (curses.KEY_ENTER, 10, ord(' ')) diff --git a/nyx/panel/config.py b/nyx/panel/config.py index 2c09a35..45f82a8 100644 --- a/nyx/panel/config.py +++ b/nyx/panel/config.py @@ -8,13 +8,14 @@ import os import nyx.controller import nyx.curses +import nyx.panel import nyx.popups import stem.control import stem.manual from nyx.curses import GREEN, CYAN, WHITE, NORMAL, BOLD, HIGHLIGHT -from nyx.util import DATA_DIR, panel, tor_controller +from nyx import DATA_DIR, tor_controller from stem.util import conf, enum, log, str_tools @@ -112,13 +113,13 @@ class ConfigEntry(object): return not self.is_set() -class ConfigPanel(panel.Panel): +class ConfigPanel(nyx.panel.Panel): """ Editor for tor's configuration. """ def __init__(self, stdscr): - panel.Panel.__init__(self, stdscr, 'configuration', 0) + nyx.panel.Panel.__init__(self, stdscr, 'configuration', 0) self._contents = [] self._scroller = nyx.curses.CursorScroller() diff --git a/nyx/panel/connection.py b/nyx/panel/connection.py index 7e01674..1f6d820 100644 --- a/nyx/panel/connection.py +++ b/nyx/panel/connection.py @@ -10,11 +10,12 @@ import itertools import threading import nyx.curses +import nyx.panel import nyx.popups -import nyx.util.tracker +import nyx.tracker from nyx.curses import WHITE, NORMAL, BOLD, HIGHLIGHT -from nyx.util import panel, tor_controller +from nyx import tor_controller from stem.control import Listener from stem.util import datetime_to_unix, conf, connection, enum, str_tools @@ -163,10 +164,10 @@ class ConnectionEntry(Entry): fingerprint, nickname, locale = None, None, None if self.get_type() in (Category.OUTBOUND, Category.CIRCUIT, Category.DIRECTORY, Category.EXIT): - fingerprint = nyx.util.tracker.get_consensus_tracker().get_relay_fingerprints(self._connection.remote_address).get(self._connection.remote_port) + fingerprint = nyx.tracker.get_consensus_tracker().get_relay_fingerprints(self._connection.remote_address).get(self._connection.remote_port) if fingerprint: - nickname = nyx.util.tracker.get_consensus_tracker().get_relay_nickname(fingerprint) + nickname = nyx.tracker.get_consensus_tracker().get_relay_nickname(fingerprint) locale = tor_controller().get_info('ip-to-country/%s' % self._connection.remote_address, None) return [Line(self, LineType.CONNECTION, self._connection, None, fingerprint, nickname, locale)] @@ -189,7 +190,7 @@ class ConnectionEntry(Entry): if self._connection.remote_port == hs_config['HiddenServicePort']: return Category.HIDDEN - fingerprint = nyx.util.tracker.get_consensus_tracker().get_relay_fingerprints(self._connection.remote_address).get(self._connection.remote_port) + fingerprint = nyx.tracker.get_consensus_tracker().get_relay_fingerprints(self._connection.remote_address).get(self._connection.remote_port) if fingerprint and LAST_RETRIEVED_CIRCUITS: for circ in LAST_RETRIEVED_CIRCUITS: @@ -214,7 +215,7 @@ class ConnectionEntry(Entry): controller = tor_controller() if controller.is_user_traffic_allowed().inbound: - return len(nyx.util.tracker.get_consensus_tracker().get_relay_fingerprints(self._connection.remote_address)) == 0 + return len(nyx.tracker.get_consensus_tracker().get_relay_fingerprints(self._connection.remote_address)) == 0 elif self.get_type() == Category.EXIT: # DNS connections exiting us aren't private (since they're hitting our # resolvers). Everything else is. @@ -232,14 +233,14 @@ class CircuitEntry(Entry): def get_lines(self): def line(fingerprint, line_type): address, port, nickname, locale = '0.0.0.0', 0, None, None - consensus_tracker = nyx.util.tracker.get_consensus_tracker() + consensus_tracker = nyx.tracker.get_consensus_tracker() if fingerprint is not None: address, port = consensus_tracker.get_relay_address(fingerprint, ('192.168.0.1', 0)) nickname = consensus_tracker.get_relay_nickname(fingerprint) locale = tor_controller().get_info('ip-to-country/%s' % address, None) - connection = nyx.util.tracker.Connection(datetime_to_unix(self._circuit.created), False, '127.0.0.1', 0, address, port, 'tcp', False) + connection = nyx.tracker.Connection(datetime_to_unix(self._circuit.created), False, '127.0.0.1', 0, address, port, 'tcp', False) return Line(self, line_type, connection, self._circuit, fingerprint, nickname, locale) header_line = line(self._circuit.path[-1][0] if self._circuit.status == 'BUILT' else None, LineType.CIRCUIT_HEADER) @@ -252,14 +253,14 @@ class CircuitEntry(Entry): return False -class ConnectionPanel(panel.Panel, threading.Thread): +class ConnectionPanel(nyx.panel.Panel, threading.Thread): """ Listing of connections tor is making, with information correlated against the current consensus and other data sources. """ def __init__(self, stdscr): - panel.Panel.__init__(self, stdscr, 'connections', 0) + nyx.panel.Panel.__init__(self, stdscr, 'connections', 0) threading.Thread.__init__(self) self.setDaemon(True) @@ -333,8 +334,8 @@ class ConnectionPanel(panel.Panel, threading.Thread): elif key.match('s'): self.show_sort_dialog() elif key.match('r'): - connection_tracker = nyx.util.tracker.get_connection_tracker() - options = ['auto'] + list(connection.Resolver) + list(nyx.util.tracker.CustomResolver) + connection_tracker = nyx.tracker.get_connection_tracker() + options = ['auto'] + list(connection.Resolver) + list(nyx.tracker.CustomResolver) resolver = connection_tracker.get_custom_resolver() selected_index = 0 if resolver is None else options.index(resolver) @@ -363,9 +364,9 @@ class ConnectionPanel(panel.Panel, threading.Thread): if not key or key.is_selection() or key.match('d'): break # closes popup elif key.match('left'): - self.handle_key(panel.KeyInput(curses.KEY_UP)) + self.handle_key(nyx.panel.KeyInput(curses.KEY_UP)) elif key.match('right'): - self.handle_key(panel.KeyInput(curses.KEY_DOWN)) + self.handle_key(nyx.panel.KeyInput(curses.KEY_DOWN)) self.set_title_visible(True) self.redraw(True) @@ -413,12 +414,12 @@ class ConnectionPanel(panel.Panel, threading.Thread): # iteration to hide the lag. if last_ran == -1: - nyx.util.tracker.get_consensus_tracker().update(tor_controller().get_network_statuses([])) + nyx.tracker.get_consensus_tracker().update(tor_controller().get_network_statuses([])) last_ran = time.time() def get_help(self): - resolver = nyx.util.tracker.get_connection_tracker().get_custom_resolver() + resolver = nyx.tracker.get_connection_tracker().get_custom_resolver() user_traffic_allowed = tor_controller().is_user_traffic_allowed() options = [ @@ -503,7 +504,7 @@ class ConnectionPanel(panel.Panel, threading.Thread): self.addstr(1, 2, 'address: %s:%s' % (address, selected.connection.remote_port), *attr) self.addstr(2, 2, 'locale: %s' % ('??' if selected.entry.is_private() else (selected.locale if selected.locale else '??')), *attr) - matches = nyx.util.tracker.get_consensus_tracker().get_relay_fingerprints(selected.connection.remote_address) + matches = nyx.tracker.get_consensus_tracker().get_relay_fingerprints(selected.connection.remote_address) if not matches: self.addstr(3, 2, 'No consensus data found', *attr) @@ -603,11 +604,11 @@ class ConnectionPanel(panel.Panel, threading.Thread): elif line.entry.get_type() in (Category.SOCKS, Category.HIDDEN, Category.CONTROL): try: port = line.connection.local_port if line.entry.get_type() == Category.HIDDEN else line.connection.remote_port - process = nyx.util.tracker.get_port_usage_tracker().fetch(port) + process = nyx.tracker.get_port_usage_tracker().fetch(port) comp = ['%s (%s)' % (process.name, process.pid) if process.pid else process.name] - except nyx.util.tracker.UnresolvedResult: + except nyx.tracker.UnresolvedResult: comp = ['resolving...'] - except nyx.util.tracker.UnknownApplication: + except nyx.tracker.UnknownApplication: comp = ['UNKNOWN'] else: comp = ['%-40s' % (line.fingerprint if line.fingerprint else 'UNKNOWN'), ' ' + (line.nickname if line.nickname else 'UNKNOWN')] @@ -658,7 +659,7 @@ class ConnectionPanel(panel.Panel, threading.Thread): LAST_RETRIEVED_CIRCUITS = controller.get_circuits([]) LAST_RETRIEVED_HS_CONF = controller.get_hidden_service_conf({}) - conn_resolver = nyx.util.tracker.get_connection_tracker() + conn_resolver = nyx.tracker.get_connection_tracker() current_resolution_count = conn_resolver.run_counter() if not conn_resolver.is_alive(): @@ -702,4 +703,4 @@ class ConnectionPanel(panel.Panel, threading.Thread): elif entry.get_type() == Category.HIDDEN: remote_ports.append(line.connection.local_port) - nyx.util.tracker.get_port_usage_tracker().query(local_ports, remote_ports) + nyx.tracker.get_port_usage_tracker().query(local_ports, remote_ports) diff --git a/nyx/panel/graph.py b/nyx/panel/graph.py index 03fd8d9..818769c 100644 --- a/nyx/panel/graph.py +++ b/nyx/panel/graph.py @@ -17,13 +17,14 @@ import curses import time import nyx.controller +import nyx.panel import nyx.popups -import nyx.util.tracker +import nyx.tracker +from nyx import join, msg, tor_controller +from nyx.curses import RED, GREEN, CYAN, BOLD, HIGHLIGHT from stem.control import EventType, Listener from stem.util import conf, enum, log, str_tools, system -from nyx.util import join, msg, panel, tor_controller -from nyx.curses import RED, GREEN, CYAN, BOLD, HIGHLIGHT GraphStat = enum.Enum(('BANDWIDTH', 'bandwidth'), ('CONNECTIONS', 'connections'), ('SYSTEM_RESOURCES', 'resources')) Interval = enum.Enum(('EACH_SECOND', 'each second'), ('FIVE_SECONDS', '5 seconds'), ('THIRTY_SECONDS', '30 seconds'), ('MINUTELY', 'minutely'), ('FIFTEEN_MINUTE', '15 minute'), ('THIRTY_MINUTE', '30 minute'), ('HOURLY', 'hourly'), ('DAILY', 'daily')) @@ -341,7 +342,7 @@ class ConnectionStats(GraphCategory): dir_ports = controller.get_ports(Listener.DIR, []) control_ports = controller.get_ports(Listener.CONTROL, []) - for entry in nyx.util.tracker.get_connection_tracker().get_value(): + for entry in nyx.tracker.get_connection_tracker().get_value(): if entry.local_port in or_ports or entry.local_port in dir_ports: inbound_count += 1 elif entry.local_port in control_ports: @@ -368,7 +369,7 @@ class ResourceStats(GraphCategory): return '%i%%' % value if is_primary else str_tools.size_label(value) def bandwidth_event(self, event): - resources = nyx.util.tracker.get_resource_tracker().get_value() + resources = nyx.tracker.get_resource_tracker().get_value() self.primary.update(resources.cpu_sample * 100) # decimal percentage to whole numbers self.secondary.update(resources.memory_bytes) @@ -376,13 +377,13 @@ class ResourceStats(GraphCategory): self._secondary_header_stats = [str_tools.size_label(self.secondary.latest_value, 1), ', avg: %s' % str_tools.size_label(self.secondary.average(), 1)] -class GraphPanel(panel.Panel): +class GraphPanel(nyx.panel.Panel): """ Panel displaying graphical information of GraphCategory instances. """ def __init__(self, stdscr): - panel.Panel.__init__(self, stdscr, 'graph', 0) + nyx.panel.Panel.__init__(self, stdscr, 'graph', 0) self._displayed_stat = None if CONFIG['features.graph.type'] == 'none' else CONFIG['features.graph.type'] self._update_interval = CONFIG['features.graph.interval'] @@ -475,7 +476,7 @@ class GraphPanel(panel.Panel): control = nyx.controller.get_controller() - with panel.CURSES_LOCK: + with nyx.panel.CURSES_LOCK: try: while True: msg = 'press the down/up to resize the graph, and enter when done' @@ -728,7 +729,7 @@ class GraphPanel(panel.Panel): if attr == '_stats': return dict([(key, type(self._stats[key])(self._stats[key])) for key in self._stats]) else: - return panel.Panel.copy_attr(self, attr) + return nyx.panel.Panel.copy_attr(self, attr) def _update_accounting(self, event): if not CONFIG['features.graph.bw.accounting.show']: diff --git a/nyx/panel/header.py b/nyx/panel/header.py index 6c4928c..3ee0e84 100644 --- a/nyx/panel/header.py +++ b/nyx/panel/header.py @@ -12,11 +12,13 @@ import threading import stem import nyx.controller +import nyx.panel import nyx.popups +import nyx.tracker from stem.control import Listener, State from stem.util import conf, log, proc, str_tools, system -from nyx.util import msg, tor_controller, panel, tracker +from nyx import msg, tor_controller from nyx.curses import RED, GREEN, YELLOW, CYAN, WHITE, BOLD @@ -30,13 +32,13 @@ CONFIG = conf.config_dict('nyx', { }) -class HeaderPanel(panel.Panel, threading.Thread): +class HeaderPanel(nyx.panel.Panel, threading.Thread): """ Top area containing tor settings and system information. """ def __init__(self, stdscr): - panel.Panel.__init__(self, stdscr, 'header', 0) + nyx.panel.Panel.__init__(self, stdscr, 'header', 0) threading.Thread.__init__(self) self.setDaemon(True) @@ -414,7 +416,7 @@ def get_sampling(last_sampling = None): retrieved = time.time() pid = controller.get_pid('') - tor_resources = tracker.get_resource_tracker().get_value() + tor_resources = nyx.tracker.get_resource_tracker().get_value() nyx_total_cpu_time = sum(os.times()[:3]) or_listeners = controller.get_listeners(Listener.OR, []) diff --git a/nyx/panel/log.py b/nyx/panel/log.py index ba22d3c..92e3f90 100644 --- a/nyx/panel/log.py +++ b/nyx/panel/log.py @@ -12,11 +12,12 @@ import stem.response.events import nyx.arguments import nyx.curses +import nyx.panel import nyx.popups -import nyx.util.log +import nyx.log +from nyx import join, tor_controller from nyx.curses import GREEN, YELLOW, WHITE, NORMAL, BOLD, HIGHLIGHT -from nyx.util import join, panel, tor_controller from stem.util import conf, log @@ -57,22 +58,22 @@ NYX_LOGGER = log.LogBuffer(log.Runlevel.DEBUG, yield_records = True) stem_logger.addHandler(NYX_LOGGER) -class LogPanel(panel.Panel, threading.Thread): +class LogPanel(nyx.panel.Panel, threading.Thread): """ Listens for and displays tor, nyx, and stem events. This prepopulates from tor's log file if it exists. """ def __init__(self, stdscr): - panel.Panel.__init__(self, stdscr, 'log', 0) + nyx.panel.Panel.__init__(self, stdscr, 'log', 0) threading.Thread.__init__(self) self.setDaemon(True) logged_events = nyx.arguments.expand_events(CONFIG['startup.events']) - self._event_log = nyx.util.log.LogGroup(CONFIG['cache.log_panel.size'], group_by_day = True) - self._event_types = nyx.util.log.listen_for_events(self._register_tor_event, logged_events) - self._log_file = nyx.util.log.LogFileOutput(CONFIG['features.logFile']) - self._filter = nyx.util.log.LogFilters(initial_filters = CONFIG['features.log.regex']) + self._event_log = nyx.log.LogGroup(CONFIG['cache.log_panel.size'], group_by_day = True) + self._event_types = nyx.log.listen_for_events(self._register_tor_event, logged_events) + self._log_file = nyx.log.LogFileOutput(CONFIG['features.logFile']) + self._filter = nyx.log.LogFilters(initial_filters = CONFIG['features.log.regex']) self._show_duplicates = CONFIG['features.log.showDuplicateEntries'] self.set_pause_attr('_event_log') @@ -85,11 +86,11 @@ class LogPanel(panel.Panel, threading.Thread): # fetches past tor events from log file, if available if CONFIG['features.log.prepopulate']: - log_location = nyx.util.log.log_file_path(tor_controller()) + log_location = nyx.log.log_file_path(tor_controller()) if log_location: try: - for entry in reversed(list(nyx.util.log.read_tor_log(log_location, CONFIG['features.log.prepopulateReadLimit']))): + for entry in reversed(list(nyx.log.read_tor_log(log_location, CONFIG['features.log.prepopulateReadLimit']))): if entry.type in self._event_types: self._event_log.add(entry) except IOError as exc: @@ -161,7 +162,7 @@ class LogPanel(panel.Panel, threading.Thread): event_types = nyx.arguments.expand_events(user_input) if event_types != self._event_types: - self._event_types = nyx.util.log.listen_for_events(self._register_tor_event, event_types) + self._event_types = nyx.log.listen_for_events(self._register_tor_event, event_types) self.redraw(True) except ValueError as exc: nyx.popups.show_msg('Invalid flags: %s' % str(exc), 2) @@ -185,7 +186,7 @@ class LogPanel(panel.Panel, threading.Thread): Clears the contents of the event log. """ - self._event_log = nyx.util.log.LogGroup(CONFIG['cache.log_panel.size'], group_by_day = True) + self._event_log = nyx.log.LogGroup(CONFIG['cache.log_panel.size'], group_by_day = True) self.redraw(True) def save_snapshot(self, path): @@ -238,7 +239,7 @@ class LogPanel(panel.Panel, threading.Thread): if key_press.match('c'): self.clear() elif key.match('f'): - with panel.CURSES_LOCK: + with nyx.panel.CURSES_LOCK: initial_selection = 1 if self._filter.selection() else 0 options = ['None'] + self._filter.latest_selections() + ['New...'] selection = nyx.popups.show_menu('Log Filter:', options, initial_selection) @@ -288,7 +289,7 @@ class LogPanel(panel.Panel, threading.Thread): # group entries by date, filtering out those that aren't visible - day_to_entries, today = {}, nyx.util.log.day_count(time.time()) + day_to_entries, today = {}, nyx.log.day_count(time.time()) for entry in event_log: if entry.is_duplicate and not show_duplicates: @@ -351,7 +352,7 @@ class LogPanel(panel.Panel, threading.Thread): """ self.addstr(0, 0, ' ' * width) # clear line - title_comp = list(nyx.util.log.condense_runlevels(*event_types)) + title_comp = list(nyx.log.condense_runlevels(*event_types)) if event_filter.selection(): title_comp.append('filter: %s' % event_filter.selection()) @@ -388,10 +389,10 @@ class LogPanel(panel.Panel, threading.Thread): responsive if additions are less frequent. """ - last_ran, last_day = -1, nyx.util.log.day_count(time.time()) + last_ran, last_day = -1, nyx.log.day_count(time.time()) while not self._halt: - current_day = nyx.util.log.day_count(time.time()) + current_day = nyx.log.day_count(time.time()) time_since_reset = time.time() - last_ran max_log_update_rate = CONFIG['features.log.maxRefreshRate'] / 1000.0 @@ -429,13 +430,13 @@ class LogPanel(panel.Panel, threading.Thread): elif isinstance(event, stem.response.events.LogEvent): msg = event.message - self._register_event(nyx.util.log.LogEntry(event.arrived_at, event.type, msg)) + self._register_event(nyx.log.LogEntry(event.arrived_at, event.type, msg)) def _register_nyx_event(self, record): if record.levelname == 'WARNING': record.levelname = 'WARN' - self._register_event(nyx.util.log.LogEntry(int(record.created), 'NYX_%s' % record.levelname, record.msg)) + self._register_event(nyx.log.LogEntry(int(record.created), 'NYX_%s' % record.levelname, record.msg)) def _register_event(self, event): if event.type not in self._event_types: diff --git a/nyx/panel/torrc.py b/nyx/panel/torrc.py index bf10efb..033fd94 100644 --- a/nyx/panel/torrc.py +++ b/nyx/panel/torrc.py @@ -8,7 +8,7 @@ import string import nyx.curses from nyx.curses import RED, GREEN, YELLOW, CYAN, WHITE, BOLD, HIGHLIGHT -from nyx.util import expand_path, msg, panel, tor_controller +from nyx import expand_path, msg, panel, tor_controller from stem import ControllerError from stem.control import State diff --git a/nyx/popups.py b/nyx/popups.py index c05db05..d594409 100644 --- a/nyx/popups.py +++ b/nyx/popups.py @@ -10,10 +10,10 @@ import operator import nyx.controller import nyx.curses +import nyx.panel -from nyx import __version__, __release_date__ +from nyx import __version__, __release_date__, tor_controller from nyx.curses import RED, GREEN, YELLOW, CYAN, WHITE, NORMAL, BOLD, HIGHLIGHT -from nyx.util import tor_controller, panel NO_STATS_MSG = "Usage stats aren't available yet, press any key..." @@ -57,7 +57,7 @@ def popup_window(height = -1, width = -1, top = 0, left = 0, below_static = True else: sticky_height = 0 - popup = panel.Panel(control.get_screen(), 'popup', top + sticky_height, left, height, width) + popup = nyx.panel.Panel(control.get_screen(), 'popup', top + sticky_height, left, height, width) popup.set_visible(True) # Redraws the popup to prepare a subwindow instance. If none is spawned then @@ -66,13 +66,13 @@ def popup_window(height = -1, width = -1, top = 0, left = 0, below_static = True popup.redraw(True) if popup.win is not None: - panel.CURSES_LOCK.acquire() + nyx.panel.CURSES_LOCK.acquire() return (popup, popup.max_x - 1, popup.max_y) else: return (None, 0, 0) def __exit__(self, exit_type, value, traceback): - panel.CURSES_LOCK.release() + nyx.panel.CURSES_LOCK.release() nyx.controller.get_controller().redraw(False) return _Popup() @@ -88,7 +88,7 @@ def input_prompt(msg, initial_value = ''): initial_value - initial value of the field """ - with panel.CURSES_LOCK: + with nyx.panel.CURSES_LOCK: control = nyx.controller.get_controller() msg_panel = control.get_panel('msg') msg_panel.set_message(msg) @@ -110,7 +110,7 @@ def show_msg(msg, max_wait = -1, attr = HIGHLIGHT): attr - attributes with which to draw the message """ - with panel.CURSES_LOCK: + with nyx.panel.CURSES_LOCK: control = nyx.controller.get_controller() control.set_msg(msg, attr, True) @@ -445,7 +445,7 @@ def show_descriptor_popup(fingerprint, color, max_width, is_close_key): :param function is_close_key: method to indicate if a key should close the dialog or not - :returns: :class:`~nyx.util.panel.KeyInput` for the keyboard input that + :returns: :class:`~nyx.panel.KeyInput` for the keyboard input that closed the dialog """ diff --git a/nyx/starter.py b/nyx/starter.py index 83825e0..6697e21 100644 --- a/nyx/starter.py +++ b/nyx/starter.py @@ -18,14 +18,14 @@ import nyx import nyx.arguments import nyx.controller import nyx.curses -import nyx.util.panel -import nyx.util.tracker +import nyx.panel +import nyx.tracker import stem import stem.util.log import stem.util.system -from nyx.util import log, init_controller, msg, uses_settings +from nyx import log, init_controller, msg, uses_settings @uses_settings @@ -95,7 +95,7 @@ def main(config): except KeyboardInterrupt: pass # skip printing a stack trace finally: - nyx.util.panel.HALT_ACTIVITY = True + nyx.panel.HALT_ACTIVITY = True _shutdown_daemons(controller) @@ -255,7 +255,7 @@ def _shutdown_daemons(controller): Stops and joins on worker threads. """ - halt_threads = [nyx.util.tracker.stop_trackers()] + halt_threads = [nyx.tracker.stop_trackers()] curses_controller = nyx.controller.get_controller() if curses_controller: diff --git a/nyx/tracker.py b/nyx/tracker.py new file mode 100644 index 0000000..a772ff9 --- /dev/null +++ b/nyx/tracker.py @@ -0,0 +1,895 @@ +""" +Background tasks for gathering information about the tor process. + +:: + + get_connection_tracker - provides a ConnectionTracker for our tor process + get_resource_tracker - provides a ResourceTracker for our tor process + get_port_usage_tracker - provides a PortUsageTracker for our system + get_consensus_tracker - provides a ConsensusTracker for our tor process + + stop_trackers - halts any active trackers + + Daemon - common parent for resolvers + |- ConnectionTracker - periodically checks the connections established by tor + | |- get_custom_resolver - provide the custom conntion resolver we're using + | |- set_custom_resolver - overwrites automatic resolver selecion with a custom resolver + | +- get_value - provides our latest connection results + | + |- ResourceTracker - periodically checks the resource usage of tor + | +- get_value - provides our latest resource usage results + | + |- PortUsageTracker - provides information about port usage on the local system + | +- get_processes_using_ports - mapping of ports to the processes using it + | + |- run_counter - number of successful runs + |- get_rate - provides the rate at which we run + |- set_rate - sets the rate at which we run + |- set_paused - pauses or continues work + +- stop - stops further work by the daemon + + ConsensusTracker - performant lookups for consensus related information + |- update - updates the consensus information we're based on + |- get_relay_nickname - provides the nickname for a given relay + |- get_relay_fingerprints - provides relays running at a location + +- get_relay_address - provides the address a relay is running at + +.. data:: Resources + + Resource usage information retrieved about the tor process. + + :var float cpu_sample: average cpu usage since we last checked + :var float cpu_average: average cpu usage since we first started tracking the process + :var float cpu_total: total cpu time the process has used since starting + :var int memory_bytes: memory usage of the process in bytes + :var float memory_percent: percentage of our memory used by this process + :var float timestamp: unix timestamp for when this information was fetched +""" + +import collections +import os +import time +import threading + +import stem.control + +from nyx import log, tor_controller +from stem.util import conf, connection, enum, proc, str_tools, system + +CONFIG = conf.config_dict('nyx', { + 'queries.connections.rate': 5, + 'queries.resources.rate': 5, + 'queries.port_usage.rate': 5, +}) + +CONNECTION_TRACKER = None +RESOURCE_TRACKER = None +PORT_USAGE_TRACKER = None +CONSENSUS_TRACKER = None + +CustomResolver = enum.Enum( + ('INFERENCE', 'by inference'), +) + +# Extending stem's Connection tuple with attributes for the uptime of the +# connection. + +Connection = collections.namedtuple('Connection', [ + 'start_time', + 'is_legacy', # boolean to indicate if the connection predated us +] + list(stem.util.connection.Connection._fields)) + +Resources = collections.namedtuple('Resources', [ + 'cpu_sample', + 'cpu_average', + 'cpu_total', + 'memory_bytes', + 'memory_percent', + 'timestamp', +]) + +Process = collections.namedtuple('Process', [ + 'pid', + 'name', +]) + + +class UnresolvedResult(Exception): + 'Indicates the application being used by a port is still being determined.' + + +class UnknownApplication(Exception): + 'No application could be determined for this port.' + + +def get_connection_tracker(): + """ + Singleton for tracking the connections established by tor. + """ + + global CONNECTION_TRACKER + + if CONNECTION_TRACKER is None: + CONNECTION_TRACKER = ConnectionTracker(CONFIG['queries.connections.rate']) + CONNECTION_TRACKER.start() + + return CONNECTION_TRACKER + + +def get_resource_tracker(): + """ + Singleton for tracking the resource usage of our tor process. + """ + + global RESOURCE_TRACKER + + if RESOURCE_TRACKER is None: + RESOURCE_TRACKER = ResourceTracker(CONFIG['queries.resources.rate']) + RESOURCE_TRACKER.start() + + return RESOURCE_TRACKER + + +def get_port_usage_tracker(): + """ + Singleton for tracking the process using a set of ports. + """ + + global PORT_USAGE_TRACKER + + if PORT_USAGE_TRACKER is None: + PORT_USAGE_TRACKER = PortUsageTracker(CONFIG['queries.port_usage.rate']) + PORT_USAGE_TRACKER.start() + + return PORT_USAGE_TRACKER + + +def get_consensus_tracker(): + """ + Singleton for tracking the connections established by tor. + """ + + global CONSENSUS_TRACKER + + if CONSENSUS_TRACKER is None: + CONSENSUS_TRACKER = ConsensusTracker() + + return CONSENSUS_TRACKER + + +def stop_trackers(): + """ + Halts active trackers, providing back the thread shutting them down. + + :returns: **threading.Thread** shutting down the daemons + """ + + def halt_trackers(): + trackers = filter(lambda t: t and t.is_alive(), [ + CONNECTION_TRACKER, + RESOURCE_TRACKER, + PORT_USAGE_TRACKER, + ]) + + for tracker in trackers: + tracker.stop() + + for tracker in trackers: + tracker.join() + + halt_thread = threading.Thread(target = halt_trackers) + halt_thread.setDaemon(True) + halt_thread.start() + return halt_thread + + +def _resources_via_ps(pid): + """ + Fetches resource usage information about a given process via ps. This returns + a tuple of the form... + + (total_cpu_time, uptime, memory_in_bytes, memory_in_percent) + + :param int pid: process to be queried + + :returns: **tuple** with the resource usage information + + :raises: **IOError** if unsuccessful + """ + + # ps results are of the form... + # + # TIME ELAPSED RSS %MEM + # 3-08:06:32 21-00:00:12 121844 23.5 + # + # ... or if Tor has only recently been started... + # + # TIME ELAPSED RSS %MEM + # 0:04.40 37:57 18772 0.9 + + ps_call = system.call('ps -p {pid} -o cputime,etime,rss,%mem'.format(pid = pid)) + + if ps_call and len(ps_call) >= 2: + stats = ps_call[1].strip().split() + + if len(stats) == 4: + try: + total_cpu_time = str_tools.parse_short_time_label(stats[0]) + uptime = str_tools.parse_short_time_label(stats[1]) + memory_bytes = int(stats[2]) * 1024 # ps size is in kb + memory_percent = float(stats[3]) / 100.0 + + return (total_cpu_time, uptime, memory_bytes, memory_percent) + except ValueError: + pass + + raise IOError('unrecognized output from ps: %s' % ps_call) + + +def _resources_via_proc(pid): + """ + Fetches resource usage information about a given process via proc. This + returns a tuple of the form... + + (total_cpu_time, uptime, memory_in_bytes, memory_in_percent) + + :param int pid: process to be queried + + :returns: **tuple** with the resource usage information + + :raises: **IOError** if unsuccessful + """ + + utime, stime, start_time = proc.stats( + pid, + proc.Stat.CPU_UTIME, + proc.Stat.CPU_STIME, + proc.Stat.START_TIME, + ) + + total_cpu_time = float(utime) + float(stime) + memory_in_bytes = proc.memory_usage(pid)[0] + total_memory = proc.physical_memory() + + uptime = time.time() - float(start_time) + memory_in_percent = float(memory_in_bytes) / total_memory + + return (total_cpu_time, uptime, memory_in_bytes, memory_in_percent) + + +def _process_for_ports(local_ports, remote_ports): + """ + Provides the name of the process using the given ports. + + :param list local_ports: local port numbers to look up + :param list remote_ports: remote port numbers to look up + + :returns: **dict** mapping the ports to the associated **Process**, or + **None** if it can't be determined + + :raises: **IOError** if unsuccessful + """ + + def _parse_lsof_line(line): + line_comp = line.split() + + if not line: + return None, None, None, None # blank line + elif len(line_comp) != 10: + raise ValueError('lines are expected to have ten fields: %s' % line) + elif line_comp[9] != '(ESTABLISHED)': + return None, None, None, None # connection isn't established + elif not line_comp[1].isdigit(): + raise ValueError('expected the pid (which is the second value) to be an integer: %s' % line) + + pid = int(line_comp[1]) + cmd = line_comp[0] + port_map = line_comp[8] + + if '->' not in port_map: + raise ValueError("'%s' is expected to be a '->' separated mapping" % port_map) + + local, remote = port_map.split('->', 1) + + if ':' not in local or ':' not in remote: + raise ValueError("'%s' is expected to be 'address:port' entries" % port_map) + + local_port = local.split(':', 1)[1] + remote_port = remote.split(':', 1)[1] + + if not connection.is_valid_port(local_port): + raise ValueError("'%s' isn't a valid port" % local_port) + elif not connection.is_valid_port(remote_port): + raise ValueError("'%s' isn't a valid port" % remote_port) + + return int(local_port), int(remote_port), pid, cmd + + # atagar@fenrir:~/Desktop/nyx$ lsof -i tcp:51849 -i tcp:37277 + # COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME + # tor 2001 atagar 14u IPv4 14048 0t0 TCP localhost:9051->localhost:37277 (ESTABLISHED) + # tor 2001 atagar 15u IPv4 22024 0t0 TCP localhost:9051->localhost:51849 (ESTABLISHED) + # python 2462 atagar 3u IPv4 14047 0t0 TCP localhost:37277->localhost:9051 (ESTABLISHED) + # python 3444 atagar 3u IPv4 22023 0t0 TCP localhost:51849->localhost:9051 (ESTABLISHED) + + try: + lsof_cmd = 'lsof -nP ' + ' '.join(['-i tcp:%s' % port for port in (local_ports + remote_ports)]) + lsof_call = system.call(lsof_cmd) + except OSError as exc: + raise IOError(exc) + + if lsof_call: + results = {} + + if lsof_call[0].startswith('COMMAND '): + lsof_call = lsof_call[1:] # strip the title line + + for line in lsof_call: + try: + local_port, remote_port, pid, cmd = _parse_lsof_line(line) + + if local_port in local_ports: + results[local_port] = Process(pid, cmd) + elif remote_port in remote_ports: + results[remote_port] = Process(pid, cmd) + except ValueError as exc: + raise IOError('unrecognized output from lsof (%s): %s' % (exc, line)) + + for unknown_port in set(local_ports).union(remote_ports).difference(results.keys()): + results[unknown_port] = None + + return results + + raise IOError('no results from lsof') + + +class Daemon(threading.Thread): + """ + Daemon that can perform a given action at a set rate. Subclasses are expected + to implement our _task() method with the work to be done. + """ + + def __init__(self, rate): + super(Daemon, self).__init__() + self.setDaemon(True) + + self._process_lock = threading.RLock() + self._process_pid = None + self._process_name = None + + self._rate = rate + self._last_ran = -1 # time when we last ran + self._run_counter = 0 # counter for the number of successful runs + + self._is_paused = False + self._pause_condition = threading.Condition() + self._halt = False # terminates thread if true + + controller = tor_controller() + controller.add_status_listener(self._tor_status_listener) + self._tor_status_listener(controller, stem.control.State.INIT, None) + + def run(self): + while not self._halt: + time_since_last_ran = time.time() - self._last_ran + + if self._is_paused or time_since_last_ran < self._rate: + sleep_duration = max(0.02, self._rate - time_since_last_ran) + + with self._pause_condition: + if not self._halt: + self._pause_condition.wait(sleep_duration) + + continue # done waiting, try again + + with self._process_lock: + if self._process_pid is not None: + is_successful = self._task(self._process_pid, self._process_name) + else: + is_successful = False + + if is_successful: + self._run_counter += 1 + + self._last_ran = time.time() + + def _task(self, process_pid, process_name): + """ + Task the resolver is meant to perform. This should be implemented by + subclasses. + + :param int process_pid: pid of the process we're tracking + :param str process_name: name of the process we're tracking + + :returns: **bool** indicating if our run was successful or not + """ + + return True + + def run_counter(self): + """ + Provides the number of times we've successful runs so far. This can be used + by callers to determine if our results have been seen by them before or + not. + + :returns: **int** for the run count we're on + """ + + return self._run_counter + + def get_rate(self): + """ + Provides the rate at which we perform our task. + + :returns: **float** for the rate in seconds at which we perform our task + """ + + return self._rate + + def set_rate(self, rate): + """ + Sets the rate at which we perform our task in seconds. + + :param float rate: rate at which to perform work in seconds + """ + + self._rate = rate + + def set_paused(self, pause): + """ + Either resumes or holds off on doing further work. + + :param bool pause: halts work if **True**, resumes otherwise + """ + + self._is_paused = pause + + def stop(self): + """ + Halts further work and terminates the thread. + """ + + with self._pause_condition: + self._halt = True + self._pause_condition.notifyAll() + + def _tor_status_listener(self, controller, event_type, _): + with self._process_lock: + if not self._halt and event_type in (stem.control.State.INIT, stem.control.State.RESET): + tor_pid = controller.get_pid(None) + tor_cmd = system.name_by_pid(tor_pid) if tor_pid else None + + self._process_pid = tor_pid + self._process_name = tor_cmd if tor_cmd else 'tor' + elif event_type == stem.control.State.CLOSED: + self._process_pid = None + self._process_name = None + + def __enter__(self): + self.start() + return self + + def __exit__(self, exit_type, value, traceback): + self.stop() + self.join() + + +class ConnectionTracker(Daemon): + """ + Periodically retrieves the connections established by tor. + """ + + def __init__(self, rate): + super(ConnectionTracker, self).__init__(rate) + + self._connections = [] + self._start_times = {} # connection => (unix_timestamp, is_legacy) + self._custom_resolver = None + self._is_first_run = True + + # Number of times in a row we've either failed with our current resolver or + # concluded that our rate is too low. + + self._failure_count = 0 + self._rate_too_low_count = 0 + + # If 'DisableDebuggerAttachment 0' is set we can do normal connection + # resolution. Otherwise connection resolution by inference is the only game + # in town. + + if tor_controller().get_conf('DisableDebuggerAttachment', None) == '0': + self._resolvers = connection.system_resolvers() + else: + self._resolvers = [CustomResolver.INFERENCE] + + log.info('tracker.available_resolvers', os = os.uname()[0], resolvers = ', '.join(self._resolvers)) + + def _task(self, process_pid, process_name): + if self._custom_resolver: + resolver = self._custom_resolver + is_default_resolver = False + elif self._resolvers: + resolver = self._resolvers[0] + is_default_resolver = True + else: + return False # nothing to resolve with + + try: + start_time = time.time() + new_connections, new_start_times = [], {} + + if resolver == CustomResolver.INFERENCE: + # provide connections going to a relay or one of our tor ports + + connections = [] + controller = tor_controller() + consensus_tracker = get_consensus_tracker() + + for conn in proc.connections(user = controller.get_user(None)): + if conn.remote_port in consensus_tracker.get_relay_fingerprints(conn.remote_address): + connections.append(conn) # outbound to another relay + elif conn.local_port in controller.get_ports(stem.control.Listener.OR, []): + connections.append(conn) # inbound to our ORPort + elif conn.local_port in controller.get_ports(stem.control.Listener.DIR, []): + connections.append(conn) # inbound to our DirPort + elif conn.local_port in controller.get_ports(stem.control.Listener.CONTROL, []): + connections.append(conn) # controller connection + else: + connections = connection.get_connections(resolver, process_pid = process_pid, process_name = process_name) + + for conn in connections: + conn_start_time, is_legacy = self._start_times.get(conn, (start_time, self._is_first_run)) + new_start_times[conn] = (conn_start_time, is_legacy) + new_connections.append(Connection(conn_start_time, is_legacy, *conn)) + + self._connections = new_connections + self._start_times = new_start_times + self._is_first_run = False + + runtime = time.time() - start_time + + if is_default_resolver: + self._failure_count = 0 + + # Reduce our rate if connection resolution is taking a long time. This is + # most often an issue for extremely busy relays. + + min_rate = 100 * runtime + + if self.get_rate() < min_rate: + self._rate_too_low_count += 1 + + if self._rate_too_low_count >= 3: + min_rate += 1 # little extra padding so we don't frequently update this + self.set_rate(min_rate) + self._rate_too_low_count = 0 + log.debug('tracker.lookup_rate_increased', seconds = "%0.1f" % min_rate) + else: + self._rate_too_low_count = 0 + + return True + except IOError as exc: + log.info('wrap', text = exc) + + # Fail over to another resolver if we've repeatedly been unable to use + # this one. + + if is_default_resolver: + self._failure_count += 1 + + if self._failure_count >= 3: + self._resolvers.pop(0) + self._failure_count = 0 + + if self._resolvers: + log.notice( + 'tracker.unable_to_use_resolver', + old_resolver = resolver, + new_resolver = self._resolvers[0], + ) + else: + log.notice('tracker.unable_to_use_all_resolvers') + + return False + + def get_custom_resolver(self): + """ + Provides the custom resolver the user has selected. This is **None** if + we're picking resolvers dynamically. + + :returns: :data:`~stem.util.connection.Resolver` we're overwritten to use + """ + + return self._custom_resolver + + def set_custom_resolver(self, resolver): + """ + Sets the resolver used for connection resolution. If **None** then this is + automatically determined based on what is available. + + :param stem.util.connection.Resolver resolver: resolver to use + """ + + self._custom_resolver = resolver + + def get_value(self): + """ + Provides a listing of tor's latest connections. + + :returns: **list** of :class:`~nyx.tracker.Connection` we last + retrieved, an empty list if our tracker's been stopped + """ + + if self._halt: + return [] + else: + return list(self._connections) + + +class ResourceTracker(Daemon): + """ + Periodically retrieves the resource usage of tor. + """ + + def __init__(self, rate): + super(ResourceTracker, self).__init__(rate) + + self._resources = None + self._use_proc = proc.is_available() # determines if we use proc or ps for lookups + self._failure_count = 0 # number of times in a row we've failed to get results + + def get_value(self): + """ + Provides tor's latest resource usage. + + :returns: latest :data:`~nyx.tracker.Resources` we've polled + """ + + result = self._resources + return result if result else Resources(0.0, 0.0, 0.0, 0, 0.0, 0.0) + + def _task(self, process_pid, process_name): + try: + resolver = _resources_via_proc if self._use_proc else _resources_via_ps + total_cpu_time, uptime, memory_in_bytes, memory_in_percent = resolver(process_pid) + + if self._resources: + cpu_sample = (total_cpu_time - self._resources.cpu_total) / self._resources.cpu_total + else: + cpu_sample = 0.0 # we need a prior datapoint to give a sampling + + self._resources = Resources( + cpu_sample = cpu_sample, + cpu_average = total_cpu_time / uptime, + cpu_total = total_cpu_time, + memory_bytes = memory_in_bytes, + memory_percent = memory_in_percent, + timestamp = time.time(), + ) + + self._failure_count = 0 + return True + except IOError as exc: + self._failure_count += 1 + + if self._use_proc: + if self._failure_count >= 3: + # We've failed three times resolving via proc. Warn, and fall back + # to ps resolutions. + + self._use_proc = False + self._failure_count = 0 + + log.info( + 'tracker.abort_getting_resources', + resolver = 'proc', + response = 'falling back to ps', + exc = exc, + ) + else: + log.debug('tracker.unable_to_get_resources', resolver = 'proc', exc = exc) + else: + if self._failure_count >= 3: + # Give up on further attempts. + + log.info( + 'tracker.abort_getting_resources', + resolver = 'ps', + response = 'giving up on getting resource usage information', + exc = exc, + ) + + self.stop() + else: + log.debug('tracker.unable_to_get_resources', resolver = 'ps', exc = exc) + + return False + + +class PortUsageTracker(Daemon): + """ + Periodically retrieves the processes using a set of ports. + """ + + def __init__(self, rate): + super(PortUsageTracker, self).__init__(rate) + + self._last_requested_local_ports = [] + self._last_requested_remote_ports = [] + self._processes_for_ports = {} + self._failure_count = 0 # number of times in a row we've failed to get results + + def fetch(self, port): + """ + Provides the process running on the given port. This retrieves the results + from our cache, so it only works if we've already issued a query() request + for it and gotten results. + + :param int port: port number to look up + + :returns: **Process** using the given port + + :raises: + * :class:`nyx.tracker.UnresolvedResult` if the application is still + being determined + * :class:`nyx.tracker.UnknownApplication` if the we tried to resolve + the application but it couldn't be determined + """ + + try: + result = self._processes_for_ports[port] + + if result is None: + raise UnknownApplication() + else: + return result + except KeyError: + raise UnresolvedResult() + + def query(self, local_ports, remote_ports): + """ + Registers a given set of ports for further lookups, and returns the last + set of 'port => process' mappings we retrieved. Note that this means that + we will not return the requested ports unless they're requested again after + a successful lookup has been performed. + + :param list local_ports: local port numbers to look up + :param list remote_ports: remote port numbers to look up + + :returns: **dict** mapping port numbers to the **Process** using it + """ + + self._last_requested_local_ports = local_ports + self._last_requested_remote_ports = remote_ports + return self._processes_for_ports + + def _task(self, process_pid, process_name): + local_ports = self._last_requested_local_ports + remote_ports = self._last_requested_remote_ports + + if not local_ports and not remote_ports: + return True + + result = {} + + # Use cached results from our last lookup if available. + + for port, process in self._processes_for_ports.items(): + if port in local_ports: + result[port] = process + local_ports.remove(port) + elif port in remote_ports: + result[port] = process + remote_ports.remove(port) + + try: + if local_ports or remote_ports: + result.update(_process_for_ports(local_ports, remote_ports)) + + self._processes_for_ports = result + self._failure_count = 0 + return True + except IOError as exc: + self._failure_count += 1 + + if self._failure_count >= 3: + log.info('tracker.abort_getting_port_usage', exc = exc) + self.stop() + else: + log.debug('tracker.unable_to_get_port_usages', exc = exc) + + return False + + +class ConsensusTracker(object): + """ + Provides performant lookups of consensus information. + """ + + def __init__(self): + self._fingerprint_cache = {} # {address => [(port, fingerprint), ..]} for relays + self._nickname_cache = {} # fingerprint => nickname lookup cache + self._address_cache = {} + + tor_controller().add_event_listener(self._new_consensus_event, stem.control.EventType.NEWCONSENSUS) + + def _new_consensus_event(self, event): + self.update(event.desc) + + def update(self, router_status_entries): + """ + Updates our cache with the given router status entries. + + :param list router_status_entries: router status entries to populate our cache with + """ + + new_fingerprint_cache = {} + new_address_cache = {} + new_nickname_cache = {} + + for desc in router_status_entries: + new_fingerprint_cache.setdefault(desc.address, []).append((desc.or_port, desc.fingerprint)) + new_address_cache[desc.fingerprint] = (desc.address, desc.or_port) + new_nickname_cache[desc.fingerprint] = desc.nickname if desc.nickname else 'Unnamed' + + self._fingerprint_cache = new_fingerprint_cache + self._address_cache = new_address_cache + self._nickname_cache = new_nickname_cache + + def get_relay_nickname(self, fingerprint): + """ + Provides the nickname associated with the given relay. + + :param str fingerprint: relay to look up + + :returns: **str** with the nickname ("Unnamed" if unset), and **None** if + no such relay exists + """ + + controller = tor_controller() + + if not fingerprint: + return None + elif fingerprint == controller.get_info('fingerprint', None): + return controller.get_conf('Nickname', 'Unnamed') + else: + return self._nickname_cache.get(fingerprint) + + def get_relay_fingerprints(self, address): + """ + Provides the relays running at a given location. + + :param str address: address to be checked + + :returns: **dict** of ORPorts to their fingerprint + """ + + controller = tor_controller() + + if address == controller.get_info('address', None): + fingerprint = controller.get_info('fingerprint', None) + ports = controller.get_ports(stem.control.Listener.OR, None) + + if fingerprint and ports: + return dict([(port, fingerprint) for port in ports]) + + return dict([(port, fp) for (port, fp) in self._fingerprint_cache.get(address, [])]) + + def get_relay_address(self, fingerprint, default): + """ + Provides the (address, port) tuple where a relay is running. + + :param str fingerprint: fingerprint to be checked + + :returns: **tuple** with a **str** address and **int** port + """ + + controller = tor_controller() + + if fingerprint == controller.get_info('fingerprint', None): + my_address = controller.get_info('address', None) + my_or_ports = controller.get_ports(stem.control.Listener.OR, []) + + if my_address and len(my_or_ports) == 1: + return (my_address, my_or_ports[0]) + + return self._address_cache.get(fingerprint, default) diff --git a/nyx/util/__init__.py b/nyx/util/__init__.py deleted file mode 100644 index 176753b..0000000 --- a/nyx/util/__init__.py +++ /dev/null @@ -1,146 +0,0 @@ -""" -General purpose utilities for a variety of tasks supporting nyx features and -safely working with curses (hiding some of the gory details). -""" - -import os -import sys - -import stem.connection -import stem.control -import stem.util.conf -import stem.util.log -import stem.util.system - -from nyx.util import log - -__all__ = [ - 'log', - 'panel', - 'tracker', -] - -TOR_CONTROLLER = None -BASE_DIR = os.path.sep.join(__file__.split(os.path.sep)[:-2]) -DATA_DIR = os.path.expanduser('~/.nyx') -TESTING = False - -# technically can change but we use this query a *lot* so needs to be cached - -stem.control.CACHEABLE_GETINFO_PARAMS = list(stem.control.CACHEABLE_GETINFO_PARAMS) + ['address'] - -# disable trace level messages about cache hits - -stem.control.LOG_CACHE_FETCHES = False - -try: - uses_settings = stem.util.conf.uses_settings('nyx', os.path.join(BASE_DIR, 'settings'), lazy_load = False) -except IOError as exc: - print("Unable to load nyx's internal configurations: %s" % exc) - sys.exit(1) - - -def tor_controller(): - """ - Singleton for getting our tor controller connection. - - :returns: :class:`~stem.control.Controller` nyx is using - """ - - return TOR_CONTROLLER - - -def init_controller(*args, **kwargs): - """ - Sets the Controller used by nyx. This is a passthrough for Stem's - :func:`~stem.connection.connect` function. - - :returns: :class:`~stem.control.Controller` nyx is using - """ - - global TOR_CONTROLLER - TOR_CONTROLLER = stem.connection.connect(*args, **kwargs) - return TOR_CONTROLLER - - -@uses_settings -def expand_path(path, config): - """ - Expands relative paths and include our chroot if one was set. - - :param str path: path to be expanded - - :returns: **str** with the expanded path - """ - - if path is None: - return None - - try: - chroot = config.get('tor.chroot', '') - tor_cwd = stem.util.system.cwd(tor_controller().get_pid(None)) - return chroot + stem.util.system.expand_path(path, tor_cwd) - except IOError as exc: - stem.util.log.info('Unable to expand a relative path (%s): %s' % (path, exc)) - return path - - -def join(entries, joiner = ' ', size = None): - """ - Joins a series of strings similar to str.join(), but only up to a given size. - This returns an empty string if none of the entries will fit. For example... - - >>> join(['This', 'is', 'a', 'looooong', 'message'], size = 18) - 'This is a looooong' - - >>> join(['This', 'is', 'a', 'looooong', 'message'], size = 17) - 'This is a' - - >>> join(['This', 'is', 'a', 'looooong', 'message'], size = 2) - '' - - :param list entries: strings to be joined - :param str joiner: strings to join the entries with - :param int size: maximum length the result can be, there's no length - limitation if **None** - - :returns: **str** of the joined entries up to the given length - """ - - if size is None: - return joiner.join(entries) - - result = '' - - for entry in entries: - new_result = joiner.join((result, entry)) if result else entry - - if len(new_result) > size: - break - else: - result = new_result - - return result - - -@uses_settings -def msg(message, config, **attr): - """ - Provides the given message. - - :param str message: message handle to log - :param dict attr: attributes to format the message with - - :returns: **str** that was requested - """ - - try: - return config.get('msg.%s' % message).format(**attr) - except: - msg = 'BUG: We attempted to use an undefined string resource (%s)' % message - - if TESTING: - raise ValueError(msg) - - stem.util.log.notice(msg) - return '' diff --git a/nyx/util/log.py b/nyx/util/log.py deleted file mode 100644 index e4f962b..0000000 --- a/nyx/util/log.py +++ /dev/null @@ -1,537 +0,0 @@ -""" -Logging utilities, primiarily short aliases for logging a message at various -runlevels. - -:: - - trace - logs a message at the TRACE runlevel - debug - logs a message at the DEBUG runlevel - info - logs a message at the INFO runlevel - notice - logs a message at the NOTICE runlevel - warn - logs a message at the WARN runlevel - error - logs a message at the ERROR runlevel - - day_count - number of days since a given timestamp - log_file_path - path of tor's log file if one is present on disk - condense_runlevels - condensed displayable listing of log events - listen_for_events - notifies listener of tor events - read_tor_log - provides LogEntry from a tor log file - - LogGroup - thread safe, deduplicated grouping of events - |- add - adds an event to the group - +- pop - removes and returns an event - - LogEntry - individual log event - |- is_duplicate_of - checks if a duplicate message of another LogEntry - +- day_count - number of days since this even occured - - LogFileOutput - writes log events to a file - +- write - persist a given message - - LogFilters - regex filtering of log events - |- select - filters by this regex - |- selection - current regex filter - |- latest_selections - past regex selections - |- match - checks if a LogEntry matches this filter - +- clone - deep clone of this LogFilters -""" - -import collections -import datetime -import os -import re -import time -import threading - -import stem.util.conf -import stem.util.log -import stem.util.system - -import nyx -import nyx.util - -try: - # added in python 3.2 - from functools import lru_cache -except ImportError: - from stem.util.lru_cache import lru_cache - -TOR_RUNLEVELS = ['DEBUG', 'INFO', 'NOTICE', 'WARN', 'ERR'] -TIMEZONE_OFFSET = time.altzone if time.localtime()[8] else time.timezone - - -def day_count(timestamp): - """ - Provoides a unique number for the day a given timestamp falls on, by local - time. Daybreaks are rolled over at midnight. - - :param int timestamp: unix timestamp to provide a count for - - :reutrns: **int** for the day it falls on - """ - - return int((timestamp - TIMEZONE_OFFSET) / 86400) - - -def log_file_path(controller): - """ - Provides the path where tor's log file resides, if one exists. - - :params stem.control.Controller controller: tor controller connection - - :returns: **str** with the absolute path of our log file, or **None** if one - doesn't exist - """ - - for log_entry in controller.get_conf('Log', [], True): - entry_comp = log_entry.split() # looking for an entry like: notice file /var/log/tor/notices.log - - if entry_comp[1] == 'file': - return nyx.util.expand_path(entry_comp[2]) - - -@lru_cache() -def condense_runlevels(*events): - """ - Provides runlevel events with condensed. For example... - - >>> condense_runlevels('DEBUG', 'NOTICE', 'WARN', 'ERR', 'NYX_NOTICE', 'NYX_WARN', 'NYX_ERR', 'BW') - ['TOR/NYX NOTICE-ERROR', 'DEBUG', 'BW'] - - :param list events: event types to be condensed - - :returns: **list** of the input events, with condensed runlevels - """ - - def ranges(runlevels): - ranges = [] - - while runlevels: - # provides the (start, end) for a contiguous range - start = end = runlevels[0] - - for r in TOR_RUNLEVELS[TOR_RUNLEVELS.index(start):]: - if r in runlevels: - runlevels.remove(r) - end = r - else: - break - - ranges.append((start, end)) - - return ranges - - events = list(events) - tor_runlevels, nyx_runlevels = [], [] - - for r in TOR_RUNLEVELS: - if r in events: - tor_runlevels.append(r) - events.remove(r) - - if 'NYX_%s' % r in events: - nyx_runlevels.append(r) - events.remove('NYX_%s' % r) - - tor_ranges = ranges(tor_runlevels) - nyx_ranges = ranges(nyx_runlevels) - - result = [] - - for runlevel_range in tor_ranges: - if runlevel_range[0] == runlevel_range[1]: - range_label = runlevel_range[0] - else: - range_label = '%s-%s' % (runlevel_range[0], runlevel_range[1]) - - if runlevel_range in nyx_ranges: - result.append('TOR/NYX %s' % range_label) - nyx_ranges.remove(runlevel_range) - else: - result.append(range_label) - - for runlevel_range in nyx_ranges: - if runlevel_range[0] == runlevel_range[1]: - result.append('NYX %s' % runlevel_range[0]) - else: - result.append('NYX %s-%s' % (runlevel_range[0], runlevel_range[1])) - - return result + events - - -def listen_for_events(listener, events): - """ - Configures tor to notify a function of these event types. If tor is - configured to notify this listener then the old listener is replaced. - - :param function listener: listener to be notified - :param list events: event types to attempt to set - - :returns: **list** of event types we're successfully now listening to - """ - - import nyx.arguments - events = set(events) # drops duplicates - - # accounts for runlevel naming difference - - tor_events = events.intersection(set(nyx.arguments.TOR_EVENT_TYPES.values())) - nyx_events = events.intersection(set(['NYX_%s' % runlevel for runlevel in TOR_RUNLEVELS])) - - # adds events unrecognized by nyx if we're listening to the 'UNKNOWN' type - - if 'UNKNOWN' in events: - tor_events.update(set(nyx.arguments.missing_event_types())) - - controller = nyx.util.tor_controller() - controller.remove_event_listener(listener) - - for event_type in list(tor_events): - try: - controller.add_event_listener(listener, event_type) - except stem.ProtocolError: - tor_events.remove(event_type) - - return sorted(tor_events.union(nyx_events)) - - -@lru_cache() -def _common_log_messages(): - """ - Provides a mapping of message types to its common log messages. These are - message prefixes unless it starts with an asterisk, in which case it can - appear anywhere in the message. - - :returns: **dict** of the form {event_type => [msg1, msg2...]} - """ - - nyx_config, messages = stem.util.conf.get_config('nyx'), {} - - for conf_key in nyx_config.keys(): - if conf_key.startswith('dedup.'): - event_type = conf_key[6:] - messages[event_type] = nyx_config.get(conf_key, []) - - return messages - - -class LogGroup(object): - """ - Thread safe collection of LogEntry instancs, which maintains a certain size - and supports deduplication. - """ - - def __init__(self, max_size, group_by_day = False): - self._max_size = max_size - self._group_by_day = group_by_day - self._entries = [] - self._lock = threading.RLock() - - def add(self, entry): - with self._lock: - duplicate = None - our_day = entry.day_count() - - for existing_entry in self._entries: - if self._group_by_day and our_day != existing_entry.day_count(): - break - elif entry.is_duplicate_of(existing_entry): - duplicate = existing_entry - break - - if duplicate: - if not duplicate.duplicates: - duplicate.duplicates = [duplicate] - - duplicate.is_duplicate = True - entry.duplicates = duplicate.duplicates - entry.duplicates.insert(0, entry) - - self._entries.insert(0, entry) - - while len(self._entries) > self._max_size: - self.pop() - - def pop(self): - with self._lock: - last_entry = self._entries.pop() - - # By design if the last entry is a duplicate it will also be the last - # item in its duplicate group. - - if last_entry.is_duplicate: - last_entry.duplicates.pop() - - def __len__(self): - with self._lock: - return len(self._entries) - - def __iter__(self): - with self._lock: - for entry in self._entries: - yield entry - - -class LogEntry(object): - """ - Individual tor or nyx log entry. - - **Note:** Tor doesn't include the date in its timestamps so the year - component may be inaccurate. (:trac:`15607`) - - :var int timestamp: unix timestamp for when the event occured - :var str type: event type - :var str message: event's message - :var str display_message: message annotated with our time and runlevel - - :var bool is_duplicate: true if this matches other messages in the group and - isn't the first - :var list duplicates: messages that are identical to thsi one - """ - - def __init__(self, timestamp, type, message): - self.timestamp = timestamp - self.type = type - self.message = message - - entry_time = time.localtime(self.timestamp) - self.display_message = '%02i:%02i:%02i [%s] %s' % (entry_time[3], entry_time[4], entry_time[5], self.type, self.message) - - self.is_duplicate = False - self.duplicates = None - - @lru_cache() - def is_duplicate_of(self, entry): - """ - Checks if we are a duplicate of the given message or not. - - :returns: **True** if the given log message is a duplicate of us and **False** otherwise - """ - - if self.type != entry.type: - return False - elif self.message == entry.message: - return True - - if self.type == 'NYX_DEBUG' and 'runtime:' in self.message and 'runtime:' in entry.message: - # most nyx debug messages show runtimes so try matching without that - - if self.message[:self.message.find('runtime:')] == entry.message[:self.message.find('runtime:')]: - return True - - for common_msg in _common_log_messages().get(self.type, []): - # if it starts with an asterisk then check the whole message rather - # than just the start - - if common_msg[0] == '*': - if common_msg[1:] in self.message and common_msg[1:] in entry.message: - return True - else: - if self.message.startswith(common_msg) and entry.message.startswith(common_msg): - return True - - return False - - def day_count(self): - """ - Provides the day this event occured on by local time. - - :reutrns: **int** with the day this occured on - """ - - return day_count(self.timestamp) - - def __eq__(self, other): - if isinstance(other, LogEntry): - return hash(self) == hash(other) - else: - return False - - def __hash__(self): - return hash(self.display_message) - - -class LogFileOutput(object): - """ - File where log messages we receive are written. If unable to do so then a - notification is logged and further write attempts are skipped. - """ - - def __init__(self, path): - self._file = None - - if path: - try: - path_dir = os.path.dirname(path) - - if not os.path.exists(path_dir): - os.makedirs(path_dir) - - self._file = open(path, 'a') - notice('nyx %s opening log file (%s)' % (nyx.__version__, path)) - except IOError as exc: - error('Unable to write to log file: %s' % exc.strerror) - except OSError as exc: - error('Unable to write to log file: %s' % exc) - - def write(self, msg): - if self._file: - try: - self._file.write(msg + '\n') - self._file.flush() - except IOError as exc: - error('Unable to write to log file: %s' % exc.strerror) - self._file = None - - -class LogFilters(object): - """ - Regular expression filtering for log output. This is thread safe and tracks - the latest selections. - """ - - def __init__(self, initial_filters = None, max_filters = 5): - self._max_filters = max_filters - self._selected = None - self._past_filters = collections.OrderedDict() - self._lock = threading.RLock() - - if initial_filters: - for regex in initial_filters: - self.select(regex) - - self.select(None) - - def select(self, regex): - with self._lock: - if regex is None: - self._selected = None - return - - if regex in self._past_filters: - del self._past_filters[regex] - - try: - self._past_filters[regex] = re.compile(regex) - self._selected = regex - - if len(self._past_filters) > self._max_filters: - self._past_filters.popitem(False) - except re.error as exc: - notice('Invalid regular expression pattern (%s): %s' % (exc, regex)) - - def selection(self): - return self._selected - - def latest_selections(self): - return list(reversed(self._past_filters.keys())) - - def match(self, message): - regex_filter = self._past_filters.get(self._selected) - return not regex_filter or bool(regex_filter.search(message)) - - def clone(self): - with self._lock: - clone = LogFilters(max_filters = self._max_filters) - clone._selected = self._selected - clone._past_filters = self._past_filters - return clone - - -def trace(msg, **attr): - _log(stem.util.log.TRACE, msg, **attr) - - -def debug(msg, **attr): - _log(stem.util.log.DEBUG, msg, **attr) - - -def info(msg, **attr): - _log(stem.util.log.INFO, msg, **attr) - - -def notice(msg, **attr): - _log(stem.util.log.NOTICE, msg, **attr) - - -def warn(msg, **attr): - _log(stem.util.log.WARN, msg, **attr) - - -def error(msg, **attr): - _log(stem.util.log.ERROR, msg, **attr) - - -def _log(runlevel, message, **attr): - """ - Logs the given message, formatted with optional attributes. - - :param stem.util.log.Runlevel runlevel: runlevel at which to log the message - :param str message: message handle to log - :param dict attr: attributes to format the message with - """ - - stem.util.log.log(runlevel, nyx.util.msg(message, **attr)) - - -def read_tor_log(path, read_limit = None): - """ - Provides logging messages from a tor log file, from newest to oldest. - - :param str path: logging location to read from - :param int read_limit: maximum number of lines to read from the file - - :returns: **iterator** for **LogEntry** for the file's contents - - :raises: - * **ValueError** if the log file has unrecognized content - * **IOError** if unable to read the file - """ - - start_time = time.time() - count, isdst = 0, time.localtime().tm_isdst - - for line in stem.util.system.tail(path, read_limit): - # entries look like: - # Jul 15 18:29:48.806 [notice] Parsing GEOIP file. - - line_comp = line.split() - - # Checks that we have all the components we expect. This could happen if - # we're either not parsing a tor log or in weird edge cases (like being - # out of disk space). - - if len(line_comp) < 4: - raise ValueError("Log located at %s has a line that doesn't match the format we expect: %s" % (path, line)) - elif len(line_comp[3]) < 3 or line_comp[3][1:-1].upper() not in TOR_RUNLEVELS: - raise ValueError('Log located at %s has an unrecognized runlevel: %s' % (path, line_comp[3])) - - runlevel = line_comp[3][1:-1].upper() - msg = ' '.join(line_comp[4:]) - current_year = str(datetime.datetime.now().year) - - # Pretending it's the current year. We don't know the actual year (#15607) - # and this may fail due to leap years when picking Feb 29th (#5265). - - try: - timestamp_str = current_year + ' ' + ' '.join(line_comp[:3]) - timestamp_str = timestamp_str.split('.', 1)[0] # drop fractional seconds - timestamp_comp = list(time.strptime(timestamp_str, '%Y %b %d %H:%M:%S')) - timestamp_comp[8] = isdst - - timestamp = int(time.mktime(tuple(timestamp_comp))) # converts local to unix time - - if timestamp > time.time(): - # log entry is from before a year boundary - timestamp_comp[0] -= 1 - timestamp = int(time.mktime(timestamp_comp)) - except ValueError: - raise ValueError("Log located at %s has a timestamp we don't recognize: %s" % (path, ' '.join(line_comp[:3]))) - - count += 1 - yield LogEntry(timestamp, runlevel, msg) - - if 'opening log file' in msg: - break # this entry marks the start of this tor instance - - info('panel.log.read_from_log_file', count = count, path = path, read_limit = read_limit if read_limit else 'none', runtime = '%0.3f' % (time.time() - start_time)) diff --git a/nyx/util/panel.py b/nyx/util/panel.py deleted file mode 100644 index 43e0236..0000000 --- a/nyx/util/panel.py +++ /dev/null @@ -1,841 +0,0 @@ -""" -Wrapper for safely working with curses subwindows. -""" - -import copy -import time -import curses -import curses.ascii -import curses.textpad -from threading import RLock - -import nyx.curses - -from nyx.curses import HIGHLIGHT -from stem.util import conf, log, str_tools - -# global ui lock governing all panel instances (curses isn't thread save and -# concurrency bugs produce especially sinister glitches) - -CURSES_LOCK = RLock() - -SCROLL_KEYS = (curses.KEY_UP, curses.KEY_DOWN, curses.KEY_PPAGE, curses.KEY_NPAGE, curses.KEY_HOME, curses.KEY_END) - -SPECIAL_KEYS = { - 'up': curses.KEY_UP, - 'down': curses.KEY_DOWN, - 'left': curses.KEY_LEFT, - 'right': curses.KEY_RIGHT, - 'home': curses.KEY_HOME, - 'end': curses.KEY_END, - 'page_up': curses.KEY_PPAGE, - 'page_down': curses.KEY_NPAGE, - 'esc': 27, -} - -PASS = -1 - - -def conf_handler(key, value): - if key == 'features.torrc.maxLineWrap': - return max(1, value) - - -CONFIG = conf.config_dict('nyx', { - 'features.maxLineWrap': 8, -}, conf_handler) - -# prevents curses redraws if set -HALT_ACTIVITY = False - - -class BasicValidator(object): - """ - Interceptor for keystrokes given to a textbox, doing the following: - - quits by setting the input to curses.ascii.BEL when escape is pressed - - stops the cursor at the end of the box's content when pressing the right - arrow - - home and end keys move to the start/end of the line - """ - - def validate(self, key, textbox): - """ - Processes the given key input for the textbox. This may modify the - textbox's content, cursor position, etc depending on the functionality - of the validator. This returns the key that the textbox should interpret, - PASS if this validator doesn't want to take any action. - - Arguments: - key - key code input from the user - textbox - curses Textbox instance the input came from - """ - - result = self.handle_key(key, textbox) - return key if result == PASS else result - - def handle_key(self, key, textbox): - y, x = textbox.win.getyx() - - if curses.ascii.isprint(key) and x < textbox.maxx: - # Shifts the existing text forward so input is an insert method rather - # than replacement. The curses.textpad accepts an insert mode flag but - # this has a couple issues... - # - The flag is only available for Python 2.6+, before that the - # constructor only accepted a subwindow argument as per: - # https://trac.torproject.org/projects/tor/ticket/2354 - # - The textpad doesn't shift text that has text attributes. This is - # because keycodes read by textbox.win.inch() includes formatting, - # causing the curses.ascii.isprint() check it does to fail. - - current_input = textbox.gather() - textbox.win.addstr(y, x + 1, current_input[x:textbox.maxx - 1]) - textbox.win.move(y, x) # reverts cursor movement during gather call - elif key == 27: - # curses.ascii.BEL is a character codes that causes textpad to terminate - - return curses.ascii.BEL - elif key == curses.KEY_HOME: - textbox.win.move(y, 0) - return None - elif key in (curses.KEY_END, curses.KEY_RIGHT): - msg_length = len(textbox.gather()) - textbox.win.move(y, x) # reverts cursor movement during gather call - - if key == curses.KEY_END and msg_length > 0 and x < msg_length - 1: - # if we're in the content then move to the end - - textbox.win.move(y, msg_length - 1) - return None - elif key == curses.KEY_RIGHT and x >= msg_length - 1: - # don't move the cursor if there's no content after it - - return None - elif key == 410: - # if we're resizing the display during text entry then cancel it - # (otherwise the input field is filled with nonprintable characters) - - return curses.ascii.BEL - - return PASS - - -class Panel(object): - """ - Wrapper for curses subwindows. This hides most of the ugliness in common - curses operations including: - - locking when concurrently drawing to multiple windows - - gracefully handle terminal resizing - - clip text that falls outside the panel - - convenience methods for word wrap, in-line formatting, etc - - This uses a design akin to Swing where panel instances provide their display - implementation by overwriting the draw() method, and are redrawn with - redraw(). - """ - - def __init__(self, parent, name, top, left = 0, height = -1, width = -1): - """ - Creates a durable wrapper for a curses subwindow in the given parent. - - Arguments: - parent - parent curses window - name - identifier for the panel - top - positioning of top within parent - left - positioning of the left edge within the parent - height - maximum height of panel (uses all available space if -1) - width - maximum width of panel (uses all available space if -1) - """ - - # The not-so-pythonic getters for these parameters are because some - # implementations aren't entirely deterministic (for instance panels - # might chose their height based on its parent's current width). - - self.panel_name = name - self.parent = parent - self.visible = False - self.title_visible = True - - # Attributes for pausing. The pause_attr contains variables our get_attr - # method is tracking, and the pause buffer has copies of the values from - # when we were last unpaused (unused unless we're paused). - - self.paused = False - self.pause_attr = [] - self.pause_buffer = {} - self.pause_time = -1 - - self.top = top - self.left = left - self.height = height - self.width = width - - # The panel's subwindow instance. This is made available to implementors - # via their draw method and shouldn't be accessed directly. - # - # This is None if either the subwindow failed to be created or needs to be - # remade before it's used. The later could be for a couple reasons: - # - The subwindow was never initialized. - # - Any of the parameters used for subwindow initialization have changed. - - self.win = None - - self.max_y, self.max_x = -1, -1 # subwindow dimensions when last redrawn - - def get_name(self): - """ - Provides panel's identifier. - """ - - return self.panel_name - - def is_title_visible(self): - """ - True if the title is configured to be visible, False otherwise. - """ - - return self.title_visible - - def set_title_visible(self, is_visible): - """ - Configures the panel's title to be visible or not when it's next redrawn. - This is not guarenteed to be respected (not all panels have a title). - """ - - self.title_visible = is_visible - - def get_parent(self): - """ - Provides the parent used to create subwindows. - """ - - return self.parent - - def set_visible(self, is_visible): - """ - Toggles if the panel is visible or not. - - Arguments: - is_visible - panel is redrawn when requested if true, skipped otherwise - """ - - self.visible = is_visible - - def is_paused(self): - """ - Provides if the panel's configured to be paused or not. - """ - - return self.paused - - def set_pause_attr(self, attr): - """ - Configures the panel to track the given attribute so that get_attr provides - the value when it was last unpaused (or its current value if we're - currently unpaused). For instance... - - > self.set_pause_attr('myVar') - > self.myVar = 5 - > self.myVar = 6 # self.get_attr('myVar') -> 6 - > self.set_paused(True) - > self.myVar = 7 # self.get_attr('myVar') -> 6 - > self.set_paused(False) - > self.myVar = 7 # self.get_attr('myVar') -> 7 - - Arguments: - attr - parameter to be tracked for get_attr - """ - - self.pause_attr.append(attr) - self.pause_buffer[attr] = self.copy_attr(attr) - - def get_attr(self, attr): - """ - Provides the value of the given attribute when we were last unpaused. If - we're currently unpaused then this is the current value. If untracked this - returns None. - - Arguments: - attr - local variable to be returned - """ - - if attr not in self.pause_attr: - return None - elif self.paused: - return self.pause_buffer[attr] - else: - return self.__dict__.get(attr) - - def copy_attr(self, attr): - """ - Provides a duplicate of the given configuration value, suitable for the - pause buffer. - - Arguments: - attr - parameter to be provided back - """ - - current_value = self.__dict__.get(attr) - return copy.copy(current_value) - - def set_paused(self, is_pause, suppress_redraw = False): - """ - Toggles if the panel is paused or not. This causes the panel to be redrawn - when toggling is pause state unless told to do otherwise. This is - important when pausing since otherwise the panel's display could change - when redrawn for other reasons. - - This returns True if the panel's pause state was changed, False otherwise. - - Arguments: - is_pause - freezes the state of the pause attributes if true, makes - them editable otherwise - suppress_redraw - if true then this will never redraw the panel - """ - - if is_pause != self.paused: - if is_pause: - self.pause_time = time.time() - - self.paused = is_pause - - if is_pause: - # copies tracked attributes so we know what they were before pausing - - for attr in self.pause_attr: - self.pause_buffer[attr] = self.copy_attr(attr) - - if not suppress_redraw: - self.redraw(True) - - return True - else: - return False - - def get_pause_time(self): - """ - Provides the time that we were last paused, returning -1 if we've never - been paused. - """ - - return self.pause_time - - def set_top(self, top): - """ - Changes the position where subwindows are placed within its parent. - - Arguments: - top - positioning of top within parent - """ - - if self.top != top: - self.top = top - self.win = None - - def get_height(self): - """ - Provides the height used for subwindows (-1 if it isn't limited). - """ - - return self.height - - def set_height(self, height): - """ - Changes the height used for subwindows. This uses all available space if -1. - - Arguments: - height - maximum height of panel (uses all available space if -1) - """ - - if self.height != height: - self.height = height - self.win = None - - def get_width(self): - """ - Provides the width used for subwindows (-1 if it isn't limited). - """ - - return self.width - - def set_width(self, width): - """ - Changes the width used for subwindows. This uses all available space if -1. - - Arguments: - width - maximum width of panel (uses all available space if -1) - """ - - if self.width != width: - self.width = width - self.win = None - - def get_preferred_size(self): - """ - Provides the dimensions the subwindow would use when next redrawn, given - that none of the properties of the panel or parent change before then. This - returns a tuple of (height, width). - """ - - new_height, new_width = self.parent.getmaxyx() - set_height, set_width = self.get_height(), self.get_width() - new_height = max(0, new_height - self.top) - new_width = max(0, new_width - self.left) - - if set_height != -1: - new_height = min(new_height, set_height) - - if set_width != -1: - new_width = min(new_width, set_width) - - return (new_height, new_width) - - def handle_key(self, key): - """ - Handler for user input. This returns true if the key press was consumed, - false otherwise. - - Arguments: - key - keycode for the key pressed - """ - - return False - - def get_help(self): - """ - Provides help information for the controls this page provides. This is a - list of tuples of the form... - (control, description, status) - """ - - return [] - - def draw(self, width, height): - """ - Draws display's content. This is meant to be overwritten by - implementations and not called directly (use redraw() instead). The - dimensions provided are the drawable dimensions, which in terms of width is - a column less than the actual space. - - Arguments: - width - horizontal space available for content - height - vertical space available for content - """ - - pass - - def redraw(self, force_redraw=False): - """ - Clears display and redraws its content. This can skip redrawing content if - able (ie, the subwindow's unchanged), instead just refreshing the display. - - Arguments: - force_redraw - forces the content to be cleared and redrawn if true - """ - - # skipped if not currently visible or activity has been halted - - if not self.visible or HALT_ACTIVITY: - return - - # if the panel's completely outside its parent then this is a no-op - - new_height, new_width = self.get_preferred_size() - - if new_height == 0 or new_width == 0: - self.win = None - return - - # recreates the subwindow if necessary - - is_new_window = self._reset_subwindow() - - # The reset argument is disregarded in a couple of situations: - # - The subwindow's been recreated (obviously it then doesn't have the old - # content to refresh). - # - The subwindow's dimensions have changed since last drawn (this will - # likely change the content's layout) - - subwin_max_y, subwin_max_x = self.win.getmaxyx() - - if is_new_window or subwin_max_y != self.max_y or subwin_max_x != self.max_x: - force_redraw = True - - self.max_y, self.max_x = subwin_max_y, subwin_max_x - - if not CURSES_LOCK.acquire(False): - return - - try: - if force_redraw: - self.win.erase() # clears any old contents - self.draw(self.max_x, self.max_y) - self.win.refresh() - finally: - CURSES_LOCK.release() - - def hline(self, y, x, length, *attributes): - """ - Draws a horizontal line. This should only be called from the context of a - panel's draw method. - - Arguments: - y - vertical location - x - horizontal location - length - length the line spans - attr - text attributes - """ - - format_attr = nyx.curses.curses_attr(*attributes) - - if self.win and self.max_x > x and self.max_y > y: - try: - draw_length = min(length, self.max_x - x) - self.win.hline(y, x, curses.ACS_HLINE | format_attr, draw_length) - except: - # in edge cases drawing could cause a _curses.error - pass - - def vline(self, y, x, length, *attributes): - """ - Draws a vertical line. This should only be called from the context of a - panel's draw method. - - Arguments: - y - vertical location - x - horizontal location - length - length the line spans - attr - text attributes - """ - - format_attr = nyx.curses.curses_attr(*attributes) - - if self.win and self.max_x > x and self.max_y > y: - try: - draw_length = min(length, self.max_y - y) - self.win.vline(y, x, curses.ACS_VLINE | format_attr, draw_length) - except: - # in edge cases drawing could cause a _curses.error - pass - - def addch(self, y, x, char, *attributes): - """ - Draws a single character. This should only be called from the context of a - panel's draw method. - - Arguments: - y - vertical location - x - horizontal location - char - character to be drawn - attr - text attributes - """ - - format_attr = nyx.curses.curses_attr(*attributes) - - if self.win and self.max_x > x and self.max_y > y: - try: - self.win.addch(y, x, char, format_attr) - return x + 1 - except: - # in edge cases drawing could cause a _curses.error - pass - - return x - - def addstr(self, y, x, msg, *attributes): - """ - Writes string to subwindow if able. This takes into account screen bounds - to avoid making curses upset. This should only be called from the context - of a panel's draw method. - - Arguments: - y - vertical location - x - horizontal location - msg - text to be added - attr - text attributes - """ - - format_attr = nyx.curses.curses_attr(*attributes) - - # subwindows need a single character buffer (either in the x or y - # direction) from actual content to prevent crash when shrank - - if self.win and self.max_x > x and self.max_y > y: - try: - drawn_msg = msg[:self.max_x - x] - self.win.addstr(y, x, drawn_msg, format_attr) - return x + len(drawn_msg) - except: - # this might produce a _curses.error during edge cases, for instance - # when resizing with visible popups - - pass - - return x - - def addstr_wrap(self, y, x, msg, width, min_x = 0, *attr): - orig_y = y - - while msg: - draw_msg, msg = str_tools.crop(msg, width - x, None, ending = None, get_remainder = True) - - if not draw_msg: - draw_msg, msg = str_tools.crop(msg, width - x), '' # first word is longer than the line - - x = self.addstr(y, x, draw_msg, *attr) - - if (y - orig_y + 1) >= CONFIG['features.maxLineWrap']: - break # maximum number we'll wrap - - if msg: - x, y = min_x, y + 1 - - return x, y - - def getstr(self, y, x, initial_text = ''): - """ - Provides a text field where the user can input a string, blocking until - they've done so and returning the result. If the user presses escape then - this terminates and provides back None. This should only be called from - the context of a panel's draw method. - - This blanks any content within the space that the input field is rendered - (otherwise stray characters would be interpreted as part of the initial - input). - - Arguments: - y - vertical location - x - horizontal location - initial_text - starting text in this field - """ - - # makes cursor visible - - try: - previous_cursor_state = curses.curs_set(1) - except curses.error: - previous_cursor_state = 0 - - # temporary subwindow for user input - - display_width = self.get_preferred_size()[1] - - input_subwindow = self.parent.subwin(1, display_width - x, self.top + y, self.left + x) - - # blanks the field's area, filling it with the font in case it's hilighting - - input_subwindow.clear() - input_subwindow.bkgd(' ', curses.A_NORMAL) - - # prepopulates the initial text - - if initial_text: - input_subwindow.addstr(0, 0, initial_text[:display_width - x - 1], curses.A_NORMAL) - - # Displays the text field, blocking until the user's done. This closes the - # text panel and returns user_input to the initial text if the user presses - # escape. - - textbox = curses.textpad.Textbox(input_subwindow) - - validator = BasicValidator() - - textbox.win.attron(curses.A_NORMAL) - user_input = textbox.edit(lambda key: validator.validate(key, textbox)).strip() - textbox.win.attroff(curses.A_NORMAL) - - if textbox.lastcmd == curses.ascii.BEL: - user_input = None - - # reverts visability settings - - try: - curses.curs_set(previous_cursor_state) - except curses.error: - pass - - return user_input - - def add_scroll_bar(self, top, bottom, size, draw_top = 0, draw_bottom = -1, draw_left = 0): - """ - Draws a left justified scroll bar reflecting position within a vertical - listing. This is shorted if necessary, and left undrawn if no space is - available. The bottom is squared off, having a layout like: - | - *| - *| - *| - | - -+ - - This should only be called from the context of a panel's draw method. - - Arguments: - top - list index for the top-most visible element - bottom - list index for the bottom-most visible element - size - size of the list in which the listed elements are contained - draw_top - starting row where the scroll bar should be drawn - draw_bottom - ending row where the scroll bar should end, -1 if it should - span to the bottom of the panel - draw_left - left offset at which to draw the scroll bar - """ - - if (self.max_y - draw_top) < 2: - return # not enough room - - # sets draw_bottom to be the actual row on which the scrollbar should end - - if draw_bottom == -1: - draw_bottom = self.max_y - 1 - else: - draw_bottom = min(draw_bottom, self.max_y - 1) - - # determines scrollbar dimensions - - scrollbar_height = draw_bottom - draw_top - slider_top = scrollbar_height * top / size - slider_size = scrollbar_height * (bottom - top) / size - - # ensures slider isn't at top or bottom unless really at those extreme bounds - - if top > 0: - slider_top = max(slider_top, 1) - - if bottom != size: - slider_top = min(slider_top, scrollbar_height - slider_size - 2) - - # avoids a rounding error that causes the scrollbar to be too low when at - # the bottom - - if bottom == size: - slider_top = scrollbar_height - slider_size - 1 - - # draws scrollbar slider - - for i in range(scrollbar_height): - if i >= slider_top and i <= slider_top + slider_size: - self.addstr(i + draw_top, draw_left, ' ', HIGHLIGHT) - else: - self.addstr(i + draw_top, draw_left, ' ') - - # draws box around the scroll bar - - self.vline(draw_top, draw_left + 1, draw_bottom - 1) - self.addch(draw_bottom, draw_left + 1, curses.ACS_LRCORNER) - self.addch(draw_bottom, draw_left, curses.ACS_HLINE) - - def _reset_subwindow(self): - """ - Create a new subwindow instance for the panel if: - - Panel currently doesn't have a subwindow (was uninitialized or - invalidated). - - There's room for the panel to grow vertically (curses automatically - lets subwindows regrow horizontally, but not vertically). - - The subwindow has been displaced. This is a curses display bug that - manifests if the terminal's shrank then re-expanded. Displaced - subwindows are never restored to their proper position, resulting in - graphical glitches if we draw to them. - - The preferred size is smaller than the actual size (should shrink). - - This returns True if a new subwindow instance was created, False otherwise. - """ - - new_height, new_width = self.get_preferred_size() - - if new_height == 0: - return False # subwindow would be outside its parent - - # determines if a new subwindow should be recreated - - recreate = self.win is None - - if self.win: - subwin_max_y, subwin_max_x = self.win.getmaxyx() - recreate |= subwin_max_y < new_height # check for vertical growth - recreate |= self.top > self.win.getparyx()[0] # check for displacement - recreate |= subwin_max_x > new_width or subwin_max_y > new_height # shrinking - - # I'm not sure if recreating subwindows is some sort of memory leak but the - # Python curses bindings seem to lack all of the following: - # - subwindow deletion (to tell curses to free the memory) - # - subwindow moving/resizing (to restore the displaced windows) - # so this is the only option (besides removing subwindows entirely which - # would mean far more complicated code and no more selective refreshing) - - if recreate: - self.win = self.parent.subwin(new_height, new_width, self.top, self.left) - - # note: doing this log before setting win produces an infinite loop - log.debug("recreating panel '%s' with the dimensions of %i/%i" % (self.get_name(), new_height, new_width)) - - return recreate - - def draw_box(self, top, left, width, height, *attributes): - """ - Draws a box in the panel with the given bounds. - - Arguments: - top - vertical position of the box's top - left - horizontal position of the box's left side - width - width of the drawn box - height - height of the drawn box - attr - text attributes - """ - - # draws the top and bottom - - self.hline(top, left + 1, width - 2, *attributes) - self.hline(top + height - 1, left + 1, width - 2, *attributes) - - # draws the left and right sides - - self.vline(top + 1, left, height - 2, *attributes) - self.vline(top + 1, left + width - 1, height - 2, *attributes) - - # draws the corners - - self.addch(top, left, curses.ACS_ULCORNER, *attributes) - self.addch(top, left + width - 1, curses.ACS_URCORNER, *attributes) - self.addch(top + height - 1, left, curses.ACS_LLCORNER, *attributes) - self.addch(top + height - 1, left + width - 1, curses.ACS_LRCORNER, *attributes) - - -class KeyInput(object): - """ - Keyboard input by the user. - """ - - def __init__(self, key): - self._key = key # pressed key as an integer - - def match(self, *keys): - """ - Checks if we have a case insensitive match with the given key. Beside - characters, this also recognizes: up, down, left, right, home, end, - page_up, page_down, and esc. - """ - - for key in keys: - if key in SPECIAL_KEYS: - if self._key == SPECIAL_KEYS[key]: - return True - elif len(key) == 1: - if self._key in (ord(key.lower()), ord(key.upper())): - return True - else: - raise ValueError("%s wasn't among our recognized key codes" % key) - - return False - - def is_scroll(self): - """ - True if the key is used for scrolling, false otherwise. - """ - - return self._key in SCROLL_KEYS - - def is_selection(self): - """ - True if the key matches the enter or space keys. - """ - - return self._key in (curses.KEY_ENTER, 10, ord(' ')) diff --git a/nyx/util/tracker.py b/nyx/util/tracker.py deleted file mode 100644 index 8378034..0000000 --- a/nyx/util/tracker.py +++ /dev/null @@ -1,896 +0,0 @@ -""" -Background tasks for gathering information about the tor process. - -:: - - get_connection_tracker - provides a ConnectionTracker for our tor process - get_resource_tracker - provides a ResourceTracker for our tor process - get_port_usage_tracker - provides a PortUsageTracker for our system - get_consensus_tracker - provides a ConsensusTracker for our tor process - - stop_trackers - halts any active trackers - - Daemon - common parent for resolvers - |- ConnectionTracker - periodically checks the connections established by tor - | |- get_custom_resolver - provide the custom conntion resolver we're using - | |- set_custom_resolver - overwrites automatic resolver selecion with a custom resolver - | +- get_value - provides our latest connection results - | - |- ResourceTracker - periodically checks the resource usage of tor - | +- get_value - provides our latest resource usage results - | - |- PortUsageTracker - provides information about port usage on the local system - | +- get_processes_using_ports - mapping of ports to the processes using it - | - |- run_counter - number of successful runs - |- get_rate - provides the rate at which we run - |- set_rate - sets the rate at which we run - |- set_paused - pauses or continues work - +- stop - stops further work by the daemon - - ConsensusTracker - performant lookups for consensus related information - |- update - updates the consensus information we're based on - |- get_relay_nickname - provides the nickname for a given relay - |- get_relay_fingerprints - provides relays running at a location - +- get_relay_address - provides the address a relay is running at - -.. data:: Resources - - Resource usage information retrieved about the tor process. - - :var float cpu_sample: average cpu usage since we last checked - :var float cpu_average: average cpu usage since we first started tracking the process - :var float cpu_total: total cpu time the process has used since starting - :var int memory_bytes: memory usage of the process in bytes - :var float memory_percent: percentage of our memory used by this process - :var float timestamp: unix timestamp for when this information was fetched -""" - -import collections -import os -import time -import threading - -import stem.control - -from stem.util import conf, connection, enum, proc, str_tools, system - -from nyx.util import log, tor_controller - -CONFIG = conf.config_dict('nyx', { - 'queries.connections.rate': 5, - 'queries.resources.rate': 5, - 'queries.port_usage.rate': 5, -}) - -CONNECTION_TRACKER = None -RESOURCE_TRACKER = None -PORT_USAGE_TRACKER = None -CONSENSUS_TRACKER = None - -CustomResolver = enum.Enum( - ('INFERENCE', 'by inference'), -) - -# Extending stem's Connection tuple with attributes for the uptime of the -# connection. - -Connection = collections.namedtuple('Connection', [ - 'start_time', - 'is_legacy', # boolean to indicate if the connection predated us -] + list(stem.util.connection.Connection._fields)) - -Resources = collections.namedtuple('Resources', [ - 'cpu_sample', - 'cpu_average', - 'cpu_total', - 'memory_bytes', - 'memory_percent', - 'timestamp', -]) - -Process = collections.namedtuple('Process', [ - 'pid', - 'name', -]) - - -class UnresolvedResult(Exception): - 'Indicates the application being used by a port is still being determined.' - - -class UnknownApplication(Exception): - 'No application could be determined for this port.' - - -def get_connection_tracker(): - """ - Singleton for tracking the connections established by tor. - """ - - global CONNECTION_TRACKER - - if CONNECTION_TRACKER is None: - CONNECTION_TRACKER = ConnectionTracker(CONFIG['queries.connections.rate']) - CONNECTION_TRACKER.start() - - return CONNECTION_TRACKER - - -def get_resource_tracker(): - """ - Singleton for tracking the resource usage of our tor process. - """ - - global RESOURCE_TRACKER - - if RESOURCE_TRACKER is None: - RESOURCE_TRACKER = ResourceTracker(CONFIG['queries.resources.rate']) - RESOURCE_TRACKER.start() - - return RESOURCE_TRACKER - - -def get_port_usage_tracker(): - """ - Singleton for tracking the process using a set of ports. - """ - - global PORT_USAGE_TRACKER - - if PORT_USAGE_TRACKER is None: - PORT_USAGE_TRACKER = PortUsageTracker(CONFIG['queries.port_usage.rate']) - PORT_USAGE_TRACKER.start() - - return PORT_USAGE_TRACKER - - -def get_consensus_tracker(): - """ - Singleton for tracking the connections established by tor. - """ - - global CONSENSUS_TRACKER - - if CONSENSUS_TRACKER is None: - CONSENSUS_TRACKER = ConsensusTracker() - - return CONSENSUS_TRACKER - - -def stop_trackers(): - """ - Halts active trackers, providing back the thread shutting them down. - - :returns: **threading.Thread** shutting down the daemons - """ - - def halt_trackers(): - trackers = filter(lambda t: t and t.is_alive(), [ - CONNECTION_TRACKER, - RESOURCE_TRACKER, - PORT_USAGE_TRACKER, - ]) - - for tracker in trackers: - tracker.stop() - - for tracker in trackers: - tracker.join() - - halt_thread = threading.Thread(target = halt_trackers) - halt_thread.setDaemon(True) - halt_thread.start() - return halt_thread - - -def _resources_via_ps(pid): - """ - Fetches resource usage information about a given process via ps. This returns - a tuple of the form... - - (total_cpu_time, uptime, memory_in_bytes, memory_in_percent) - - :param int pid: process to be queried - - :returns: **tuple** with the resource usage information - - :raises: **IOError** if unsuccessful - """ - - # ps results are of the form... - # - # TIME ELAPSED RSS %MEM - # 3-08:06:32 21-00:00:12 121844 23.5 - # - # ... or if Tor has only recently been started... - # - # TIME ELAPSED RSS %MEM - # 0:04.40 37:57 18772 0.9 - - ps_call = system.call('ps -p {pid} -o cputime,etime,rss,%mem'.format(pid = pid)) - - if ps_call and len(ps_call) >= 2: - stats = ps_call[1].strip().split() - - if len(stats) == 4: - try: - total_cpu_time = str_tools.parse_short_time_label(stats[0]) - uptime = str_tools.parse_short_time_label(stats[1]) - memory_bytes = int(stats[2]) * 1024 # ps size is in kb - memory_percent = float(stats[3]) / 100.0 - - return (total_cpu_time, uptime, memory_bytes, memory_percent) - except ValueError: - pass - - raise IOError('unrecognized output from ps: %s' % ps_call) - - -def _resources_via_proc(pid): - """ - Fetches resource usage information about a given process via proc. This - returns a tuple of the form... - - (total_cpu_time, uptime, memory_in_bytes, memory_in_percent) - - :param int pid: process to be queried - - :returns: **tuple** with the resource usage information - - :raises: **IOError** if unsuccessful - """ - - utime, stime, start_time = proc.stats( - pid, - proc.Stat.CPU_UTIME, - proc.Stat.CPU_STIME, - proc.Stat.START_TIME, - ) - - total_cpu_time = float(utime) + float(stime) - memory_in_bytes = proc.memory_usage(pid)[0] - total_memory = proc.physical_memory() - - uptime = time.time() - float(start_time) - memory_in_percent = float(memory_in_bytes) / total_memory - - return (total_cpu_time, uptime, memory_in_bytes, memory_in_percent) - - -def _process_for_ports(local_ports, remote_ports): - """ - Provides the name of the process using the given ports. - - :param list local_ports: local port numbers to look up - :param list remote_ports: remote port numbers to look up - - :returns: **dict** mapping the ports to the associated **Process**, or - **None** if it can't be determined - - :raises: **IOError** if unsuccessful - """ - - def _parse_lsof_line(line): - line_comp = line.split() - - if not line: - return None, None, None, None # blank line - elif len(line_comp) != 10: - raise ValueError('lines are expected to have ten fields: %s' % line) - elif line_comp[9] != '(ESTABLISHED)': - return None, None, None, None # connection isn't established - elif not line_comp[1].isdigit(): - raise ValueError('expected the pid (which is the second value) to be an integer: %s' % line) - - pid = int(line_comp[1]) - cmd = line_comp[0] - port_map = line_comp[8] - - if '->' not in port_map: - raise ValueError("'%s' is expected to be a '->' separated mapping" % port_map) - - local, remote = port_map.split('->', 1) - - if ':' not in local or ':' not in remote: - raise ValueError("'%s' is expected to be 'address:port' entries" % port_map) - - local_port = local.split(':', 1)[1] - remote_port = remote.split(':', 1)[1] - - if not connection.is_valid_port(local_port): - raise ValueError("'%s' isn't a valid port" % local_port) - elif not connection.is_valid_port(remote_port): - raise ValueError("'%s' isn't a valid port" % remote_port) - - return int(local_port), int(remote_port), pid, cmd - - # atagar@fenrir:~/Desktop/nyx$ lsof -i tcp:51849 -i tcp:37277 - # COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME - # tor 2001 atagar 14u IPv4 14048 0t0 TCP localhost:9051->localhost:37277 (ESTABLISHED) - # tor 2001 atagar 15u IPv4 22024 0t0 TCP localhost:9051->localhost:51849 (ESTABLISHED) - # python 2462 atagar 3u IPv4 14047 0t0 TCP localhost:37277->localhost:9051 (ESTABLISHED) - # python 3444 atagar 3u IPv4 22023 0t0 TCP localhost:51849->localhost:9051 (ESTABLISHED) - - try: - lsof_cmd = 'lsof -nP ' + ' '.join(['-i tcp:%s' % port for port in (local_ports + remote_ports)]) - lsof_call = system.call(lsof_cmd) - except OSError as exc: - raise IOError(exc) - - if lsof_call: - results = {} - - if lsof_call[0].startswith('COMMAND '): - lsof_call = lsof_call[1:] # strip the title line - - for line in lsof_call: - try: - local_port, remote_port, pid, cmd = _parse_lsof_line(line) - - if local_port in local_ports: - results[local_port] = Process(pid, cmd) - elif remote_port in remote_ports: - results[remote_port] = Process(pid, cmd) - except ValueError as exc: - raise IOError('unrecognized output from lsof (%s): %s' % (exc, line)) - - for unknown_port in set(local_ports).union(remote_ports).difference(results.keys()): - results[unknown_port] = None - - return results - - raise IOError('no results from lsof') - - -class Daemon(threading.Thread): - """ - Daemon that can perform a given action at a set rate. Subclasses are expected - to implement our _task() method with the work to be done. - """ - - def __init__(self, rate): - super(Daemon, self).__init__() - self.setDaemon(True) - - self._process_lock = threading.RLock() - self._process_pid = None - self._process_name = None - - self._rate = rate - self._last_ran = -1 # time when we last ran - self._run_counter = 0 # counter for the number of successful runs - - self._is_paused = False - self._pause_condition = threading.Condition() - self._halt = False # terminates thread if true - - controller = tor_controller() - controller.add_status_listener(self._tor_status_listener) - self._tor_status_listener(controller, stem.control.State.INIT, None) - - def run(self): - while not self._halt: - time_since_last_ran = time.time() - self._last_ran - - if self._is_paused or time_since_last_ran < self._rate: - sleep_duration = max(0.02, self._rate - time_since_last_ran) - - with self._pause_condition: - if not self._halt: - self._pause_condition.wait(sleep_duration) - - continue # done waiting, try again - - with self._process_lock: - if self._process_pid is not None: - is_successful = self._task(self._process_pid, self._process_name) - else: - is_successful = False - - if is_successful: - self._run_counter += 1 - - self._last_ran = time.time() - - def _task(self, process_pid, process_name): - """ - Task the resolver is meant to perform. This should be implemented by - subclasses. - - :param int process_pid: pid of the process we're tracking - :param str process_name: name of the process we're tracking - - :returns: **bool** indicating if our run was successful or not - """ - - return True - - def run_counter(self): - """ - Provides the number of times we've successful runs so far. This can be used - by callers to determine if our results have been seen by them before or - not. - - :returns: **int** for the run count we're on - """ - - return self._run_counter - - def get_rate(self): - """ - Provides the rate at which we perform our task. - - :returns: **float** for the rate in seconds at which we perform our task - """ - - return self._rate - - def set_rate(self, rate): - """ - Sets the rate at which we perform our task in seconds. - - :param float rate: rate at which to perform work in seconds - """ - - self._rate = rate - - def set_paused(self, pause): - """ - Either resumes or holds off on doing further work. - - :param bool pause: halts work if **True**, resumes otherwise - """ - - self._is_paused = pause - - def stop(self): - """ - Halts further work and terminates the thread. - """ - - with self._pause_condition: - self._halt = True - self._pause_condition.notifyAll() - - def _tor_status_listener(self, controller, event_type, _): - with self._process_lock: - if not self._halt and event_type in (stem.control.State.INIT, stem.control.State.RESET): - tor_pid = controller.get_pid(None) - tor_cmd = system.name_by_pid(tor_pid) if tor_pid else None - - self._process_pid = tor_pid - self._process_name = tor_cmd if tor_cmd else 'tor' - elif event_type == stem.control.State.CLOSED: - self._process_pid = None - self._process_name = None - - def __enter__(self): - self.start() - return self - - def __exit__(self, exit_type, value, traceback): - self.stop() - self.join() - - -class ConnectionTracker(Daemon): - """ - Periodically retrieves the connections established by tor. - """ - - def __init__(self, rate): - super(ConnectionTracker, self).__init__(rate) - - self._connections = [] - self._start_times = {} # connection => (unix_timestamp, is_legacy) - self._custom_resolver = None - self._is_first_run = True - - # Number of times in a row we've either failed with our current resolver or - # concluded that our rate is too low. - - self._failure_count = 0 - self._rate_too_low_count = 0 - - # If 'DisableDebuggerAttachment 0' is set we can do normal connection - # resolution. Otherwise connection resolution by inference is the only game - # in town. - - if tor_controller().get_conf('DisableDebuggerAttachment', None) == '0': - self._resolvers = connection.system_resolvers() - else: - self._resolvers = [CustomResolver.INFERENCE] - - log.info('tracker.available_resolvers', os = os.uname()[0], resolvers = ', '.join(self._resolvers)) - - def _task(self, process_pid, process_name): - if self._custom_resolver: - resolver = self._custom_resolver - is_default_resolver = False - elif self._resolvers: - resolver = self._resolvers[0] - is_default_resolver = True - else: - return False # nothing to resolve with - - try: - start_time = time.time() - new_connections, new_start_times = [], {} - - if resolver == CustomResolver.INFERENCE: - # provide connections going to a relay or one of our tor ports - - connections = [] - controller = tor_controller() - consensus_tracker = get_consensus_tracker() - - for conn in proc.connections(user = controller.get_user(None)): - if conn.remote_port in consensus_tracker.get_relay_fingerprints(conn.remote_address): - connections.append(conn) # outbound to another relay - elif conn.local_port in controller.get_ports(stem.control.Listener.OR, []): - connections.append(conn) # inbound to our ORPort - elif conn.local_port in controller.get_ports(stem.control.Listener.DIR, []): - connections.append(conn) # inbound to our DirPort - elif conn.local_port in controller.get_ports(stem.control.Listener.CONTROL, []): - connections.append(conn) # controller connection - else: - connections = connection.get_connections(resolver, process_pid = process_pid, process_name = process_name) - - for conn in connections: - conn_start_time, is_legacy = self._start_times.get(conn, (start_time, self._is_first_run)) - new_start_times[conn] = (conn_start_time, is_legacy) - new_connections.append(Connection(conn_start_time, is_legacy, *conn)) - - self._connections = new_connections - self._start_times = new_start_times - self._is_first_run = False - - runtime = time.time() - start_time - - if is_default_resolver: - self._failure_count = 0 - - # Reduce our rate if connection resolution is taking a long time. This is - # most often an issue for extremely busy relays. - - min_rate = 100 * runtime - - if self.get_rate() < min_rate: - self._rate_too_low_count += 1 - - if self._rate_too_low_count >= 3: - min_rate += 1 # little extra padding so we don't frequently update this - self.set_rate(min_rate) - self._rate_too_low_count = 0 - log.debug('tracker.lookup_rate_increased', seconds = "%0.1f" % min_rate) - else: - self._rate_too_low_count = 0 - - return True - except IOError as exc: - log.info('wrap', text = exc) - - # Fail over to another resolver if we've repeatedly been unable to use - # this one. - - if is_default_resolver: - self._failure_count += 1 - - if self._failure_count >= 3: - self._resolvers.pop(0) - self._failure_count = 0 - - if self._resolvers: - log.notice( - 'tracker.unable_to_use_resolver', - old_resolver = resolver, - new_resolver = self._resolvers[0], - ) - else: - log.notice('tracker.unable_to_use_all_resolvers') - - return False - - def get_custom_resolver(self): - """ - Provides the custom resolver the user has selected. This is **None** if - we're picking resolvers dynamically. - - :returns: :data:`~stem.util.connection.Resolver` we're overwritten to use - """ - - return self._custom_resolver - - def set_custom_resolver(self, resolver): - """ - Sets the resolver used for connection resolution. If **None** then this is - automatically determined based on what is available. - - :param stem.util.connection.Resolver resolver: resolver to use - """ - - self._custom_resolver = resolver - - def get_value(self): - """ - Provides a listing of tor's latest connections. - - :returns: **list** of :class:`~nyx.util.tracker.Connection` we last - retrieved, an empty list if our tracker's been stopped - """ - - if self._halt: - return [] - else: - return list(self._connections) - - -class ResourceTracker(Daemon): - """ - Periodically retrieves the resource usage of tor. - """ - - def __init__(self, rate): - super(ResourceTracker, self).__init__(rate) - - self._resources = None - self._use_proc = proc.is_available() # determines if we use proc or ps for lookups - self._failure_count = 0 # number of times in a row we've failed to get results - - def get_value(self): - """ - Provides tor's latest resource usage. - - :returns: latest :data:`~nyx.util.tracker.Resources` we've polled - """ - - result = self._resources - return result if result else Resources(0.0, 0.0, 0.0, 0, 0.0, 0.0) - - def _task(self, process_pid, process_name): - try: - resolver = _resources_via_proc if self._use_proc else _resources_via_ps - total_cpu_time, uptime, memory_in_bytes, memory_in_percent = resolver(process_pid) - - if self._resources: - cpu_sample = (total_cpu_time - self._resources.cpu_total) / self._resources.cpu_total - else: - cpu_sample = 0.0 # we need a prior datapoint to give a sampling - - self._resources = Resources( - cpu_sample = cpu_sample, - cpu_average = total_cpu_time / uptime, - cpu_total = total_cpu_time, - memory_bytes = memory_in_bytes, - memory_percent = memory_in_percent, - timestamp = time.time(), - ) - - self._failure_count = 0 - return True - except IOError as exc: - self._failure_count += 1 - - if self._use_proc: - if self._failure_count >= 3: - # We've failed three times resolving via proc. Warn, and fall back - # to ps resolutions. - - self._use_proc = False - self._failure_count = 0 - - log.info( - 'tracker.abort_getting_resources', - resolver = 'proc', - response = 'falling back to ps', - exc = exc, - ) - else: - log.debug('tracker.unable_to_get_resources', resolver = 'proc', exc = exc) - else: - if self._failure_count >= 3: - # Give up on further attempts. - - log.info( - 'tracker.abort_getting_resources', - resolver = 'ps', - response = 'giving up on getting resource usage information', - exc = exc, - ) - - self.stop() - else: - log.debug('tracker.unable_to_get_resources', resolver = 'ps', exc = exc) - - return False - - -class PortUsageTracker(Daemon): - """ - Periodically retrieves the processes using a set of ports. - """ - - def __init__(self, rate): - super(PortUsageTracker, self).__init__(rate) - - self._last_requested_local_ports = [] - self._last_requested_remote_ports = [] - self._processes_for_ports = {} - self._failure_count = 0 # number of times in a row we've failed to get results - - def fetch(self, port): - """ - Provides the process running on the given port. This retrieves the results - from our cache, so it only works if we've already issued a query() request - for it and gotten results. - - :param int port: port number to look up - - :returns: **Process** using the given port - - :raises: - * :class:`nyx.util.tracker.UnresolvedResult` if the application is still - being determined - * :class:`nyx.util.tracker.UnknownApplication` if the we tried to resolve - the application but it couldn't be determined - """ - - try: - result = self._processes_for_ports[port] - - if result is None: - raise UnknownApplication() - else: - return result - except KeyError: - raise UnresolvedResult() - - def query(self, local_ports, remote_ports): - """ - Registers a given set of ports for further lookups, and returns the last - set of 'port => process' mappings we retrieved. Note that this means that - we will not return the requested ports unless they're requested again after - a successful lookup has been performed. - - :param list local_ports: local port numbers to look up - :param list remote_ports: remote port numbers to look up - - :returns: **dict** mapping port numbers to the **Process** using it - """ - - self._last_requested_local_ports = local_ports - self._last_requested_remote_ports = remote_ports - return self._processes_for_ports - - def _task(self, process_pid, process_name): - local_ports = self._last_requested_local_ports - remote_ports = self._last_requested_remote_ports - - if not local_ports and not remote_ports: - return True - - result = {} - - # Use cached results from our last lookup if available. - - for port, process in self._processes_for_ports.items(): - if port in local_ports: - result[port] = process - local_ports.remove(port) - elif port in remote_ports: - result[port] = process - remote_ports.remove(port) - - try: - if local_ports or remote_ports: - result.update(_process_for_ports(local_ports, remote_ports)) - - self._processes_for_ports = result - self._failure_count = 0 - return True - except IOError as exc: - self._failure_count += 1 - - if self._failure_count >= 3: - log.info('tracker.abort_getting_port_usage', exc = exc) - self.stop() - else: - log.debug('tracker.unable_to_get_port_usages', exc = exc) - - return False - - -class ConsensusTracker(object): - """ - Provides performant lookups of consensus information. - """ - - def __init__(self): - self._fingerprint_cache = {} # {address => [(port, fingerprint), ..]} for relays - self._nickname_cache = {} # fingerprint => nickname lookup cache - self._address_cache = {} - - tor_controller().add_event_listener(self._new_consensus_event, stem.control.EventType.NEWCONSENSUS) - - def _new_consensus_event(self, event): - self.update(event.desc) - - def update(self, router_status_entries): - """ - Updates our cache with the given router status entries. - - :param list router_status_entries: router status entries to populate our cache with - """ - - new_fingerprint_cache = {} - new_address_cache = {} - new_nickname_cache = {} - - for desc in router_status_entries: - new_fingerprint_cache.setdefault(desc.address, []).append((desc.or_port, desc.fingerprint)) - new_address_cache[desc.fingerprint] = (desc.address, desc.or_port) - new_nickname_cache[desc.fingerprint] = desc.nickname if desc.nickname else 'Unnamed' - - self._fingerprint_cache = new_fingerprint_cache - self._address_cache = new_address_cache - self._nickname_cache = new_nickname_cache - - def get_relay_nickname(self, fingerprint): - """ - Provides the nickname associated with the given relay. - - :param str fingerprint: relay to look up - - :returns: **str** with the nickname ("Unnamed" if unset), and **None** if - no such relay exists - """ - - controller = tor_controller() - - if not fingerprint: - return None - elif fingerprint == controller.get_info('fingerprint', None): - return controller.get_conf('Nickname', 'Unnamed') - else: - return self._nickname_cache.get(fingerprint) - - def get_relay_fingerprints(self, address): - """ - Provides the relays running at a given location. - - :param str address: address to be checked - - :returns: **dict** of ORPorts to their fingerprint - """ - - controller = tor_controller() - - if address == controller.get_info('address', None): - fingerprint = controller.get_info('fingerprint', None) - ports = controller.get_ports(stem.control.Listener.OR, None) - - if fingerprint and ports: - return dict([(port, fingerprint) for port in ports]) - - return dict([(port, fp) for (port, fp) in self._fingerprint_cache.get(address, [])]) - - def get_relay_address(self, fingerprint, default): - """ - Provides the (address, port) tuple where a relay is running. - - :param str fingerprint: fingerprint to be checked - - :returns: **tuple** with a **str** address and **int** port - """ - - controller = tor_controller() - - if fingerprint == controller.get_info('fingerprint', None): - my_address = controller.get_info('address', None) - my_or_ports = controller.get_ports(stem.control.Listener.OR, []) - - if my_address and len(my_or_ports) == 1: - return (my_address, my_or_ports[0]) - - return self._address_cache.get(fingerprint, default) diff --git a/run_tests.py b/run_tests.py index 15dd45b..ae587bf 100755 --- a/run_tests.py +++ b/run_tests.py @@ -13,7 +13,7 @@ import unittest import stem.util.conf import stem.util.test_tools -import nyx.util +import nyx NYX_BASE = os.path.dirname(__file__) @@ -25,9 +25,9 @@ SRC_PATHS = [os.path.join(NYX_BASE, path) for path in ( )] -(a)nyx.util.uses_settings +(a)nyx.uses_settings def main(): - nyx.util.TESTING = True + nyx.TESTING = True test_config = stem.util.conf.get_config('test') test_config.load(os.path.join(NYX_BASE, 'test', 'settings.cfg')) diff --git a/setup.py b/setup.py index 326cc61..b446c67 100644 --- a/setup.py +++ b/setup.py @@ -99,7 +99,7 @@ setup( author = nyx.__author__, author_email = nyx.__contact__, url = nyx.__url__, - packages = ['nyx', 'nyx.panel', 'nyx.menu', 'nyx.util'], + packages = ['nyx', 'nyx.panel', 'nyx.menu'], keywords = 'tor onion controller', install_requires = ['stem>=1.4.1'], package_data = {'nyx': ['settings/*', 'resources/*']}, diff --git a/test/__init__.py b/test/__init__.py index 5c60061..99ed76f 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -4,4 +4,8 @@ Unit tests for nyx. __all__ = [ 'arguments', + 'expand_path', + 'installation', + 'log', + 'tracker', ] diff --git a/test/expand_path.py b/test/expand_path.py new file mode 100644 index 0000000..559712a --- /dev/null +++ b/test/expand_path.py @@ -0,0 +1,21 @@ +import unittest + +from nyx import expand_path, uses_settings + +from mock import patch, Mock + + +class TestExpandPath(unittest.TestCase): + @patch('nyx.tor_controller') + @patch('stem.util.system.cwd', Mock(return_value = '/your_cwd')) + @uses_settings + def test_expand_path(self, tor_controller_mock, config): + tor_controller_mock().get_pid.return_value = 12345 + self.assertEqual('/absolute/path/to/torrc', expand_path('/absolute/path/to/torrc')) + self.assertEqual('/your_cwd/torrc', expand_path('torrc')) + + config.set('tor.chroot', '/chroot') + self.assertEqual('/chroot/absolute/path/to/torrc', expand_path('/absolute/path/to/torrc')) + self.assertEqual('/chroot/your_cwd/torrc', expand_path('torrc')) + + config.set('tor.chroot', None) diff --git a/test/log/__init__.py b/test/log/__init__.py new file mode 100644 index 0000000..9896955 --- /dev/null +++ b/test/log/__init__.py @@ -0,0 +1,8 @@ +""" +Unit tests for nyx's log utilities. +""" + +__all__ = [ + 'deduplication', + 'read_tor_log', +] diff --git a/test/log/condense_runlevels.py b/test/log/condense_runlevels.py new file mode 100644 index 0000000..f7c831c --- /dev/null +++ b/test/log/condense_runlevels.py @@ -0,0 +1,13 @@ +import unittest + +from nyx.log import condense_runlevels + + +class TestCondenseRunlevels(unittest.TestCase): + def test_condense_runlevels(self): + self.assertEqual([], condense_runlevels()) + self.assertEqual(['BW'], condense_runlevels('BW')) + self.assertEqual(['DEBUG', 'NOTICE', 'ERR'], condense_runlevels('DEBUG', 'NOTICE', 'ERR')) + self.assertEqual(['DEBUG-NOTICE', 'NYX DEBUG-INFO'], condense_runlevels('DEBUG', 'NYX_DEBUG', 'INFO', 'NYX_INFO', 'NOTICE')) + self.assertEqual(['TOR/NYX NOTICE-ERR'], condense_runlevels('NOTICE', 'WARN', 'ERR', 'NYX_NOTICE', 'NYX_WARN', 'NYX_ERR')) + self.assertEqual(['DEBUG', 'TOR/NYX NOTICE-ERR', 'BW'], condense_runlevels('DEBUG', 'NOTICE', 'WARN', 'ERR', 'NYX_NOTICE', 'NYX_WARN', 'NYX_ERR', 'BW')) diff --git a/test/log/data/daybreak_deduplication b/test/log/data/daybreak_deduplication new file mode 100644 index 0000000..1849ad3 --- /dev/null +++ b/test/log/data/daybreak_deduplication @@ -0,0 +1,37 @@ +Apr 24 19:50:42.000 [notice] Heartbeat: Tor's uptime is 12 days 0:00 hours, with 0 circuits open. I've sent 4.94 MB and received 130.17 MB. +Apr 24 19:50:42.000 [notice] Average packaged cell fullness: 65.101%. TLS write overhead: 11% +Apr 25 01:50:42.000 [notice] Heartbeat: Tor's uptime is 12 days 6:00 hours, with 0 circuits open. I've sent 5.00 MB and received 131.87 MB. +Apr 25 01:50:42.000 [notice] Average packaged cell fullness: 64.927%. TLS write overhead: 11% +Apr 25 07:50:42.000 [notice] Heartbeat: Tor's uptime is 12 days 12:00 hours, with 0 circuits open. I've sent 5.08 MB and received 134.19 MB. +Apr 25 07:50:42.000 [notice] Average packaged cell fullness: 64.587%. TLS write overhead: 11% +Apr 25 11:44:21.000 [notice] New control connection opened from 127.0.0.1. +Apr 25 11:44:33.000 [notice] Interrupt: exiting cleanly. +Apr 25 11:44:36.000 [notice] Tor 0.2.7.0-alpha-dev (git-63a90f2df4dcd7ff) opening log file. +Apr 25 11:44:36.492 [notice] Tor v0.2.7.0-alpha-dev (git-63a90f2df4dcd7ff) running on Linux with Libevent 2.0.16-stable, OpenSSL 1.0.1 and Zlib 1.2.3.4. +Apr 25 11:44:36.492 [notice] Tor can't help you if you use it wrong! Learn how to be safe at https://www.torproject.org/download/download#warning +Apr 25 11:44:36.492 [notice] This version is not a stable Tor release. Expect more bugs than usual. +Apr 25 11:44:36.525 [notice] Read configuration file "/home/atagar/.tor/torrc". +Apr 25 11:44:36.530 [notice] Opening Socks listener on 127.0.0.1:9050 +Apr 25 11:44:36.530 [notice] Opening Control listener on 127.0.0.1:9051 +Apr 25 11:44:36.000 [notice] Bootstrapped 0%: Starting +Apr 25 11:44:39.000 [notice] Bootstrapped 45%: Asking for relay descriptors +Apr 25 11:44:40.000 [notice] Bootstrapped 50%: Loading relay descriptors +Apr 25 11:44:47.000 [notice] Bootstrapped 55%: Loading relay descriptors +Apr 25 11:44:47.000 [notice] Bootstrapped 62%: Loading relay descriptors +Apr 25 11:44:48.000 [notice] Bootstrapped 72%: Loading relay descriptors +Apr 25 11:44:48.000 [notice] Bootstrapped 80%: Connecting to the Tor network +Apr 25 11:44:48.000 [notice] Bootstrapped 90%: Establishing a Tor circuit +Apr 25 11:44:49.000 [notice] Tor has successfully opened a circuit. Looks like client functionality is working. +Apr 25 11:44:49.000 [notice] Bootstrapped 100%: Done +Apr 25 11:45:03.000 [notice] New control connection opened from 127.0.0.1. +Apr 25 17:44:40.000 [notice] Heartbeat: Tor's uptime is 6:00 hours, with 0 circuits open. I've sent 539 kB and received 4.25 MB. +Apr 25 18:11:56.000 [notice] New control connection opened from 127.0.0.1. +Apr 25 18:52:47.000 [notice] New control connection opened from 127.0.0.1. +Apr 25 19:02:44.000 [notice] New control connection opened from 127.0.0.1. +Apr 25 23:44:40.000 [notice] Heartbeat: Tor's uptime is 12:00 hours, with 1 circuits open. I've sent 794 kB and received 7.32 MB. +Apr 26 05:44:40.000 [notice] Heartbeat: Tor's uptime is 18:00 hours, with 0 circuits open. I've sent 862 kB and received 9.05 MB. +Apr 26 10:16:38.000 [notice] New control connection opened from 127.0.0.1. +Apr 26 10:19:24.000 [notice] New control connection opened from 127.0.0.1. +Apr 26 10:21:31.000 [notice] New control connection opened from 127.0.0.1. +Apr 26 10:24:27.000 [notice] New control connection opened from 127.0.0.1. +Apr 26 10:24:46.000 [notice] New control connection opened from 127.0.0.1. diff --git a/test/log/data/empty_file b/test/log/data/empty_file new file mode 100644 index 0000000..e69de29 diff --git a/test/log/data/malformed_date b/test/log/data/malformed_date new file mode 100644 index 0000000..712149d --- /dev/null +++ b/test/log/data/malformed_date @@ -0,0 +1,21 @@ +Apr 06 11:03:39.000 [notice] Tor 0.2.7.0-alpha-dev (git-4247ce99e5d9b7b2) opening new log file. +Apr 06 11:03:39.832 [notice] Tor v0.2.7.0-alpha-dev (git-4247ce99e5d9b7b2) running on Linux with Libevent 2.0.16-stable, OpenSSL 1.0.1 and Zlib 1.2.3.4. +Apr 06 11:03:39.832 [notice] Tor can't help you if you use it wrong! Learn how to be safe at https://www.torproject.org/download/download#warning +Apr 06 11:03:39.833 [notice] This version is not a stable Tor release. Expect more bugs than usual. +Apr 06 11:03:39.833 [notice] Read configuration file "/home/atagar/.tor/torrc". +Apr 06 11:03:39.838 [notice] Opening Socks listener on 127.0.0.1:9050 +Apr 06 11:03:39.838 [notice] Opening Control listener on 127.0.0.1:9051 +Apr 06 11:03:39.000 [notice] Bootstrapped 0%: Starting +Apr 06 11:03:42.000 [notice] Bootstrapped 45%: Asking for relay descriptors +Apr 06 11:03:43.000 [notice] Bootstrapped 50%: Loading relay descriptors +Apr 06 11:03:49.000 [notice] New control connection opened from 127.0.0.1. +Apr 06 11:03:51.000 [notice] Bootstrapped 55%: Loading relay descriptors +Apr 06 11:03:51.000 [notice] Bootstrapped 63%: Loading relay descriptors +Apr 06 11:03:51.000 [notice] Bootstrapped 69%: Loading relay descriptors +Apr 06 11:03:51.000 [notice] Bootstrapped 74%: Loading relay descriptors +Apr 06 11:03:52.000 [notice] Bootstrapped 80%: Connecting to the Tor network +Zed 06 11:03:52.000 [notice] Bootstrapped 90%: Establishing a Tor circuit +Apr 06 11:03:53.000 [notice] Tor has successfully opened a circuit. Looks like client functionality is working. +Apr 06 11:03:53.000 [notice] Bootstrapped 100%: Done +Apr 06 11:03:55.000 [notice] New control connection opened from 127.0.0.1. +Apr 06 11:53:46.000 [notice] Interrupt: exiting cleanly. diff --git a/test/log/data/malformed_line b/test/log/data/malformed_line new file mode 100644 index 0000000..ac9a367 --- /dev/null +++ b/test/log/data/malformed_line @@ -0,0 +1,21 @@ +Apr 06 11:03:39.000 [notice] Tor 0.2.7.0-alpha-dev (git-4247ce99e5d9b7b2) opening new log file. +Apr 06 11:03:39.832 [notice] Tor v0.2.7.0-alpha-dev (git-4247ce99e5d9b7b2) running on Linux with Libevent 2.0.16-stable, OpenSSL 1.0.1 and Zlib 1.2.3.4. +Apr 06 11:03:39.832 [notice] Tor can't help you if you use it wrong! Learn how to be safe at https://www.torproject.org/download/download#warning +Apr 06 11:03:39.833 [notice] This version is not a stable Tor release. Expect more bugs than usual. +Apr 06 11:03:39.833 [notice] Read configuration file "/home/atagar/.tor/torrc". +Apr 06 11:03:39.838 [notice] Opening Socks listener on 127.0.0.1:9050 +Apr 06 11:03:39.838 [notice] Opening Control listener on 127.0.0.1:9051 +Apr 06 11:03:39.000 [notice] Bootstrapped 0%: Starting +Apr 06 11:03:42.000 [notice] Bootstrapped 45%: Asking for relay descriptors +Apr 06 11:03:43.000 [notice] Bootstrapped 50%: Loading relay descriptors +Apr 06 11:03:49.000 [notice] New control connection opened from 127.0.0.1. +Apr 06 11:03:51.000 [notice] Bootstrapped 55%: Loading relay descriptors +Apr 06 11:03:51.000 [notice] Bootstrapped 63%: Loading relay descriptors +Apr 06 11:03:51.000 [notice] Bootstrapped 69%: Loading relay descriptors +Apr 06 11:03:51.000 [notice] Bootstrapped 74%: Loading relay descriptors +Apr 06 11:03:52.000 [notice] Bootstrapped 80%: Connecting to the Tor network +Apr 06 11:03:52.000 [notice] Bootstrapped 90%: Establishing a Tor circuit +Apr 06 11:03:53.000 +Apr 06 11:03:53.000 [notice] Bootstrapped 100%: Done +Apr 06 11:03:55.000 [notice] New control connection opened from 127.0.0.1. +Apr 06 11:53:46.000 [notice] Interrupt: exiting cleanly. diff --git a/test/log/data/malformed_runlevel b/test/log/data/malformed_runlevel new file mode 100644 index 0000000..dd8810b --- /dev/null +++ b/test/log/data/malformed_runlevel @@ -0,0 +1,21 @@ +Apr 06 11:03:39.000 [notice] Tor 0.2.7.0-alpha-dev (git-4247ce99e5d9b7b2) opening new log file. +Apr 06 11:03:39.832 [notice] Tor v0.2.7.0-alpha-dev (git-4247ce99e5d9b7b2) running on Linux with Libevent 2.0.16-stable, OpenSSL 1.0.1 and Zlib 1.2.3.4. +Apr 06 11:03:39.832 [notice] Tor can't help you if you use it wrong! Learn how to be safe at https://www.torproject.org/download/download#warning +Apr 06 11:03:39.833 [notice] This version is not a stable Tor release. Expect more bugs than usual. +Apr 06 11:03:39.833 [notice] Read configuration file "/home/atagar/.tor/torrc". +Apr 06 11:03:39.838 [notice] Opening Socks listener on 127.0.0.1:9050 +Apr 06 11:03:39.838 [notice] Opening Control listener on 127.0.0.1:9051 +Apr 06 11:03:39.000 [notice] Bootstrapped 0%: Starting +Apr 06 11:03:42.000 [notice] Bootstrapped 45%: Asking for relay descriptors +Apr 06 11:03:43.000 [notice] Bootstrapped 50%: Loading relay descriptors +Apr 06 11:03:49.000 [notice] New control connection opened from 127.0.0.1. +Apr 06 11:03:51.000 [notice] Bootstrapped 55%: Loading relay descriptors +Apr 06 11:03:51.000 [notice] Bootstrapped 63%: Loading relay descriptors +Apr 06 11:03:51.000 [notice] Bootstrapped 69%: Loading relay descriptors +Apr 06 11:03:51.000 [notice] Bootstrapped 74%: Loading relay descriptors +Apr 06 11:03:52.000 [notice] Bootstrapped 80%: Connecting to the Tor network +Apr 06 11:03:52.000 [unrecognized] Bootstrapped 90%: Establishing a Tor circuit +Apr 06 11:03:53.000 [notice] Tor has successfully opened a circuit. Looks like client functionality is working. +Apr 06 11:03:53.000 [notice] Bootstrapped 100%: Done +Apr 06 11:03:55.000 [notice] New control connection opened from 127.0.0.1. +Apr 06 11:53:46.000 [notice] Interrupt: exiting cleanly. diff --git a/test/log/data/multiple_tor_instances b/test/log/data/multiple_tor_instances new file mode 100644 index 0000000..1793ce5 --- /dev/null +++ b/test/log/data/multiple_tor_instances @@ -0,0 +1,33 @@ +Apr 06 11:03:39.000 [notice] Tor 0.2.7.0-alpha-dev (git-4247ce99e5d9b7b2) opening new log file. +Apr 06 11:03:39.832 [notice] Tor v0.2.7.0-alpha-dev (git-4247ce99e5d9b7b2) running on Linux with Libevent 2.0.16-stable, OpenSSL 1.0.1 and Zlib 1.2.3.4. +Apr 06 11:03:39.832 [notice] Tor can't help you if you use it wrong! Learn how to be safe at https://www.torproject.org/download/download#warning +Apr 06 11:03:39.833 [notice] This version is not a stable Tor release. Expect more bugs than usual. +Apr 06 11:03:39.833 [notice] Read configuration file "/home/atagar/.tor/torrc". +Apr 06 11:03:39.838 [notice] Opening Socks listener on 127.0.0.1:9050 +Apr 06 11:03:39.838 [notice] Opening Control listener on 127.0.0.1:9051 +Apr 06 11:03:39.000 [notice] Bootstrapped 0%: Starting +Apr 06 11:03:42.000 [notice] Bootstrapped 45%: Asking for relay descriptors +Apr 06 11:03:43.000 [notice] Bootstrapped 50%: Loading relay descriptors +Apr 06 11:03:49.000 [notice] New control connection opened from 127.0.0.1. +Apr 06 11:03:51.000 [notice] Bootstrapped 55%: Loading relay descriptors +Apr 06 11:03:51.000 [notice] Bootstrapped 63%: Loading relay descriptors +Apr 06 11:03:51.000 [notice] Bootstrapped 69%: Loading relay descriptors +Apr 06 11:03:51.000 [notice] Bootstrapped 74%: Loading relay descriptors +Apr 06 11:03:52.000 [notice] Bootstrapped 80%: Connecting to the Tor network +Apr 06 11:03:52.000 [notice] Bootstrapped 90%: Establishing a Tor circuit +Apr 06 11:03:53.000 [notice] Tor has successfully opened a circuit. Looks like client functionality is working. +Apr 06 11:03:53.000 [notice] Bootstrapped 100%: Done +Apr 06 11:03:55.000 [notice] New control connection opened from 127.0.0.1. +Apr 06 11:53:46.000 [notice] Interrupt: exiting cleanly. +Apr 06 11:53:54.000 [notice] Tor 0.2.7.0-alpha-dev (git-4247ce99e5d9b7b2) opening log file. +Apr 06 11:53:54.392 [notice] Tor v0.2.7.0-alpha-dev (git-4247ce99e5d9b7b2) running on Linux with Libevent 2.0.16-stable, OpenSSL 1.0.1 and Zlib 1.2.3.4. +Apr 06 11:53:54.392 [notice] Tor can't help you if you use it wrong! Learn how to be safe at https://www.torproject.org/download/download#warning +Apr 06 11:53:54.392 [notice] This version is not a stable Tor release. Expect more bugs than usual. +Apr 06 11:53:54.392 [notice] Read configuration file "/home/atagar/.tor/torrc". +Apr 06 11:53:54.396 [notice] Opening Socks listener on 127.0.0.1:9050 +Apr 06 11:53:54.396 [notice] Opening Control listener on 127.0.0.1:9051 +Apr 06 11:53:54.000 [warn] Your log may contain sensitive information - you're logging more than "notice". Don't log unless it serves an important reason. Overwrite the log afterwards. +Apr 06 11:53:54.000 [debug] tor_disable_debugger_attach(): Attemping to disable debugger attachment to Tor for unprivileged users. +Apr 06 11:53:54.000 [debug] tor_disable_debugger_attach(): Debugger attachment disabled for unprivileged users. +Apr 06 11:53:54.000 [info] tor_lockfile_lock(): Locking "/home/atagar/.tor/lock" +Apr 06 11:53:54.000 [debug] parse_dir_authority_line(): Trusted 100 dirserver at 128.31.0.39:9131 (9695) diff --git a/test/log/data/tor_log b/test/log/data/tor_log new file mode 100644 index 0000000..0a4464e --- /dev/null +++ b/test/log/data/tor_log @@ -0,0 +1,21 @@ +Apr 06 11:03:39.000 [notice] Tor 0.2.7.0-alpha-dev (git-4247ce99e5d9b7b2) opening new log file. +Apr 06 11:03:39.832 [notice] Tor v0.2.7.0-alpha-dev (git-4247ce99e5d9b7b2) running on Linux with Libevent 2.0.16-stable, OpenSSL 1.0.1 and Zlib 1.2.3.4. +Apr 06 11:03:39.832 [notice] Tor can't help you if you use it wrong! Learn how to be safe at https://www.torproject.org/download/download#warning +Apr 06 11:03:39.833 [notice] This version is not a stable Tor release. Expect more bugs than usual. +Apr 06 11:03:39.833 [notice] Read configuration file "/home/atagar/.tor/torrc". +Apr 06 11:03:39.838 [notice] Opening Socks listener on 127.0.0.1:9050 +Apr 06 11:03:39.838 [notice] Opening Control listener on 127.0.0.1:9051 +Apr 06 11:03:39.000 [notice] Bootstrapped 0%: Starting +Apr 06 11:03:42.000 [notice] Bootstrapped 45%: Asking for relay descriptors +Apr 06 11:03:43.000 [notice] Bootstrapped 50%: Loading relay descriptors +Apr 06 11:03:49.000 [notice] New control connection opened from 127.0.0.1. +Apr 06 11:03:51.000 [notice] Bootstrapped 55%: Loading relay descriptors +Apr 06 11:03:51.000 [notice] Bootstrapped 63%: Loading relay descriptors +Apr 06 11:03:51.000 [notice] Bootstrapped 69%: Loading relay descriptors +Apr 06 11:03:51.000 [notice] Bootstrapped 74%: Loading relay descriptors +Apr 06 11:03:52.000 [notice] Bootstrapped 80%: Connecting to the Tor network +Apr 06 11:03:52.000 [notice] Bootstrapped 90%: Establishing a Tor circuit +Apr 06 11:03:53.000 [notice] Tor has successfully opened a circuit. Looks like client functionality is working. +Apr 06 11:03:53.000 [notice] Bootstrapped 100%: Done +Apr 06 11:03:55.000 [notice] New control connection opened from 127.0.0.1. +Apr 06 11:53:46.000 [notice] Interrupt: exiting cleanly. diff --git a/test/log/log_entry.py b/test/log/log_entry.py new file mode 100644 index 0000000..ffac7d3 --- /dev/null +++ b/test/log/log_entry.py @@ -0,0 +1,27 @@ +import unittest + +from nyx.log import LogEntry + + +class TestLogEntry(unittest.TestCase): + def test_deduplication_matches_identical_messages(self): + # Simple case is that we match the same message but different timestamp. + + entry = LogEntry(1333738434, 'INFO', 'tor_lockfile_lock(): Locking "/home/atagar/.tor/lock"') + self.assertTrue(entry.is_duplicate_of(LogEntry(1333738457, 'INFO', 'tor_lockfile_lock(): Locking "/home/atagar/.tor/lock"'))) + + # ... but we shouldn't match if the runlevel differs. + + self.assertFalse(entry.is_duplicate_of(LogEntry(1333738457, 'DEBUG', 'tor_lockfile_lock(): Locking "/home/atagar/.tor/lock"'))) + + def test_deduplication_matches_based_on_prefix(self): + # matches using a prefix specified in dedup.cfg + + entry = LogEntry(1333738434, 'NYX_DEBUG', 'GETCONF MyFamily (runtime: 0.0007)') + self.assertTrue(entry.is_duplicate_of(LogEntry(1333738457, 'NYX_DEBUG', 'GETCONF MyFamily (runtime: 0.0015)'))) + + def test_deduplication_matches_with_wildcard(self): + # matches using a wildcard specified in dedup.cfg + + entry = LogEntry(1333738434, 'NOTICE', 'Bootstrapped 72%: Loading relay descriptors.') + self.assertTrue(entry.is_duplicate_of(LogEntry(1333738457, 'NOTICE', 'Bootstrapped 55%: Loading relay descriptors.'))) diff --git a/test/log/log_group.py b/test/log/log_group.py new file mode 100644 index 0000000..9a10d13 --- /dev/null +++ b/test/log/log_group.py @@ -0,0 +1,146 @@ +import os +import unittest + +from nyx.log import LogGroup, LogEntry, read_tor_log + + +class TestLogGroup(unittest.TestCase): + def test_maintains_certain_size(self): + group = LogGroup(5) + self.assertEqual(0, len(group)) + + group.add(LogEntry(1333738410, 'INFO', 'tor_lockfile_lock(): Locking "/home/atagar/.tor/lock"')) + self.assertEqual([LogEntry(1333738410, 'INFO', 'tor_lockfile_lock(): Locking "/home/atagar/.tor/lock"')], list(group)) + self.assertEqual(1, len(group)) + + group.add(LogEntry(1333738420, 'NYX_DEBUG', 'GETCONF MyFamily (runtime: 0.0007)')) + group.add(LogEntry(1333738430, 'NOTICE', 'Bootstrapped 72%: Loading relay descriptors.')) + group.add(LogEntry(1333738440, 'NOTICE', 'Bootstrapped 75%: Loading relay descriptors.')) + group.add(LogEntry(1333738450, 'NOTICE', 'Bootstrapped 78%: Loading relay descriptors.')) + self.assertEqual(5, len(group)) + + # group should now be full, adding more entries pops others off + + group.add(LogEntry(1333738460, 'NOTICE', 'Bootstrapped 80%: Loading relay descriptors.')) + self.assertFalse(LogEntry(1333738410, 'INFO', 'tor_lockfile_lock(): Locking "/home/atagar/.tor/lock"') in list(group)) + self.assertEqual(5, len(group)) + + # try adding a bunch that will be deduplicated, and make sure we still maintain the size + + group.add(LogEntry(1333738510, 'NOTICE', 'Bootstrapped 80%: Loading relay descriptors.')) + group.add(LogEntry(1333738520, 'NOTICE', 'Bootstrapped 80%: Loading relay descriptors.')) + group.add(LogEntry(1333738530, 'NOTICE', 'Bootstrapped 80%: Loading relay descriptors.')) + group.add(LogEntry(1333738540, 'NOTICE', 'Bootstrapped 80%: Loading relay descriptors.')) + group.add(LogEntry(1333738550, 'NOTICE', 'Bootstrapped 80%: Loading relay descriptors.')) + group.add(LogEntry(1333738560, 'NOTICE', 'Bootstrapped 80%: Loading relay descriptors.')) + group.add(LogEntry(1333738570, 'NOTICE', 'Bootstrapped 80%: Loading relay descriptors.')) + self.assertEqual([1333738570, 1333738560, 1333738550, 1333738540, 1333738530], [e.timestamp for e in group]) + self.assertEqual(5, len(group)) + + def test_deduplication(self): + group = LogGroup(5) + group.add(LogEntry(1333738410, 'NOTICE', 'Bootstrapped 72%: Loading relay descriptors.')) + group.add(LogEntry(1333738420, 'NOTICE', 'Bootstrapped 75%: Loading relay descriptors.')) + group.add(LogEntry(1333738430, 'NYX_DEBUG', 'GETCONF MyFamily (runtime: 0.0007)')) + group.add(LogEntry(1333738440, 'NOTICE', 'Bootstrapped 78%: Loading relay descriptors.')) + group.add(LogEntry(1333738450, 'NOTICE', 'Bootstrapped 80%: Loading relay descriptors.')) + self.assertEqual([1333738450, 1333738440, 1333738430, 1333738420, 1333738410], [e.timestamp for e in group]) + + bootstrap_messages = [ + 'Bootstrapped 80%: Loading relay descriptors.', + 'Bootstrapped 78%: Loading relay descriptors.', + 'Bootstrapped 75%: Loading relay descriptors.', + 'Bootstrapped 72%: Loading relay descriptors.', + ] + + group_items = list(group) + self.assertEqual(bootstrap_messages, [e.message for e in group_items[0].duplicates]) + self.assertEqual([False, True, False, True, True], [e.is_duplicate for e in group_items]) + + # add another duplicate message that pops the last + + group.add(LogEntry(1333738460, 'NOTICE', 'Bootstrapped 90%: Loading relay descriptors.')) + + bootstrap_messages = [ + 'Bootstrapped 90%: Loading relay descriptors.', + 'Bootstrapped 80%: Loading relay descriptors.', + 'Bootstrapped 78%: Loading relay descriptors.', + 'Bootstrapped 75%: Loading relay descriptors.', + ] + + group_items = list(group) + self.assertEqual(bootstrap_messages, [e.message for e in group_items[0].duplicates]) + self.assertEqual([False, True, True, False, True], [e.is_duplicate for e in group_items]) + + # add another non-duplicate message that pops the last + + group.add(LogEntry(1333738470, 'INFO', 'tor_lockfile_lock(): Locking "/home/atagar/.tor/lock"')) + + bootstrap_messages = [ + 'Bootstrapped 90%: Loading relay descriptors.', + 'Bootstrapped 80%: Loading relay descriptors.', + 'Bootstrapped 78%: Loading relay descriptors.', + ] + + group_items = list(group) + self.assertEqual(None, group_items[0].duplicates) + self.assertEqual(bootstrap_messages, [e.message for e in group_items[1].duplicates]) + self.assertEqual([False, False, True, True, False], [e.is_duplicate for e in group_items]) + + def test_deduplication_with_daybreaks(self): + group = LogGroup(100, group_by_day = True) + test_log_path = os.path.join(os.path.dirname(__file__), 'data', 'daybreak_deduplication') + + for entry in reversed(list(read_tor_log(test_log_path))): + group.add(entry) + + # Entries should consist of two days of results... + # + # Day 1: + # 10:24:27 [NOTICE] New control connection opened from 127.0.0.1. + # 10:21:31 [NOTICE] New control connection opened from 127.0.0.1. + # 10:19:24 [NOTICE] New control connection opened from 127.0.0.1. + # 10:16:38 [NOTICE] New control connection opened from 127.0.0.1. + # 10:16:38 [NOTICE] New control connection opened from 127.0.0.1. + # 05:44:40 [NOTICE] Heartbeat: Tor's uptime is 18:00 hours, with 0 circuits open. I've sent 862 kB and received 9.05 MB. + # + # Day 2: + # 23:44:40 [NOTICE] Heartbeat: Tor's uptime is 12:00 hours, with 1 circuits open. I've sent 794 kB and received 7.32 MB. + # 19:02:44 [NOTICE] New control connection opened from 127.0.0.1. + # 18:52:47 [NOTICE] New control connection opened from 127.0.0.1. + # 18:11:56 [NOTICE] New control connection opened from 127.0.0.1. + # 17:44:40 [NOTICE] Heartbeat: Tor's uptime is 6:00 hours, with 0 circuits open. I've sent 539 kB and received 4.25 MB. + # 11:45:03 [NOTICE] New control connection opened from 127.0.0.1. + # 11:44:49 [NOTICE] Bootstrapped 100%: Done + # ... etc... + + group_items = list(group) + + # First day + + self.assertEqual('New control connection opened from 127.0.0.1.', group_items[0].message) + self.assertEqual(5, len(group_items[0].duplicates)) + self.assertFalse(group_items[0].is_duplicate) + + for entry in group_items[1:5]: + self.assertEqual('New control connection opened from 127.0.0.1.', entry.message) + self.assertEqual(5, len(entry.duplicates)) + self.assertTrue(entry.is_duplicate) + + self.assertEqual("Heartbeat: Tor's uptime is 18:00 hours, with 0 circuits open. I've sent 862 kB and received 9.05 MB.", group_items[5].message) + self.assertEqual(None, group_items[5].duplicates) + self.assertFalse(group_items[5].is_duplicate) + + # Second day + + self.assertEqual("Heartbeat: Tor's uptime is 12:00 hours, with 1 circuits open. I've sent 794 kB and received 7.32 MB.", group_items[6].message) + self.assertEqual(2, len(group_items[6].duplicates)) + self.assertFalse(group_items[6].is_duplicate) + + self.assertEqual('New control connection opened from 127.0.0.1.', group_items[8].message) + self.assertEqual(4, len(group_items[8].duplicates)) + self.assertTrue(group_items[8].is_duplicate) + + self.assertEqual("Heartbeat: Tor's uptime is 6:00 hours, with 0 circuits open. I've sent 539 kB and received 4.25 MB.", group_items[10].message) + self.assertEqual(2, len(group_items[10].duplicates)) + self.assertTrue(group_items[10].is_duplicate) diff --git a/test/log/read_tor_log.py b/test/log/read_tor_log.py new file mode 100644 index 0000000..5fa8369 --- /dev/null +++ b/test/log/read_tor_log.py @@ -0,0 +1,59 @@ +import os +import unittest + +from nyx.log import read_tor_log + + +def data_path(filename): + return os.path.join(os.path.dirname(__file__), 'data', filename) + + +class TestReadTorLog(unittest.TestCase): + def test_general_log(self): + entries = list(read_tor_log(data_path('tor_log'))) + self.assertEqual(21, len(entries)) + + self.assertEqual('Interrupt: exiting cleanly.', entries[0].message) + self.assertEqual('Tor 0.2.7.0-alpha-dev (git-4247ce99e5d9b7b2) opening new log file.', entries[-1].message) + + def test_with_multiple_tor_instances(self): + entries = list(read_tor_log(data_path('multiple_tor_instances'))) + self.assertEqual(12, len(entries)) + + self.assertEqual('parse_dir_authority_line(): Trusted 100 dirserver at 128.31.0.39:9131 (9695)', entries[0].message) + self.assertEqual('tor_lockfile_lock(): Locking "/home/atagar/.tor/lock"', entries[1].message) + self.assertEqual('Tor 0.2.7.0-alpha-dev (git-4247ce99e5d9b7b2) opening log file.', entries[-1].message) + + def test_with_read_limit(self): + entries = list(read_tor_log(data_path('tor_log'), 5)) + self.assertEqual(5, len(entries)) + self.assertEqual('Interrupt: exiting cleanly.', entries[0].message) + self.assertEqual('Bootstrapped 90%: Establishing a Tor circuit', entries[-1].message) + + def test_with_empty_file(self): + entries = list(read_tor_log(data_path('empty_file'))) + self.assertEqual(0, len(entries)) + + def test_with_missing_path(self): + self.assertRaises(IOError, list, read_tor_log(data_path('no_such_path'))) + + def test_with_malformed_line(self): + try: + list(read_tor_log(data_path('malformed_line'))) + self.fail("Malformed content should've raised a ValueError") + except ValueError as exc: + self.assertTrue("has a line that doesn't match the format we expect: Apr 06 11:03:53.000" in str(exc)) + + def test_with_malformed_runlevel(self): + try: + list(read_tor_log(data_path('malformed_runlevel'))) + self.fail("Malformed content should've raised a ValueError") + except ValueError as exc: + self.assertTrue('has an unrecognized runlevel: [unrecognized]' in str(exc)) + + def test_with_malformed_date(self): + try: + list(read_tor_log(data_path('malformed_date'))) + self.fail("Malformed content should've raised a ValueError") + except ValueError as exc: + self.assertTrue("has a timestamp we don't recognize: Zed 06 11:03:52.000" in str(exc)) diff --git a/test/tracker/__init__.py b/test/tracker/__init__.py new file mode 100644 index 0000000..1b182e4 --- /dev/null +++ b/test/tracker/__init__.py @@ -0,0 +1,10 @@ +""" +Unit tests for nyx's tracker utilities. +""" + +__all__ = [ + 'connection_tracker', + 'daemon', + 'port_usage_tracker', + 'resource_tracker', +] diff --git a/test/tracker/connection_tracker.py b/test/tracker/connection_tracker.py new file mode 100644 index 0000000..39ded42 --- /dev/null +++ b/test/tracker/connection_tracker.py @@ -0,0 +1,115 @@ +import time +import unittest + +from nyx.tracker import ConnectionTracker + +from stem.util import connection + +from mock import Mock, patch + +STEM_CONNECTIONS = [ + connection.Connection('127.0.0.1', 3531, '75.119.206.243', 22, 'tcp', False), + connection.Connection('127.0.0.1', 1766, '86.59.30.40', 443, 'tcp', False), + connection.Connection('127.0.0.1', 1059, '74.125.28.106', 80, 'tcp', False) +] + + +class TestConnectionTracker(unittest.TestCase): + @patch('nyx.tracker.tor_controller') + @patch('nyx.tracker.connection.get_connections') + @patch('nyx.tracker.system', Mock(return_value = Mock())) + @patch('nyx.tracker.connection.system_resolvers', Mock(return_value = [connection.Resolver.NETSTAT])) + def test_fetching_connections(self, get_value_mock, tor_controller_mock): + tor_controller_mock().get_pid.return_value = 12345 + tor_controller_mock().get_conf.return_value = '0' + get_value_mock.return_value = STEM_CONNECTIONS + + with ConnectionTracker(0.04) as daemon: + time.sleep(0.01) + + connections = daemon.get_value() + + self.assertEqual(1, daemon.run_counter()) + self.assertEqual([conn.remote_address for conn in STEM_CONNECTIONS], [conn.remote_address for conn in connections]) + + get_value_mock.return_value = [] # no connection results + time.sleep(0.05) + connections = daemon.get_value() + + self.assertEqual(2, daemon.run_counter()) + self.assertEqual([], connections) + + @patch('nyx.tracker.tor_controller') + @patch('nyx.tracker.connection.get_connections') + @patch('nyx.tracker.system', Mock(return_value = Mock())) + @patch('nyx.tracker.connection.system_resolvers', Mock(return_value = [connection.Resolver.NETSTAT, connection.Resolver.LSOF])) + def test_resolver_failover(self, get_value_mock, tor_controller_mock): + tor_controller_mock().get_pid.return_value = 12345 + tor_controller_mock().get_conf.return_value = '0' + get_value_mock.side_effect = IOError() + + with ConnectionTracker(0.01) as daemon: + time.sleep(0.03) + + self.assertEqual([connection.Resolver.NETSTAT, connection.Resolver.LSOF], daemon._resolvers) + self.assertEqual([], daemon.get_value()) + + time.sleep(0.05) + + self.assertEqual([connection.Resolver.LSOF], daemon._resolvers) + self.assertEqual([], daemon.get_value()) + + time.sleep(0.05) + + self.assertEqual([], daemon._resolvers) + self.assertEqual([], daemon.get_value()) + + # Now make connection resolution work. We still shouldn't provide any + # results since we stopped looking. + + get_value_mock.return_value = STEM_CONNECTIONS[:2] + get_value_mock.side_effect = None + time.sleep(0.05) + self.assertEqual([], daemon.get_value()) + + # Finally, select a custom resolver. This should cause us to query again + # reguardless of our prior failures. + + daemon.set_custom_resolver(connection.Resolver.NETSTAT) + time.sleep(0.05) + self.assertEqual([conn.remote_address for conn in STEM_CONNECTIONS[:2]], [conn.remote_address for conn in daemon.get_value()]) + + @patch('nyx.tracker.tor_controller') + @patch('nyx.tracker.connection.get_connections') + @patch('nyx.tracker.system', Mock(return_value = Mock())) + @patch('nyx.tracker.connection.system_resolvers', Mock(return_value = [connection.Resolver.NETSTAT])) + def test_tracking_uptime(self, get_value_mock, tor_controller_mock): + tor_controller_mock().get_pid.return_value = 12345 + tor_controller_mock().get_conf.return_value = '0' + get_value_mock.return_value = [STEM_CONNECTIONS[0]] + first_start_time = time.time() + + with ConnectionTracker(0.04) as daemon: + time.sleep(0.01) + + connections = daemon.get_value() + self.assertEqual(1, len(connections)) + + self.assertEqual(STEM_CONNECTIONS[0].remote_address, connections[0].remote_address) + self.assertTrue(first_start_time < connections[0].start_time < time.time()) + self.assertTrue(connections[0].is_legacy) + + second_start_time = time.time() + get_value_mock.return_value = STEM_CONNECTIONS[:2] + time.sleep(0.05) + + connections = daemon.get_value() + self.assertEqual(2, len(connections)) + + self.assertEqual(STEM_CONNECTIONS[0].remote_address, connections[0].remote_address) + self.assertTrue(first_start_time < connections[0].start_time < time.time()) + self.assertTrue(connections[0].is_legacy) + + self.assertEqual(STEM_CONNECTIONS[1].remote_address, connections[1].remote_address) + self.assertTrue(second_start_time < connections[1].start_time < time.time()) + self.assertFalse(connections[1].is_legacy) diff --git a/test/tracker/daemon.py b/test/tracker/daemon.py new file mode 100644 index 0000000..4a527f7 --- /dev/null +++ b/test/tracker/daemon.py @@ -0,0 +1,76 @@ +import time +import unittest + +from nyx.tracker import Daemon + +from mock import Mock, patch + + +class TestDaemon(unittest.TestCase): + @patch('nyx.tracker.tor_controller') + @patch('nyx.tracker.system') + def test_init(self, system_mock, tor_controller_mock): + # Check that we register ourselves to listen for status changes, and + # properly retrieve the process' pid and name. + + tor_controller_mock().get_pid.return_value = 12345 + system_mock.name_by_pid.return_value = 'local_tor' + + daemon = Daemon(0.05) + + self.assertEqual(0.05, daemon.get_rate()) + self.assertEqual(12345, daemon._process_pid) + self.assertEqual('local_tor', daemon._process_name) + + tor_controller_mock().add_status_listener.assert_called_with(daemon._tor_status_listener) + system_mock.name_by_pid.assert_called_with(12345) + + @patch('nyx.tracker.tor_controller') + @patch('nyx.tracker.system') + def test_init_without_name(self, system_mock, tor_controller_mock): + # Check when we default to 'tor' if unable to determine the process' name. + + tor_controller_mock().get_pid.return_value = 12345 + system_mock.name_by_pid.return_value = None + + daemon = Daemon(0.05) + self.assertEqual('tor', daemon._process_name) + + @patch('nyx.tracker.tor_controller') + @patch('nyx.tracker.system') + def test_init_without_pid(self, system_mock, tor_controller_mock): + # Check when we can't determine tor's pid. + + tor_controller_mock().get_pid.return_value = None + + daemon = Daemon(0.05) + self.assertEqual(None, daemon._process_pid) + self.assertEqual('tor', daemon._process_name) + self.assertEqual(0, system_mock.call_count) + + @patch('nyx.tracker.tor_controller', Mock(return_value = Mock())) + @patch('nyx.tracker.system', Mock(return_value = Mock())) + def test_daemon_calls_task(self): + # Check that our Daemon calls the task method at the given rate. + + with Daemon(0.01) as daemon: + time.sleep(0.05) + self.assertTrue(2 < daemon.run_counter()) + + @patch('nyx.tracker.tor_controller', Mock(return_value = Mock())) + @patch('nyx.tracker.system', Mock(return_value = Mock())) + def test_pausing_daemon(self): + # Check that we can pause and unpause daemon. + + with Daemon(0.01) as daemon: + time.sleep(0.2) + self.assertTrue(2 < daemon.run_counter()) + + daemon.set_paused(True) + daemon._run_counter = 0 + time.sleep(0.05) + self.assertEqual(0, daemon.run_counter()) + + daemon.set_paused(False) + time.sleep(0.2) + self.assertTrue(2 < daemon.run_counter()) diff --git a/test/tracker/port_usage_tracker.py b/test/tracker/port_usage_tracker.py new file mode 100644 index 0000000..edf1157 --- /dev/null +++ b/test/tracker/port_usage_tracker.py @@ -0,0 +1,117 @@ +import time +import unittest + +from nyx.tracker import Process, PortUsageTracker, _process_for_ports + +from mock import Mock, patch + +LSOF_OUTPUT = """\ +COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME +tor 2001 atagar 14u IPv4 14048 0t0 TCP localhost:9051->localhost:37277 (ESTABLISHED) +tor 2001 atagar 15u IPv4 22024 0t0 TCP localhost:9051->localhost:51849 (ESTABLISHED) +python 2462 atagar 3u IPv4 14047 0t0 TCP localhost:37277->localhost:9051 (ESTABLISHED) +python 3444 atagar 3u IPv4 22023 0t0 TCP localhost:51849->localhost:9051 (ESTABLISHED) +""" + +BAD_LSOF_OUTPUT_NO_ENTRY = """\ +COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME +""" + +BAD_LSOF_OUTPUT_NOT_ESTABLISHED = """\ +COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME +tor 2001 atagar 14u IPv4 14048 0t0 TCP localhost:9051->localhost:37277 (CLOSE_WAIT) +""" + +BAD_LSOF_OUTPUT_MISSING_FIELD = """\ +COMMAND PID USER TYPE DEVICE SIZE/OFF NODE NAME +tor 2001 atagar IPv4 14048 0t0 TCP localhost:9051->localhost:37277 (ESTABLISHED) +""" + +BAD_LSOF_OUTPUT_UNRECOGNIZED_MAPPING = """\ +COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME +tor 2001 atagar 14u IPv4 14048 0t0 TCP localhost:9051=>localhost:37277 (ESTABLISHED) +""" + +BAD_LSOF_OUTPUT_NO_ADDRESS = """\ +COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME +tor 2001 atagar 14u IPv4 14048 0t0 TCP 9051->localhost:37277 (ESTABLISHED) +""" + +BAD_LSOF_OUTPUT_INVALID_PORT = """\ +COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME +tor 2001 atagar 14u IPv4 14048 0t0 TCP localhost:9037351->localhost:37277 (ESTABLISHED) +""" + + +class TestPortUsageTracker(unittest.TestCase): + @patch('nyx.tracker.system.call', Mock(return_value = LSOF_OUTPUT.split('\n'))) + def test_process_for_ports(self): + self.assertEqual({}, _process_for_ports([], [])) + self.assertEqual({80: None, 443: None}, _process_for_ports([80, 443], [])) + self.assertEqual({80: None, 443: None}, _process_for_ports([], [80, 443])) + + self.assertEqual({37277: Process(2462, 'python'), 51849: Process(2001, 'tor')}, _process_for_ports([37277], [51849])) + + @patch('nyx.tracker.system.call') + def test_process_for_ports_malformed(self, call_mock): + # Issues that are valid, but should result in us not having any content. + + test_inputs = ( + BAD_LSOF_OUTPUT_NO_ENTRY, + BAD_LSOF_OUTPUT_NOT_ESTABLISHED, + ) + + for test_input in test_inputs: + call_mock.return_value = test_input.split('\n') + self.assertEqual({80: None, 443: None}, _process_for_ports([80], [443])) + + # Isuses that are reported as errors. + + call_mock.return_value = [] + self.assertRaises(IOError, _process_for_ports, [80], [443]) + + test_inputs = ( + BAD_LSOF_OUTPUT_MISSING_FIELD, + BAD_LSOF_OUTPUT_UNRECOGNIZED_MAPPING, + BAD_LSOF_OUTPUT_NO_ADDRESS, + BAD_LSOF_OUTPUT_INVALID_PORT, + ) + + for test_input in test_inputs: + call_mock.return_value = test_input.split('\n') + self.assertRaises(IOError, _process_for_ports, [80], [443]) + + @patch('nyx.tracker.tor_controller') + @patch('nyx.tracker._process_for_ports') + @patch('nyx.tracker.system', Mock(return_value = Mock())) + def test_fetching_samplings(self, process_for_ports_mock, tor_controller_mock): + tor_controller_mock().get_pid.return_value = 12345 + process_for_ports_mock.return_value = {37277: 'python', 51849: 'tor'} + + with PortUsageTracker(0.02) as daemon: + time.sleep(0.01) + + self.assertEqual({}, daemon.query([37277, 51849], [])) + time.sleep(0.04) + + self.assertEqual({37277: 'python', 51849: 'tor'}, daemon.query([37277, 51849], [])) + + @patch('nyx.tracker.tor_controller') + @patch('nyx.tracker._process_for_ports') + @patch('nyx.tracker.system', Mock(return_value = Mock())) + def test_resolver_failover(self, process_for_ports_mock, tor_controller_mock): + tor_controller_mock().get_pid.return_value = 12345 + process_for_ports_mock.side_effect = IOError() + + with PortUsageTracker(0.01) as daemon: + # We shouldn't attempt lookups (nor encounter failures) without ports to + # query. + + time.sleep(0.05) + self.assertEqual(0, daemon._failure_count) + + daemon.query([37277, 51849], []) + time.sleep(0.03) + self.assertTrue(daemon.is_alive()) + time.sleep(0.1) + self.assertFalse(daemon.is_alive()) diff --git a/test/tracker/resource_tracker.py b/test/tracker/resource_tracker.py new file mode 100644 index 0000000..3f9d927 --- /dev/null +++ b/test/tracker/resource_tracker.py @@ -0,0 +1,143 @@ +import time +import unittest + +from nyx.tracker import ResourceTracker, _resources_via_ps, _resources_via_proc + +from mock import Mock, patch + +PS_OUTPUT = """\ + TIME ELAPSED RSS %MEM +00:00:02 00:18 18848 0.4 +""" + + +class TestResourceTracker(unittest.TestCase): + @patch('nyx.tracker.tor_controller') + @patch('nyx.tracker._resources_via_proc') + @patch('nyx.tracker.system', Mock(return_value = Mock())) + @patch('nyx.tracker.proc.is_available', Mock(return_value = True)) + def test_fetching_samplings(self, resources_via_proc_mock, tor_controller_mock): + tor_controller_mock().get_pid.return_value = 12345 + resources_via_proc_mock.return_value = (105.3, 2.4, 8072, 0.3) + + with ResourceTracker(0.04) as daemon: + time.sleep(0.01) + + resources = daemon.get_value() + + self.assertEqual(1, daemon.run_counter()) + self.assertEqual(0.0, resources.cpu_sample) + self.assertEqual(43.875, resources.cpu_average) + self.assertEqual(105.3, resources.cpu_total) + self.assertEqual(8072, resources.memory_bytes) + self.assertEqual(0.3, resources.memory_percent) + self.assertTrue((time.time() - resources.timestamp) < 0.5) + + resources_via_proc_mock.return_value = (800.3, 3.2, 6020, 0.26) + time.sleep(0.05) + resources = daemon.get_value() + + self.assertEqual(2, daemon.run_counter()) + self.assertEqual(6.600189933523267, resources.cpu_sample) + self.assertEqual(250.09374999999997, resources.cpu_average) + self.assertEqual(800.3, resources.cpu_total) + self.assertEqual(6020, resources.memory_bytes) + self.assertEqual(0.26, resources.memory_percent) + self.assertTrue((time.time() - resources.timestamp) < 0.5) + + resources_via_proc_mock.assert_called_with(12345) + + @patch('nyx.tracker.tor_controller') + @patch('nyx.tracker.proc.is_available') + @patch('nyx.tracker._resources_via_ps', Mock(return_value = (105.3, 2.4, 8072, 0.3))) + @patch('nyx.tracker._resources_via_proc', Mock(return_value = (340.3, 3.2, 6020, 0.26))) + @patch('nyx.tracker.system', Mock(return_value = Mock())) + def test_picking_proc_or_ps(self, is_proc_available_mock, tor_controller_mock): + tor_controller_mock().get_pid.return_value = 12345 + + is_proc_available_mock.return_value = True + + with ResourceTracker(0.04) as daemon: + time.sleep(0.01) + + resources = daemon.get_value() + + self.assertEqual(1, daemon.run_counter()) + self.assertEqual(0.0, resources.cpu_sample) + self.assertEqual(106.34375, resources.cpu_average) + self.assertEqual(340.3, resources.cpu_total) + self.assertEqual(6020, resources.memory_bytes) + self.assertEqual(0.26, resources.memory_percent) + self.assertTrue((time.time() - resources.timestamp) < 0.5) + + is_proc_available_mock.return_value = False + + with ResourceTracker(0.04) as daemon: + time.sleep(0.01) + + resources = daemon.get_value() + + self.assertEqual(1, daemon.run_counter()) + self.assertEqual(0.0, resources.cpu_sample) + self.assertEqual(43.875, resources.cpu_average) + self.assertEqual(105.3, resources.cpu_total) + self.assertEqual(8072, resources.memory_bytes) + self.assertEqual(0.3, resources.memory_percent) + self.assertTrue((time.time() - resources.timestamp) < 0.5) + + @patch('nyx.tracker.tor_controller') + @patch('nyx.tracker._resources_via_ps', Mock(return_value = (105.3, 2.4, 8072, 0.3))) + @patch('nyx.tracker._resources_via_proc', Mock(side_effect = IOError())) + @patch('nyx.tracker.system', Mock(return_value = Mock())) + @patch('nyx.tracker.proc.is_available', Mock(return_value = True)) + def test_failing_over_to_ps(self, tor_controller_mock): + tor_controller_mock().get_pid.return_value = 12345 + + with ResourceTracker(0.01) as daemon: + time.sleep(0.03) + + self.assertEqual(True, daemon._use_proc) + resources = daemon.get_value() + + self.assertEqual(0, daemon.run_counter()) + self.assertEqual(0.0, resources.cpu_sample) + self.assertEqual(0.0, resources.cpu_average) + self.assertEqual(0, resources.cpu_total) + self.assertEqual(0, resources.memory_bytes) + self.assertEqual(0.0, resources.memory_percent) + self.assertEqual(0.0, resources.timestamp) + + while daemon.run_counter() < 1: + time.sleep(0.01) + + self.assertEqual(False, daemon._use_proc) + + resources = daemon.get_value() + + self.assertEqual(0.0, resources.cpu_sample) + self.assertEqual(43.875, resources.cpu_average) + self.assertEqual(105.3, resources.cpu_total) + self.assertEqual(8072, resources.memory_bytes) + self.assertEqual(0.3, resources.memory_percent) + self.assertTrue((time.time() - resources.timestamp) < 0.5) + + @patch('nyx.tracker.system.call', Mock(return_value = PS_OUTPUT.split('\n'))) + def test_resources_via_ps(self): + total_cpu_time, uptime, memory_in_bytes, memory_in_percent = _resources_via_ps(12345) + + self.assertEqual(2.0, total_cpu_time) + self.assertEqual(18, uptime) + self.assertEqual(19300352, memory_in_bytes) + self.assertEqual(0.004, memory_in_percent) + + @patch('time.time', Mock(return_value = 1388967218.973117)) + @patch('nyx.tracker.proc.stats', Mock(return_value = (1.5, 0.5, 1388967200.9))) + @patch('nyx.tracker.proc.memory_usage', Mock(return_value = (19300352, 6432))) + @patch('nyx.tracker.proc.physical_memory', Mock(return_value = 4825088000)) + def test_resources_via_proc(self): + total_cpu_time, uptime, memory_in_bytes, memory_in_percent = _resources_via_proc(12345) + + self.assertEqual(2.0, total_cpu_time) + self.assertEqual(18, int(uptime)) + self.assertEqual(19300352, memory_in_bytes) + self.assertEqual(0.004, memory_in_percent) diff --git a/test/util/__init__.py b/test/util/__init__.py deleted file mode 100644 index a03ad4f..0000000 --- a/test/util/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -""" -Unit tests for nyx's utilities. -""" - -__all__ = ['log', 'tracker'] diff --git a/test/util/expand_path.py b/test/util/expand_path.py deleted file mode 100644 index 3f3d5ff..0000000 --- a/test/util/expand_path.py +++ /dev/null @@ -1,21 +0,0 @@ -import unittest - -from nyx.util import expand_path, uses_settings - -from mock import patch, Mock - - -class TestExpandPath(unittest.TestCase): - @patch('nyx.util.tor_controller') - @patch('stem.util.system.cwd', Mock(return_value = '/your_cwd')) - @uses_settings - def test_expand_path(self, tor_controller_mock, config): - tor_controller_mock().get_pid.return_value = 12345 - self.assertEqual('/absolute/path/to/torrc', expand_path('/absolute/path/to/torrc')) - self.assertEqual('/your_cwd/torrc', expand_path('torrc')) - - config.set('tor.chroot', '/chroot') - self.assertEqual('/chroot/absolute/path/to/torrc', expand_path('/absolute/path/to/torrc')) - self.assertEqual('/chroot/your_cwd/torrc', expand_path('torrc')) - - config.set('tor.chroot', None) diff --git a/test/util/log/__init__.py b/test/util/log/__init__.py deleted file mode 100644 index 9896955..0000000 --- a/test/util/log/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -""" -Unit tests for nyx's log utilities. -""" - -__all__ = [ - 'deduplication', - 'read_tor_log', -] diff --git a/test/util/log/condense_runlevels.py b/test/util/log/condense_runlevels.py deleted file mode 100644 index d9e62cd..0000000 --- a/test/util/log/condense_runlevels.py +++ /dev/null @@ -1,13 +0,0 @@ -import unittest - -from nyx.util.log import condense_runlevels - - -class TestCondenseRunlevels(unittest.TestCase): - def test_condense_runlevels(self): - self.assertEqual([], condense_runlevels()) - self.assertEqual(['BW'], condense_runlevels('BW')) - self.assertEqual(['DEBUG', 'NOTICE', 'ERR'], condense_runlevels('DEBUG', 'NOTICE', 'ERR')) - self.assertEqual(['DEBUG-NOTICE', 'NYX DEBUG-INFO'], condense_runlevels('DEBUG', 'NYX_DEBUG', 'INFO', 'NYX_INFO', 'NOTICE')) - self.assertEqual(['TOR/NYX NOTICE-ERR'], condense_runlevels('NOTICE', 'WARN', 'ERR', 'NYX_NOTICE', 'NYX_WARN', 'NYX_ERR')) - self.assertEqual(['DEBUG', 'TOR/NYX NOTICE-ERR', 'BW'], condense_runlevels('DEBUG', 'NOTICE', 'WARN', 'ERR', 'NYX_NOTICE', 'NYX_WARN', 'NYX_ERR', 'BW')) diff --git a/test/util/log/data/daybreak_deduplication b/test/util/log/data/daybreak_deduplication deleted file mode 100644 index 1849ad3..0000000 --- a/test/util/log/data/daybreak_deduplication +++ /dev/null @@ -1,37 +0,0 @@ -Apr 24 19:50:42.000 [notice] Heartbeat: Tor's uptime is 12 days 0:00 hours, with 0 circuits open. I've sent 4.94 MB and received 130.17 MB. -Apr 24 19:50:42.000 [notice] Average packaged cell fullness: 65.101%. TLS write overhead: 11% -Apr 25 01:50:42.000 [notice] Heartbeat: Tor's uptime is 12 days 6:00 hours, with 0 circuits open. I've sent 5.00 MB and received 131.87 MB. -Apr 25 01:50:42.000 [notice] Average packaged cell fullness: 64.927%. TLS write overhead: 11% -Apr 25 07:50:42.000 [notice] Heartbeat: Tor's uptime is 12 days 12:00 hours, with 0 circuits open. I've sent 5.08 MB and received 134.19 MB. -Apr 25 07:50:42.000 [notice] Average packaged cell fullness: 64.587%. TLS write overhead: 11% -Apr 25 11:44:21.000 [notice] New control connection opened from 127.0.0.1. -Apr 25 11:44:33.000 [notice] Interrupt: exiting cleanly. -Apr 25 11:44:36.000 [notice] Tor 0.2.7.0-alpha-dev (git-63a90f2df4dcd7ff) opening log file. -Apr 25 11:44:36.492 [notice] Tor v0.2.7.0-alpha-dev (git-63a90f2df4dcd7ff) running on Linux with Libevent 2.0.16-stable, OpenSSL 1.0.1 and Zlib 1.2.3.4. -Apr 25 11:44:36.492 [notice] Tor can't help you if you use it wrong! Learn how to be safe at https://www.torproject.org/download/download#warning -Apr 25 11:44:36.492 [notice] This version is not a stable Tor release. Expect more bugs than usual. -Apr 25 11:44:36.525 [notice] Read configuration file "/home/atagar/.tor/torrc". -Apr 25 11:44:36.530 [notice] Opening Socks listener on 127.0.0.1:9050 -Apr 25 11:44:36.530 [notice] Opening Control listener on 127.0.0.1:9051 -Apr 25 11:44:36.000 [notice] Bootstrapped 0%: Starting -Apr 25 11:44:39.000 [notice] Bootstrapped 45%: Asking for relay descriptors -Apr 25 11:44:40.000 [notice] Bootstrapped 50%: Loading relay descriptors -Apr 25 11:44:47.000 [notice] Bootstrapped 55%: Loading relay descriptors -Apr 25 11:44:47.000 [notice] Bootstrapped 62%: Loading relay descriptors -Apr 25 11:44:48.000 [notice] Bootstrapped 72%: Loading relay descriptors -Apr 25 11:44:48.000 [notice] Bootstrapped 80%: Connecting to the Tor network -Apr 25 11:44:48.000 [notice] Bootstrapped 90%: Establishing a Tor circuit -Apr 25 11:44:49.000 [notice] Tor has successfully opened a circuit. Looks like client functionality is working. -Apr 25 11:44:49.000 [notice] Bootstrapped 100%: Done -Apr 25 11:45:03.000 [notice] New control connection opened from 127.0.0.1. -Apr 25 17:44:40.000 [notice] Heartbeat: Tor's uptime is 6:00 hours, with 0 circuits open. I've sent 539 kB and received 4.25 MB. -Apr 25 18:11:56.000 [notice] New control connection opened from 127.0.0.1. -Apr 25 18:52:47.000 [notice] New control connection opened from 127.0.0.1. -Apr 25 19:02:44.000 [notice] New control connection opened from 127.0.0.1. -Apr 25 23:44:40.000 [notice] Heartbeat: Tor's uptime is 12:00 hours, with 1 circuits open. I've sent 794 kB and received 7.32 MB. -Apr 26 05:44:40.000 [notice] Heartbeat: Tor's uptime is 18:00 hours, with 0 circuits open. I've sent 862 kB and received 9.05 MB. -Apr 26 10:16:38.000 [notice] New control connection opened from 127.0.0.1. -Apr 26 10:19:24.000 [notice] New control connection opened from 127.0.0.1. -Apr 26 10:21:31.000 [notice] New control connection opened from 127.0.0.1. -Apr 26 10:24:27.000 [notice] New control connection opened from 127.0.0.1. -Apr 26 10:24:46.000 [notice] New control connection opened from 127.0.0.1. diff --git a/test/util/log/data/empty_file b/test/util/log/data/empty_file deleted file mode 100644 index e69de29..0000000 diff --git a/test/util/log/data/malformed_date b/test/util/log/data/malformed_date deleted file mode 100644 index 712149d..0000000 --- a/test/util/log/data/malformed_date +++ /dev/null @@ -1,21 +0,0 @@ -Apr 06 11:03:39.000 [notice] Tor 0.2.7.0-alpha-dev (git-4247ce99e5d9b7b2) opening new log file. -Apr 06 11:03:39.832 [notice] Tor v0.2.7.0-alpha-dev (git-4247ce99e5d9b7b2) running on Linux with Libevent 2.0.16-stable, OpenSSL 1.0.1 and Zlib 1.2.3.4. -Apr 06 11:03:39.832 [notice] Tor can't help you if you use it wrong! Learn how to be safe at https://www.torproject.org/download/download#warning -Apr 06 11:03:39.833 [notice] This version is not a stable Tor release. Expect more bugs than usual. -Apr 06 11:03:39.833 [notice] Read configuration file "/home/atagar/.tor/torrc". -Apr 06 11:03:39.838 [notice] Opening Socks listener on 127.0.0.1:9050 -Apr 06 11:03:39.838 [notice] Opening Control listener on 127.0.0.1:9051 -Apr 06 11:03:39.000 [notice] Bootstrapped 0%: Starting -Apr 06 11:03:42.000 [notice] Bootstrapped 45%: Asking for relay descriptors -Apr 06 11:03:43.000 [notice] Bootstrapped 50%: Loading relay descriptors -Apr 06 11:03:49.000 [notice] New control connection opened from 127.0.0.1. -Apr 06 11:03:51.000 [notice] Bootstrapped 55%: Loading relay descriptors -Apr 06 11:03:51.000 [notice] Bootstrapped 63%: Loading relay descriptors -Apr 06 11:03:51.000 [notice] Bootstrapped 69%: Loading relay descriptors -Apr 06 11:03:51.000 [notice] Bootstrapped 74%: Loading relay descriptors -Apr 06 11:03:52.000 [notice] Bootstrapped 80%: Connecting to the Tor network -Zed 06 11:03:52.000 [notice] Bootstrapped 90%: Establishing a Tor circuit -Apr 06 11:03:53.000 [notice] Tor has successfully opened a circuit. Looks like client functionality is working. -Apr 06 11:03:53.000 [notice] Bootstrapped 100%: Done -Apr 06 11:03:55.000 [notice] New control connection opened from 127.0.0.1. -Apr 06 11:53:46.000 [notice] Interrupt: exiting cleanly. diff --git a/test/util/log/data/malformed_line b/test/util/log/data/malformed_line deleted file mode 100644 index ac9a367..0000000 --- a/test/util/log/data/malformed_line +++ /dev/null @@ -1,21 +0,0 @@ -Apr 06 11:03:39.000 [notice] Tor 0.2.7.0-alpha-dev (git-4247ce99e5d9b7b2) opening new log file. -Apr 06 11:03:39.832 [notice] Tor v0.2.7.0-alpha-dev (git-4247ce99e5d9b7b2) running on Linux with Libevent 2.0.16-stable, OpenSSL 1.0.1 and Zlib 1.2.3.4. -Apr 06 11:03:39.832 [notice] Tor can't help you if you use it wrong! Learn how to be safe at https://www.torproject.org/download/download#warning -Apr 06 11:03:39.833 [notice] This version is not a stable Tor release. Expect more bugs than usual. -Apr 06 11:03:39.833 [notice] Read configuration file "/home/atagar/.tor/torrc". -Apr 06 11:03:39.838 [notice] Opening Socks listener on 127.0.0.1:9050 -Apr 06 11:03:39.838 [notice] Opening Control listener on 127.0.0.1:9051 -Apr 06 11:03:39.000 [notice] Bootstrapped 0%: Starting -Apr 06 11:03:42.000 [notice] Bootstrapped 45%: Asking for relay descriptors -Apr 06 11:03:43.000 [notice] Bootstrapped 50%: Loading relay descriptors -Apr 06 11:03:49.000 [notice] New control connection opened from 127.0.0.1. -Apr 06 11:03:51.000 [notice] Bootstrapped 55%: Loading relay descriptors -Apr 06 11:03:51.000 [notice] Bootstrapped 63%: Loading relay descriptors -Apr 06 11:03:51.000 [notice] Bootstrapped 69%: Loading relay descriptors -Apr 06 11:03:51.000 [notice] Bootstrapped 74%: Loading relay descriptors -Apr 06 11:03:52.000 [notice] Bootstrapped 80%: Connecting to the Tor network -Apr 06 11:03:52.000 [notice] Bootstrapped 90%: Establishing a Tor circuit -Apr 06 11:03:53.000 -Apr 06 11:03:53.000 [notice] Bootstrapped 100%: Done -Apr 06 11:03:55.000 [notice] New control connection opened from 127.0.0.1. -Apr 06 11:53:46.000 [notice] Interrupt: exiting cleanly. diff --git a/test/util/log/data/malformed_runlevel b/test/util/log/data/malformed_runlevel deleted file mode 100644 index dd8810b..0000000 --- a/test/util/log/data/malformed_runlevel +++ /dev/null @@ -1,21 +0,0 @@ -Apr 06 11:03:39.000 [notice] Tor 0.2.7.0-alpha-dev (git-4247ce99e5d9b7b2) opening new log file. -Apr 06 11:03:39.832 [notice] Tor v0.2.7.0-alpha-dev (git-4247ce99e5d9b7b2) running on Linux with Libevent 2.0.16-stable, OpenSSL 1.0.1 and Zlib 1.2.3.4. -Apr 06 11:03:39.832 [notice] Tor can't help you if you use it wrong! Learn how to be safe at https://www.torproject.org/download/download#warning -Apr 06 11:03:39.833 [notice] This version is not a stable Tor release. Expect more bugs than usual. -Apr 06 11:03:39.833 [notice] Read configuration file "/home/atagar/.tor/torrc". -Apr 06 11:03:39.838 [notice] Opening Socks listener on 127.0.0.1:9050 -Apr 06 11:03:39.838 [notice] Opening Control listener on 127.0.0.1:9051 -Apr 06 11:03:39.000 [notice] Bootstrapped 0%: Starting -Apr 06 11:03:42.000 [notice] Bootstrapped 45%: Asking for relay descriptors -Apr 06 11:03:43.000 [notice] Bootstrapped 50%: Loading relay descriptors -Apr 06 11:03:49.000 [notice] New control connection opened from 127.0.0.1. -Apr 06 11:03:51.000 [notice] Bootstrapped 55%: Loading relay descriptors -Apr 06 11:03:51.000 [notice] Bootstrapped 63%: Loading relay descriptors -Apr 06 11:03:51.000 [notice] Bootstrapped 69%: Loading relay descriptors -Apr 06 11:03:51.000 [notice] Bootstrapped 74%: Loading relay descriptors -Apr 06 11:03:52.000 [notice] Bootstrapped 80%: Connecting to the Tor network -Apr 06 11:03:52.000 [unrecognized] Bootstrapped 90%: Establishing a Tor circuit -Apr 06 11:03:53.000 [notice] Tor has successfully opened a circuit. Looks like client functionality is working. -Apr 06 11:03:53.000 [notice] Bootstrapped 100%: Done -Apr 06 11:03:55.000 [notice] New control connection opened from 127.0.0.1. -Apr 06 11:53:46.000 [notice] Interrupt: exiting cleanly. diff --git a/test/util/log/data/multiple_tor_instances b/test/util/log/data/multiple_tor_instances deleted file mode 100644 index 1793ce5..0000000 --- a/test/util/log/data/multiple_tor_instances +++ /dev/null @@ -1,33 +0,0 @@ -Apr 06 11:03:39.000 [notice] Tor 0.2.7.0-alpha-dev (git-4247ce99e5d9b7b2) opening new log file. -Apr 06 11:03:39.832 [notice] Tor v0.2.7.0-alpha-dev (git-4247ce99e5d9b7b2) running on Linux with Libevent 2.0.16-stable, OpenSSL 1.0.1 and Zlib 1.2.3.4. -Apr 06 11:03:39.832 [notice] Tor can't help you if you use it wrong! Learn how to be safe at https://www.torproject.org/download/download#warning -Apr 06 11:03:39.833 [notice] This version is not a stable Tor release. Expect more bugs than usual. -Apr 06 11:03:39.833 [notice] Read configuration file "/home/atagar/.tor/torrc". -Apr 06 11:03:39.838 [notice] Opening Socks listener on 127.0.0.1:9050 -Apr 06 11:03:39.838 [notice] Opening Control listener on 127.0.0.1:9051 -Apr 06 11:03:39.000 [notice] Bootstrapped 0%: Starting -Apr 06 11:03:42.000 [notice] Bootstrapped 45%: Asking for relay descriptors -Apr 06 11:03:43.000 [notice] Bootstrapped 50%: Loading relay descriptors -Apr 06 11:03:49.000 [notice] New control connection opened from 127.0.0.1. -Apr 06 11:03:51.000 [notice] Bootstrapped 55%: Loading relay descriptors -Apr 06 11:03:51.000 [notice] Bootstrapped 63%: Loading relay descriptors -Apr 06 11:03:51.000 [notice] Bootstrapped 69%: Loading relay descriptors -Apr 06 11:03:51.000 [notice] Bootstrapped 74%: Loading relay descriptors -Apr 06 11:03:52.000 [notice] Bootstrapped 80%: Connecting to the Tor network -Apr 06 11:03:52.000 [notice] Bootstrapped 90%: Establishing a Tor circuit -Apr 06 11:03:53.000 [notice] Tor has successfully opened a circuit. Looks like client functionality is working. -Apr 06 11:03:53.000 [notice] Bootstrapped 100%: Done -Apr 06 11:03:55.000 [notice] New control connection opened from 127.0.0.1. -Apr 06 11:53:46.000 [notice] Interrupt: exiting cleanly. -Apr 06 11:53:54.000 [notice] Tor 0.2.7.0-alpha-dev (git-4247ce99e5d9b7b2) opening log file. -Apr 06 11:53:54.392 [notice] Tor v0.2.7.0-alpha-dev (git-4247ce99e5d9b7b2) running on Linux with Libevent 2.0.16-stable, OpenSSL 1.0.1 and Zlib 1.2.3.4. -Apr 06 11:53:54.392 [notice] Tor can't help you if you use it wrong! Learn how to be safe at https://www.torproject.org/download/download#warning -Apr 06 11:53:54.392 [notice] This version is not a stable Tor release. Expect more bugs than usual. -Apr 06 11:53:54.392 [notice] Read configuration file "/home/atagar/.tor/torrc". -Apr 06 11:53:54.396 [notice] Opening Socks listener on 127.0.0.1:9050 -Apr 06 11:53:54.396 [notice] Opening Control listener on 127.0.0.1:9051 -Apr 06 11:53:54.000 [warn] Your log may contain sensitive information - you're logging more than "notice". Don't log unless it serves an important reason. Overwrite the log afterwards. -Apr 06 11:53:54.000 [debug] tor_disable_debugger_attach(): Attemping to disable debugger attachment to Tor for unprivileged users. -Apr 06 11:53:54.000 [debug] tor_disable_debugger_attach(): Debugger attachment disabled for unprivileged users. -Apr 06 11:53:54.000 [info] tor_lockfile_lock(): Locking "/home/atagar/.tor/lock" -Apr 06 11:53:54.000 [debug] parse_dir_authority_line(): Trusted 100 dirserver at 128.31.0.39:9131 (9695) diff --git a/test/util/log/data/tor_log b/test/util/log/data/tor_log deleted file mode 100644 index 0a4464e..0000000 --- a/test/util/log/data/tor_log +++ /dev/null @@ -1,21 +0,0 @@ -Apr 06 11:03:39.000 [notice] Tor 0.2.7.0-alpha-dev (git-4247ce99e5d9b7b2) opening new log file. -Apr 06 11:03:39.832 [notice] Tor v0.2.7.0-alpha-dev (git-4247ce99e5d9b7b2) running on Linux with Libevent 2.0.16-stable, OpenSSL 1.0.1 and Zlib 1.2.3.4. -Apr 06 11:03:39.832 [notice] Tor can't help you if you use it wrong! Learn how to be safe at https://www.torproject.org/download/download#warning -Apr 06 11:03:39.833 [notice] This version is not a stable Tor release. Expect more bugs than usual. -Apr 06 11:03:39.833 [notice] Read configuration file "/home/atagar/.tor/torrc". -Apr 06 11:03:39.838 [notice] Opening Socks listener on 127.0.0.1:9050 -Apr 06 11:03:39.838 [notice] Opening Control listener on 127.0.0.1:9051 -Apr 06 11:03:39.000 [notice] Bootstrapped 0%: Starting -Apr 06 11:03:42.000 [notice] Bootstrapped 45%: Asking for relay descriptors -Apr 06 11:03:43.000 [notice] Bootstrapped 50%: Loading relay descriptors -Apr 06 11:03:49.000 [notice] New control connection opened from 127.0.0.1. -Apr 06 11:03:51.000 [notice] Bootstrapped 55%: Loading relay descriptors -Apr 06 11:03:51.000 [notice] Bootstrapped 63%: Loading relay descriptors -Apr 06 11:03:51.000 [notice] Bootstrapped 69%: Loading relay descriptors -Apr 06 11:03:51.000 [notice] Bootstrapped 74%: Loading relay descriptors -Apr 06 11:03:52.000 [notice] Bootstrapped 80%: Connecting to the Tor network -Apr 06 11:03:52.000 [notice] Bootstrapped 90%: Establishing a Tor circuit -Apr 06 11:03:53.000 [notice] Tor has successfully opened a circuit. Looks like client functionality is working. -Apr 06 11:03:53.000 [notice] Bootstrapped 100%: Done -Apr 06 11:03:55.000 [notice] New control connection opened from 127.0.0.1. -Apr 06 11:53:46.000 [notice] Interrupt: exiting cleanly. diff --git a/test/util/log/log_entry.py b/test/util/log/log_entry.py deleted file mode 100644 index bd570f4..0000000 --- a/test/util/log/log_entry.py +++ /dev/null @@ -1,27 +0,0 @@ -import unittest - -from nyx.util.log import LogEntry - - -class TestLogEntry(unittest.TestCase): - def test_deduplication_matches_identical_messages(self): - # Simple case is that we match the same message but different timestamp. - - entry = LogEntry(1333738434, 'INFO', 'tor_lockfile_lock(): Locking "/home/atagar/.tor/lock"') - self.assertTrue(entry.is_duplicate_of(LogEntry(1333738457, 'INFO', 'tor_lockfile_lock(): Locking "/home/atagar/.tor/lock"'))) - - # ... but we shouldn't match if the runlevel differs. - - self.assertFalse(entry.is_duplicate_of(LogEntry(1333738457, 'DEBUG', 'tor_lockfile_lock(): Locking "/home/atagar/.tor/lock"'))) - - def test_deduplication_matches_based_on_prefix(self): - # matches using a prefix specified in dedup.cfg - - entry = LogEntry(1333738434, 'NYX_DEBUG', 'GETCONF MyFamily (runtime: 0.0007)') - self.assertTrue(entry.is_duplicate_of(LogEntry(1333738457, 'NYX_DEBUG', 'GETCONF MyFamily (runtime: 0.0015)'))) - - def test_deduplication_matches_with_wildcard(self): - # matches using a wildcard specified in dedup.cfg - - entry = LogEntry(1333738434, 'NOTICE', 'Bootstrapped 72%: Loading relay descriptors.') - self.assertTrue(entry.is_duplicate_of(LogEntry(1333738457, 'NOTICE', 'Bootstrapped 55%: Loading relay descriptors.'))) diff --git a/test/util/log/log_group.py b/test/util/log/log_group.py deleted file mode 100644 index f4716ca..0000000 --- a/test/util/log/log_group.py +++ /dev/null @@ -1,146 +0,0 @@ -import os -import unittest - -from nyx.util.log import LogGroup, LogEntry, read_tor_log - - -class TestLogGroup(unittest.TestCase): - def test_maintains_certain_size(self): - group = LogGroup(5) - self.assertEqual(0, len(group)) - - group.add(LogEntry(1333738410, 'INFO', 'tor_lockfile_lock(): Locking "/home/atagar/.tor/lock"')) - self.assertEqual([LogEntry(1333738410, 'INFO', 'tor_lockfile_lock(): Locking "/home/atagar/.tor/lock"')], list(group)) - self.assertEqual(1, len(group)) - - group.add(LogEntry(1333738420, 'NYX_DEBUG', 'GETCONF MyFamily (runtime: 0.0007)')) - group.add(LogEntry(1333738430, 'NOTICE', 'Bootstrapped 72%: Loading relay descriptors.')) - group.add(LogEntry(1333738440, 'NOTICE', 'Bootstrapped 75%: Loading relay descriptors.')) - group.add(LogEntry(1333738450, 'NOTICE', 'Bootstrapped 78%: Loading relay descriptors.')) - self.assertEqual(5, len(group)) - - # group should now be full, adding more entries pops others off - - group.add(LogEntry(1333738460, 'NOTICE', 'Bootstrapped 80%: Loading relay descriptors.')) - self.assertFalse(LogEntry(1333738410, 'INFO', 'tor_lockfile_lock(): Locking "/home/atagar/.tor/lock"') in list(group)) - self.assertEqual(5, len(group)) - - # try adding a bunch that will be deduplicated, and make sure we still maintain the size - - group.add(LogEntry(1333738510, 'NOTICE', 'Bootstrapped 80%: Loading relay descriptors.')) - group.add(LogEntry(1333738520, 'NOTICE', 'Bootstrapped 80%: Loading relay descriptors.')) - group.add(LogEntry(1333738530, 'NOTICE', 'Bootstrapped 80%: Loading relay descriptors.')) - group.add(LogEntry(1333738540, 'NOTICE', 'Bootstrapped 80%: Loading relay descriptors.')) - group.add(LogEntry(1333738550, 'NOTICE', 'Bootstrapped 80%: Loading relay descriptors.')) - group.add(LogEntry(1333738560, 'NOTICE', 'Bootstrapped 80%: Loading relay descriptors.')) - group.add(LogEntry(1333738570, 'NOTICE', 'Bootstrapped 80%: Loading relay descriptors.')) - self.assertEqual([1333738570, 1333738560, 1333738550, 1333738540, 1333738530], [e.timestamp for e in group]) - self.assertEqual(5, len(group)) - - def test_deduplication(self): - group = LogGroup(5) - group.add(LogEntry(1333738410, 'NOTICE', 'Bootstrapped 72%: Loading relay descriptors.')) - group.add(LogEntry(1333738420, 'NOTICE', 'Bootstrapped 75%: Loading relay descriptors.')) - group.add(LogEntry(1333738430, 'NYX_DEBUG', 'GETCONF MyFamily (runtime: 0.0007)')) - group.add(LogEntry(1333738440, 'NOTICE', 'Bootstrapped 78%: Loading relay descriptors.')) - group.add(LogEntry(1333738450, 'NOTICE', 'Bootstrapped 80%: Loading relay descriptors.')) - self.assertEqual([1333738450, 1333738440, 1333738430, 1333738420, 1333738410], [e.timestamp for e in group]) - - bootstrap_messages = [ - 'Bootstrapped 80%: Loading relay descriptors.', - 'Bootstrapped 78%: Loading relay descriptors.', - 'Bootstrapped 75%: Loading relay descriptors.', - 'Bootstrapped 72%: Loading relay descriptors.', - ] - - group_items = list(group) - self.assertEqual(bootstrap_messages, [e.message for e in group_items[0].duplicates]) - self.assertEqual([False, True, False, True, True], [e.is_duplicate for e in group_items]) - - # add another duplicate message that pops the last - - group.add(LogEntry(1333738460, 'NOTICE', 'Bootstrapped 90%: Loading relay descriptors.')) - - bootstrap_messages = [ - 'Bootstrapped 90%: Loading relay descriptors.', - 'Bootstrapped 80%: Loading relay descriptors.', - 'Bootstrapped 78%: Loading relay descriptors.', - 'Bootstrapped 75%: Loading relay descriptors.', - ] - - group_items = list(group) - self.assertEqual(bootstrap_messages, [e.message for e in group_items[0].duplicates]) - self.assertEqual([False, True, True, False, True], [e.is_duplicate for e in group_items]) - - # add another non-duplicate message that pops the last - - group.add(LogEntry(1333738470, 'INFO', 'tor_lockfile_lock(): Locking "/home/atagar/.tor/lock"')) - - bootstrap_messages = [ - 'Bootstrapped 90%: Loading relay descriptors.', - 'Bootstrapped 80%: Loading relay descriptors.', - 'Bootstrapped 78%: Loading relay descriptors.', - ] - - group_items = list(group) - self.assertEqual(None, group_items[0].duplicates) - self.assertEqual(bootstrap_messages, [e.message for e in group_items[1].duplicates]) - self.assertEqual([False, False, True, True, False], [e.is_duplicate for e in group_items]) - - def test_deduplication_with_daybreaks(self): - group = LogGroup(100, group_by_day = True) - test_log_path = os.path.join(os.path.dirname(__file__), 'data', 'daybreak_deduplication') - - for entry in reversed(list(read_tor_log(test_log_path))): - group.add(entry) - - # Entries should consist of two days of results... - # - # Day 1: - # 10:24:27 [NOTICE] New control connection opened from 127.0.0.1. - # 10:21:31 [NOTICE] New control connection opened from 127.0.0.1. - # 10:19:24 [NOTICE] New control connection opened from 127.0.0.1. - # 10:16:38 [NOTICE] New control connection opened from 127.0.0.1. - # 10:16:38 [NOTICE] New control connection opened from 127.0.0.1. - # 05:44:40 [NOTICE] Heartbeat: Tor's uptime is 18:00 hours, with 0 circuits open. I've sent 862 kB and received 9.05 MB. - # - # Day 2: - # 23:44:40 [NOTICE] Heartbeat: Tor's uptime is 12:00 hours, with 1 circuits open. I've sent 794 kB and received 7.32 MB. - # 19:02:44 [NOTICE] New control connection opened from 127.0.0.1. - # 18:52:47 [NOTICE] New control connection opened from 127.0.0.1. - # 18:11:56 [NOTICE] New control connection opened from 127.0.0.1. - # 17:44:40 [NOTICE] Heartbeat: Tor's uptime is 6:00 hours, with 0 circuits open. I've sent 539 kB and received 4.25 MB. - # 11:45:03 [NOTICE] New control connection opened from 127.0.0.1. - # 11:44:49 [NOTICE] Bootstrapped 100%: Done - # ... etc... - - group_items = list(group) - - # First day - - self.assertEqual('New control connection opened from 127.0.0.1.', group_items[0].message) - self.assertEqual(5, len(group_items[0].duplicates)) - self.assertFalse(group_items[0].is_duplicate) - - for entry in group_items[1:5]: - self.assertEqual('New control connection opened from 127.0.0.1.', entry.message) - self.assertEqual(5, len(entry.duplicates)) - self.assertTrue(entry.is_duplicate) - - self.assertEqual("Heartbeat: Tor's uptime is 18:00 hours, with 0 circuits open. I've sent 862 kB and received 9.05 MB.", group_items[5].message) - self.assertEqual(None, group_items[5].duplicates) - self.assertFalse(group_items[5].is_duplicate) - - # Second day - - self.assertEqual("Heartbeat: Tor's uptime is 12:00 hours, with 1 circuits open. I've sent 794 kB and received 7.32 MB.", group_items[6].message) - self.assertEqual(2, len(group_items[6].duplicates)) - self.assertFalse(group_items[6].is_duplicate) - - self.assertEqual('New control connection opened from 127.0.0.1.', group_items[8].message) - self.assertEqual(4, len(group_items[8].duplicates)) - self.assertTrue(group_items[8].is_duplicate) - - self.assertEqual("Heartbeat: Tor's uptime is 6:00 hours, with 0 circuits open. I've sent 539 kB and received 4.25 MB.", group_items[10].message) - self.assertEqual(2, len(group_items[10].duplicates)) - self.assertTrue(group_items[10].is_duplicate) diff --git a/test/util/log/read_tor_log.py b/test/util/log/read_tor_log.py deleted file mode 100644 index 60e3762..0000000 --- a/test/util/log/read_tor_log.py +++ /dev/null @@ -1,59 +0,0 @@ -import os -import unittest - -from nyx.util.log import read_tor_log - - -def data_path(filename): - return os.path.join(os.path.dirname(__file__), 'data', filename) - - -class TestReadTorLog(unittest.TestCase): - def test_general_log(self): - entries = list(read_tor_log(data_path('tor_log'))) - self.assertEqual(21, len(entries)) - - self.assertEqual('Interrupt: exiting cleanly.', entries[0].message) - self.assertEqual('Tor 0.2.7.0-alpha-dev (git-4247ce99e5d9b7b2) opening new log file.', entries[-1].message) - - def test_with_multiple_tor_instances(self): - entries = list(read_tor_log(data_path('multiple_tor_instances'))) - self.assertEqual(12, len(entries)) - - self.assertEqual('parse_dir_authority_line(): Trusted 100 dirserver at 128.31.0.39:9131 (9695)', entries[0].message) - self.assertEqual('tor_lockfile_lock(): Locking "/home/atagar/.tor/lock"', entries[1].message) - self.assertEqual('Tor 0.2.7.0-alpha-dev (git-4247ce99e5d9b7b2) opening log file.', entries[-1].message) - - def test_with_read_limit(self): - entries = list(read_tor_log(data_path('tor_log'), 5)) - self.assertEqual(5, len(entries)) - self.assertEqual('Interrupt: exiting cleanly.', entries[0].message) - self.assertEqual('Bootstrapped 90%: Establishing a Tor circuit', entries[-1].message) - - def test_with_empty_file(self): - entries = list(read_tor_log(data_path('empty_file'))) - self.assertEqual(0, len(entries)) - - def test_with_missing_path(self): - self.assertRaises(IOError, list, read_tor_log(data_path('no_such_path'))) - - def test_with_malformed_line(self): - try: - list(read_tor_log(data_path('malformed_line'))) - self.fail("Malformed content should've raised a ValueError") - except ValueError as exc: - self.assertTrue("has a line that doesn't match the format we expect: Apr 06 11:03:53.000" in str(exc)) - - def test_with_malformed_runlevel(self): - try: - list(read_tor_log(data_path('malformed_runlevel'))) - self.fail("Malformed content should've raised a ValueError") - except ValueError as exc: - self.assertTrue('has an unrecognized runlevel: [unrecognized]' in str(exc)) - - def test_with_malformed_date(self): - try: - list(read_tor_log(data_path('malformed_date'))) - self.fail("Malformed content should've raised a ValueError") - except ValueError as exc: - self.assertTrue("has a timestamp we don't recognize: Zed 06 11:03:52.000" in str(exc)) diff --git a/test/util/tracker/__init__.py b/test/util/tracker/__init__.py deleted file mode 100644 index 1b182e4..0000000 --- a/test/util/tracker/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -""" -Unit tests for nyx's tracker utilities. -""" - -__all__ = [ - 'connection_tracker', - 'daemon', - 'port_usage_tracker', - 'resource_tracker', -] diff --git a/test/util/tracker/connection_tracker.py b/test/util/tracker/connection_tracker.py deleted file mode 100644 index 52fed10..0000000 --- a/test/util/tracker/connection_tracker.py +++ /dev/null @@ -1,115 +0,0 @@ -import time -import unittest - -from nyx.util.tracker import ConnectionTracker - -from stem.util import connection - -from mock import Mock, patch - -STEM_CONNECTIONS = [ - connection.Connection('127.0.0.1', 3531, '75.119.206.243', 22, 'tcp', False), - connection.Connection('127.0.0.1', 1766, '86.59.30.40', 443, 'tcp', False), - connection.Connection('127.0.0.1', 1059, '74.125.28.106', 80, 'tcp', False) -] - - -class TestConnectionTracker(unittest.TestCase): - @patch('nyx.util.tracker.tor_controller') - @patch('nyx.util.tracker.connection.get_connections') - @patch('nyx.util.tracker.system', Mock(return_value = Mock())) - @patch('nyx.util.tracker.connection.system_resolvers', Mock(return_value = [connection.Resolver.NETSTAT])) - def test_fetching_connections(self, get_value_mock, tor_controller_mock): - tor_controller_mock().get_pid.return_value = 12345 - tor_controller_mock().get_conf.return_value = '0' - get_value_mock.return_value = STEM_CONNECTIONS - - with ConnectionTracker(0.04) as daemon: - time.sleep(0.01) - - connections = daemon.get_value() - - self.assertEqual(1, daemon.run_counter()) - self.assertEqual([conn.remote_address for conn in STEM_CONNECTIONS], [conn.remote_address for conn in connections]) - - get_value_mock.return_value = [] # no connection results - time.sleep(0.05) - connections = daemon.get_value() - - self.assertEqual(2, daemon.run_counter()) - self.assertEqual([], connections) - - @patch('nyx.util.tracker.tor_controller') - @patch('nyx.util.tracker.connection.get_connections') - @patch('nyx.util.tracker.system', Mock(return_value = Mock())) - @patch('nyx.util.tracker.connection.system_resolvers', Mock(return_value = [connection.Resolver.NETSTAT, connection.Resolver.LSOF])) - def test_resolver_failover(self, get_value_mock, tor_controller_mock): - tor_controller_mock().get_pid.return_value = 12345 - tor_controller_mock().get_conf.return_value = '0' - get_value_mock.side_effect = IOError() - - with ConnectionTracker(0.01) as daemon: - time.sleep(0.03) - - self.assertEqual([connection.Resolver.NETSTAT, connection.Resolver.LSOF], daemon._resolvers) - self.assertEqual([], daemon.get_value()) - - time.sleep(0.05) - - self.assertEqual([connection.Resolver.LSOF], daemon._resolvers) - self.assertEqual([], daemon.get_value()) - - time.sleep(0.05) - - self.assertEqual([], daemon._resolvers) - self.assertEqual([], daemon.get_value()) - - # Now make connection resolution work. We still shouldn't provide any - # results since we stopped looking. - - get_value_mock.return_value = STEM_CONNECTIONS[:2] - get_value_mock.side_effect = None - time.sleep(0.05) - self.assertEqual([], daemon.get_value()) - - # Finally, select a custom resolver. This should cause us to query again - # reguardless of our prior failures. - - daemon.set_custom_resolver(connection.Resolver.NETSTAT) - time.sleep(0.05) - self.assertEqual([conn.remote_address for conn in STEM_CONNECTIONS[:2]], [conn.remote_address for conn in daemon.get_value()]) - - @patch('nyx.util.tracker.tor_controller') - @patch('nyx.util.tracker.connection.get_connections') - @patch('nyx.util.tracker.system', Mock(return_value = Mock())) - @patch('nyx.util.tracker.connection.system_resolvers', Mock(return_value = [connection.Resolver.NETSTAT])) - def test_tracking_uptime(self, get_value_mock, tor_controller_mock): - tor_controller_mock().get_pid.return_value = 12345 - tor_controller_mock().get_conf.return_value = '0' - get_value_mock.return_value = [STEM_CONNECTIONS[0]] - first_start_time = time.time() - - with ConnectionTracker(0.04) as daemon: - time.sleep(0.01) - - connections = daemon.get_value() - self.assertEqual(1, len(connections)) - - self.assertEqual(STEM_CONNECTIONS[0].remote_address, connections[0].remote_address) - self.assertTrue(first_start_time < connections[0].start_time < time.time()) - self.assertTrue(connections[0].is_legacy) - - second_start_time = time.time() - get_value_mock.return_value = STEM_CONNECTIONS[:2] - time.sleep(0.05) - - connections = daemon.get_value() - self.assertEqual(2, len(connections)) - - self.assertEqual(STEM_CONNECTIONS[0].remote_address, connections[0].remote_address) - self.assertTrue(first_start_time < connections[0].start_time < time.time()) - self.assertTrue(connections[0].is_legacy) - - self.assertEqual(STEM_CONNECTIONS[1].remote_address, connections[1].remote_address) - self.assertTrue(second_start_time < connections[1].start_time < time.time()) - self.assertFalse(connections[1].is_legacy) diff --git a/test/util/tracker/daemon.py b/test/util/tracker/daemon.py deleted file mode 100644 index 4894f6a..0000000 --- a/test/util/tracker/daemon.py +++ /dev/null @@ -1,76 +0,0 @@ -import time -import unittest - -from nyx.util.tracker import Daemon - -from mock import Mock, patch - - -class TestDaemon(unittest.TestCase): - @patch('nyx.util.tracker.tor_controller') - @patch('nyx.util.tracker.system') - def test_init(self, system_mock, tor_controller_mock): - # Check that we register ourselves to listen for status changes, and - # properly retrieve the process' pid and name. - - tor_controller_mock().get_pid.return_value = 12345 - system_mock.name_by_pid.return_value = 'local_tor' - - daemon = Daemon(0.05) - - self.assertEqual(0.05, daemon.get_rate()) - self.assertEqual(12345, daemon._process_pid) - self.assertEqual('local_tor', daemon._process_name) - - tor_controller_mock().add_status_listener.assert_called_with(daemon._tor_status_listener) - system_mock.name_by_pid.assert_called_with(12345) - - @patch('nyx.util.tracker.tor_controller') - @patch('nyx.util.tracker.system') - def test_init_without_name(self, system_mock, tor_controller_mock): - # Check when we default to 'tor' if unable to determine the process' name. - - tor_controller_mock().get_pid.return_value = 12345 - system_mock.name_by_pid.return_value = None - - daemon = Daemon(0.05) - self.assertEqual('tor', daemon._process_name) - - @patch('nyx.util.tracker.tor_controller') - @patch('nyx.util.tracker.system') - def test_init_without_pid(self, system_mock, tor_controller_mock): - # Check when we can't determine tor's pid. - - tor_controller_mock().get_pid.return_value = None - - daemon = Daemon(0.05) - self.assertEqual(None, daemon._process_pid) - self.assertEqual('tor', daemon._process_name) - self.assertEqual(0, system_mock.call_count) - - @patch('nyx.util.tracker.tor_controller', Mock(return_value = Mock())) - @patch('nyx.util.tracker.system', Mock(return_value = Mock())) - def test_daemon_calls_task(self): - # Check that our Daemon calls the task method at the given rate. - - with Daemon(0.01) as daemon: - time.sleep(0.05) - self.assertTrue(2 < daemon.run_counter()) - - @patch('nyx.util.tracker.tor_controller', Mock(return_value = Mock())) - @patch('nyx.util.tracker.system', Mock(return_value = Mock())) - def test_pausing_daemon(self): - # Check that we can pause and unpause daemon. - - with Daemon(0.01) as daemon: - time.sleep(0.2) - self.assertTrue(2 < daemon.run_counter()) - - daemon.set_paused(True) - daemon._run_counter = 0 - time.sleep(0.05) - self.assertEqual(0, daemon.run_counter()) - - daemon.set_paused(False) - time.sleep(0.2) - self.assertTrue(2 < daemon.run_counter()) diff --git a/test/util/tracker/port_usage_tracker.py b/test/util/tracker/port_usage_tracker.py deleted file mode 100644 index 31bdd4c..0000000 --- a/test/util/tracker/port_usage_tracker.py +++ /dev/null @@ -1,117 +0,0 @@ -import time -import unittest - -from nyx.util.tracker import Process, PortUsageTracker, _process_for_ports - -from mock import Mock, patch - -LSOF_OUTPUT = """\ -COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME -tor 2001 atagar 14u IPv4 14048 0t0 TCP localhost:9051->localhost:37277 (ESTABLISHED) -tor 2001 atagar 15u IPv4 22024 0t0 TCP localhost:9051->localhost:51849 (ESTABLISHED) -python 2462 atagar 3u IPv4 14047 0t0 TCP localhost:37277->localhost:9051 (ESTABLISHED) -python 3444 atagar 3u IPv4 22023 0t0 TCP localhost:51849->localhost:9051 (ESTABLISHED) -""" - -BAD_LSOF_OUTPUT_NO_ENTRY = """\ -COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME -""" - -BAD_LSOF_OUTPUT_NOT_ESTABLISHED = """\ -COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME -tor 2001 atagar 14u IPv4 14048 0t0 TCP localhost:9051->localhost:37277 (CLOSE_WAIT) -""" - -BAD_LSOF_OUTPUT_MISSING_FIELD = """\ -COMMAND PID USER TYPE DEVICE SIZE/OFF NODE NAME -tor 2001 atagar IPv4 14048 0t0 TCP localhost:9051->localhost:37277 (ESTABLISHED) -""" - -BAD_LSOF_OUTPUT_UNRECOGNIZED_MAPPING = """\ -COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME -tor 2001 atagar 14u IPv4 14048 0t0 TCP localhost:9051=>localhost:37277 (ESTABLISHED) -""" - -BAD_LSOF_OUTPUT_NO_ADDRESS = """\ -COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME -tor 2001 atagar 14u IPv4 14048 0t0 TCP 9051->localhost:37277 (ESTABLISHED) -""" - -BAD_LSOF_OUTPUT_INVALID_PORT = """\ -COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME -tor 2001 atagar 14u IPv4 14048 0t0 TCP localhost:9037351->localhost:37277 (ESTABLISHED) -""" - - -class TestPortUsageTracker(unittest.TestCase): - @patch('nyx.util.tracker.system.call', Mock(return_value = LSOF_OUTPUT.split('\n'))) - def test_process_for_ports(self): - self.assertEqual({}, _process_for_ports([], [])) - self.assertEqual({80: None, 443: None}, _process_for_ports([80, 443], [])) - self.assertEqual({80: None, 443: None}, _process_for_ports([], [80, 443])) - - self.assertEqual({37277: Process(2462, 'python'), 51849: Process(2001, 'tor')}, _process_for_ports([37277], [51849])) - - @patch('nyx.util.tracker.system.call') - def test_process_for_ports_malformed(self, call_mock): - # Issues that are valid, but should result in us not having any content. - - test_inputs = ( - BAD_LSOF_OUTPUT_NO_ENTRY, - BAD_LSOF_OUTPUT_NOT_ESTABLISHED, - ) - - for test_input in test_inputs: - call_mock.return_value = test_input.split('\n') - self.assertEqual({80: None, 443: None}, _process_for_ports([80], [443])) - - # Isuses that are reported as errors. - - call_mock.return_value = [] - self.assertRaises(IOError, _process_for_ports, [80], [443]) - - test_inputs = ( - BAD_LSOF_OUTPUT_MISSING_FIELD, - BAD_LSOF_OUTPUT_UNRECOGNIZED_MAPPING, - BAD_LSOF_OUTPUT_NO_ADDRESS, - BAD_LSOF_OUTPUT_INVALID_PORT, - ) - - for test_input in test_inputs: - call_mock.return_value = test_input.split('\n') - self.assertRaises(IOError, _process_for_ports, [80], [443]) - - @patch('nyx.util.tracker.tor_controller') - @patch('nyx.util.tracker._process_for_ports') - @patch('nyx.util.tracker.system', Mock(return_value = Mock())) - def test_fetching_samplings(self, process_for_ports_mock, tor_controller_mock): - tor_controller_mock().get_pid.return_value = 12345 - process_for_ports_mock.return_value = {37277: 'python', 51849: 'tor'} - - with PortUsageTracker(0.02) as daemon: - time.sleep(0.01) - - self.assertEqual({}, daemon.query([37277, 51849], [])) - time.sleep(0.04) - - self.assertEqual({37277: 'python', 51849: 'tor'}, daemon.query([37277, 51849], [])) - - @patch('nyx.util.tracker.tor_controller') - @patch('nyx.util.tracker._process_for_ports') - @patch('nyx.util.tracker.system', Mock(return_value = Mock())) - def test_resolver_failover(self, process_for_ports_mock, tor_controller_mock): - tor_controller_mock().get_pid.return_value = 12345 - process_for_ports_mock.side_effect = IOError() - - with PortUsageTracker(0.01) as daemon: - # We shouldn't attempt lookups (nor encounter failures) without ports to - # query. - - time.sleep(0.05) - self.assertEqual(0, daemon._failure_count) - - daemon.query([37277, 51849], []) - time.sleep(0.03) - self.assertTrue(daemon.is_alive()) - time.sleep(0.1) - self.assertFalse(daemon.is_alive()) diff --git a/test/util/tracker/resource_tracker.py b/test/util/tracker/resource_tracker.py deleted file mode 100644 index affe01e..0000000 --- a/test/util/tracker/resource_tracker.py +++ /dev/null @@ -1,143 +0,0 @@ -import time -import unittest - -from nyx.util.tracker import ResourceTracker, _resources_via_ps, _resources_via_proc - -from mock import Mock, patch - -PS_OUTPUT = """\ - TIME ELAPSED RSS %MEM -00:00:02 00:18 18848 0.4 -""" - - -class TestResourceTracker(unittest.TestCase): - @patch('nyx.util.tracker.tor_controller') - @patch('nyx.util.tracker._resources_via_proc') - @patch('nyx.util.tracker.system', Mock(return_value = Mock())) - @patch('nyx.util.tracker.proc.is_available', Mock(return_value = True)) - def test_fetching_samplings(self, resources_via_proc_mock, tor_controller_mock): - tor_controller_mock().get_pid.return_value = 12345 - resources_via_proc_mock.return_value = (105.3, 2.4, 8072, 0.3) - - with ResourceTracker(0.04) as daemon: - time.sleep(0.01) - - resources = daemon.get_value() - - self.assertEqual(1, daemon.run_counter()) - self.assertEqual(0.0, resources.cpu_sample) - self.assertEqual(43.875, resources.cpu_average) - self.assertEqual(105.3, resources.cpu_total) - self.assertEqual(8072, resources.memory_bytes) - self.assertEqual(0.3, resources.memory_percent) - self.assertTrue((time.time() - resources.timestamp) < 0.5) - - resources_via_proc_mock.return_value = (800.3, 3.2, 6020, 0.26) - time.sleep(0.05) - resources = daemon.get_value() - - self.assertEqual(2, daemon.run_counter()) - self.assertEqual(6.600189933523267, resources.cpu_sample) - self.assertEqual(250.09374999999997, resources.cpu_average) - self.assertEqual(800.3, resources.cpu_total) - self.assertEqual(6020, resources.memory_bytes) - self.assertEqual(0.26, resources.memory_percent) - self.assertTrue((time.time() - resources.timestamp) < 0.5) - - resources_via_proc_mock.assert_called_with(12345) - - @patch('nyx.util.tracker.tor_controller') - @patch('nyx.util.tracker.proc.is_available') - @patch('nyx.util.tracker._resources_via_ps', Mock(return_value = (105.3, 2.4, 8072, 0.3))) - @patch('nyx.util.tracker._resources_via_proc', Mock(return_value = (340.3, 3.2, 6020, 0.26))) - @patch('nyx.util.tracker.system', Mock(return_value = Mock())) - def test_picking_proc_or_ps(self, is_proc_available_mock, tor_controller_mock): - tor_controller_mock().get_pid.return_value = 12345 - - is_proc_available_mock.return_value = True - - with ResourceTracker(0.04) as daemon: - time.sleep(0.01) - - resources = daemon.get_value() - - self.assertEqual(1, daemon.run_counter()) - self.assertEqual(0.0, resources.cpu_sample) - self.assertEqual(106.34375, resources.cpu_average) - self.assertEqual(340.3, resources.cpu_total) - self.assertEqual(6020, resources.memory_bytes) - self.assertEqual(0.26, resources.memory_percent) - self.assertTrue((time.time() - resources.timestamp) < 0.5) - - is_proc_available_mock.return_value = False - - with ResourceTracker(0.04) as daemon: - time.sleep(0.01) - - resources = daemon.get_value() - - self.assertEqual(1, daemon.run_counter()) - self.assertEqual(0.0, resources.cpu_sample) - self.assertEqual(43.875, resources.cpu_average) - self.assertEqual(105.3, resources.cpu_total) - self.assertEqual(8072, resources.memory_bytes) - self.assertEqual(0.3, resources.memory_percent) - self.assertTrue((time.time() - resources.timestamp) < 0.5) - - @patch('nyx.util.tracker.tor_controller') - @patch('nyx.util.tracker._resources_via_ps', Mock(return_value = (105.3, 2.4, 8072, 0.3))) - @patch('nyx.util.tracker._resources_via_proc', Mock(side_effect = IOError())) - @patch('nyx.util.tracker.system', Mock(return_value = Mock())) - @patch('nyx.util.tracker.proc.is_available', Mock(return_value = True)) - def test_failing_over_to_ps(self, tor_controller_mock): - tor_controller_mock().get_pid.return_value = 12345 - - with ResourceTracker(0.01) as daemon: - time.sleep(0.03) - - self.assertEqual(True, daemon._use_proc) - resources = daemon.get_value() - - self.assertEqual(0, daemon.run_counter()) - self.assertEqual(0.0, resources.cpu_sample) - self.assertEqual(0.0, resources.cpu_average) - self.assertEqual(0, resources.cpu_total) - self.assertEqual(0, resources.memory_bytes) - self.assertEqual(0.0, resources.memory_percent) - self.assertEqual(0.0, resources.timestamp) - - while daemon.run_counter() < 1: - time.sleep(0.01) - - self.assertEqual(False, daemon._use_proc) - - resources = daemon.get_value() - - self.assertEqual(0.0, resources.cpu_sample) - self.assertEqual(43.875, resources.cpu_average) - self.assertEqual(105.3, resources.cpu_total) - self.assertEqual(8072, resources.memory_bytes) - self.assertEqual(0.3, resources.memory_percent) - self.assertTrue((time.time() - resources.timestamp) < 0.5) - - @patch('nyx.util.tracker.system.call', Mock(return_value = PS_OUTPUT.split('\n'))) - def test_resources_via_ps(self): - total_cpu_time, uptime, memory_in_bytes, memory_in_percent = _resources_via_ps(12345) - - self.assertEqual(2.0, total_cpu_time) - self.assertEqual(18, uptime) - self.assertEqual(19300352, memory_in_bytes) - self.assertEqual(0.004, memory_in_percent) - - @patch('time.time', Mock(return_value = 1388967218.973117)) - @patch('nyx.util.tracker.proc.stats', Mock(return_value = (1.5, 0.5, 1388967200.9))) - @patch('nyx.util.tracker.proc.memory_usage', Mock(return_value = (19300352, 6432))) - @patch('nyx.util.tracker.proc.physical_memory', Mock(return_value = 4825088000)) - def test_resources_via_proc(self): - total_cpu_time, uptime, memory_in_bytes, memory_in_percent = _resources_via_proc(12345) - - self.assertEqual(2.0, total_cpu_time) - self.assertEqual(18, int(uptime)) - self.assertEqual(19300352, memory_in_bytes) - self.assertEqual(0.004, memory_in_percent)

1 0