May 2015 - tor-commits - lists.torproject.org

[nyx/master] Rewrite log panel's draw function
by atagar＠torproject.org 05 May '15

05 May '15

commit 9f4329ea439b815e8b220e1e851699f0b53fcbbe Author: Damian Johnson <atagar(a)torproject.org> Date: Mon May 4 21:31:59 2015 -0700 Rewrite log panel's draw function --- nyx/log_panel.py | 117 +++++++++++++++++--------------------------------- nyx/util/ui_tools.py | 25 +++++++---- 2 files changed, 57 insertions(+), 85 deletions(-) diff --git a/nyx/log_panel.py b/nyx/log_panel.py index 4211e09..9685c77 100644 --- a/nyx/log_panel.py +++ b/nyx/log_panel.py @@ -300,106 +300,53 @@ class LogPanel(panel.Panel, threading.Thread): def draw(self, width, height): with self._lock: event_log = list(self.get_attr('_logged_events')) - - # draws the top label - - if self.is_title_visible(): - title_comp = list(nyx.util.log.condense_runlevels(*self._logged_event_types)) - - if self._filter.selection(): - title_comp.append('filter: %s' % self._filter.selection()) - - title_comp_str = join(title_comp, ', ', width - 10) - title = 'Events (%s):' % title_comp_str if title_comp_str else 'Events:' - - self.addstr(0, 0, title, curses.A_STANDOUT) - - # restricts scroll location to valid bounds - self._scroll = max(0, min(self._scroll, self._last_content_height - height + 1)) - # draws left-hand scroll bar if content's longer than the height - - msg_indent, divider_indent = 1, 0 # offsets for scroll bar is_scroll_bar_visible = self._last_content_height > height - 1 if is_scroll_bar_visible: - msg_indent, divider_indent = 3, 2 self.add_scroll_bar(self._scroll, self._scroll + height - 1, self._last_content_height, 1) - # draws log entries - - line_count = 1 - self._scroll - seen_first_date_divider = False - divider_attr = (curses.A_BOLD, 'yellow') + x, y = 3 if is_scroll_bar_visible else 1, 1 - self._scroll - # determines if we have the minimum width to show date dividers + # group entries by date, filtering out those that aren't visible - show_daybreaks = width - divider_indent >= 3 - last_day = event_log[0].days_since() if event_log else 0 - - for i in range(len(event_log)): - entry = event_log[i] - is_last = i == len(event_log) - 1 + days_ago_to_entries = {} + for entry in event_log: if entry.is_duplicate and not CONFIG['features.log.showDuplicateEntries']: continue # deduplicated message elif not self._filter.match(entry.display_message): continue # filter doesn't match log message - # checks if we should be showing a divider with the date - - if last_day != entry.days_since(): - # bottom of the divider - - if seen_first_date_divider: - if line_count >= 1 and line_count < height and show_daybreaks: - self.addch(line_count, divider_indent, curses.ACS_LLCORNER, *divider_attr) - self.hline(line_count, divider_indent + 1, width - divider_indent - 2, *divider_attr) - self.addch(line_count, width - 1, curses.ACS_LRCORNER, *divider_attr) + days_ago_to_entries.setdefault(entry.days_since(), []).append(entry) - line_count += 1 + for days_ago in sorted(days_ago_to_entries.keys()): + if days_ago == 0: + for entry in days_ago_to_entries[days_ago]: + y = self._draw_entry(x, y, width, entry) + else: + original_y, y = y, y + 1 - # top of the divider + for entry in days_ago_to_entries[days_ago]: + y = self._draw_entry(x, y, width, entry) - if line_count >= 1 and line_count < height and show_daybreaks: - time_label = time.strftime(' %B %d, %Y ', time.localtime(entry.timestamp)) - self.addch(line_count, divider_indent, curses.ACS_ULCORNER, *divider_attr) - self.addch(line_count, divider_indent + 1, curses.ACS_HLINE, *divider_attr) - self.addstr(line_count, divider_indent + 2, time_label, curses.A_BOLD, *divider_attr) + ui_tools.draw_box(self, original_y, x - 1, width - x + 1, y - original_y + 1, curses.A_BOLD, 'yellow') + time_label = time.strftime(' %B %d, %Y ', time.localtime(days_ago_to_entries[days_ago][0].timestamp)) + self.addstr(original_y, x + 1, time_label, curses.A_BOLD, curses.A_BOLD, 'yellow') - line_length = width - divider_indent - len(time_label) - 3 - self.hline(line_count, divider_indent + len(time_label) + 2, line_length, *divider_attr) - self.addch(line_count, divider_indent + len(time_label) + 2 + line_length, curses.ACS_URCORNER, *divider_attr) + y += 1 - seen_first_date_divider = True - line_count += 1 + # drawing the title after the content, so we'll clear content from the top line - line_count_start = line_count - line_count = self._draw_entry(msg_indent, line_count, width, entry, height) - - for y in range(line_count_start, line_count): - if seen_first_date_divider and width - divider_indent >= 3 and show_daybreaks: - self.addch(y, divider_indent, curses.ACS_VLINE, *divider_attr) - self.addch(y, width - 1, curses.ACS_VLINE, *divider_attr) - - # if this is the last line and there's room, then draw the bottom of the divider - - if is_last and seen_first_date_divider: - if line_count < height and show_daybreaks: - self.addch(line_count, divider_indent, curses.ACS_LLCORNER, *divider_attr) - self.hline(line_count, divider_indent + 1, width - divider_indent - 2, *divider_attr) - self.addch(line_count, width - 1, curses.ACS_LRCORNER, *divider_attr) - - line_count += 1 - - last_day = entry.days_since() + if self.is_title_visible(): + self._draw_title(width) # redraw the display if... # - last_content_height was off by too much # - we're off the bottom of the page - new_content_height = line_count + self._scroll - 1 + new_content_height = y + self._scroll - 1 content_height_delta = abs(self._last_content_height - new_content_height) force_redraw, force_redraw_reason = True, '' @@ -421,7 +368,23 @@ class LogPanel(panel.Panel, threading.Thread): log.debug('redrawing the log panel with the corrected content height (%s)' % force_redraw_reason) self.redraw(True) - def _draw_entry(self, x, y, width, entry, height): + def _draw_title(self, width): + """ + Panel title with the event types we're logging and our regex filter if set. + """ + + self.addstr(0, 0, ' ' * width) # clear line + title_comp = list(nyx.util.log.condense_runlevels(*self._logged_event_types)) + + if self._filter.selection(): + title_comp.append('filter: %s' % self._filter.selection()) + + title_comp_str = join(title_comp, ', ', width - 10) + title = 'Events (%s):' % title_comp_str if title_comp_str else 'Events:' + + self.addstr(0, 0, title, curses.A_STANDOUT) + + def _draw_entry(self, x, y, width, entry): """ Presents a log entry with line wrapping. """ @@ -429,7 +392,7 @@ class LogPanel(panel.Panel, threading.Thread): def draw_line(x, y, width, msg, *attr): msg, remaining_lines = msg.split('\n', 1) if ('\n' in msg) else (msg, '') msg, cropped = str_tools.crop(msg, width - x - 1, min_crop = 4, ending = str_tools.Ending.HYPHEN, get_remainder = True) - x = self.addstr(y, x, msg, *attr) if y > 0 else 0 # draw unless it would cover the title + x = self.addstr(y, x, msg, *attr) return x, (cropped + '\n' + remaining_lines).strip() def draw_msg(min_x, x, y, width, msg, *attr): @@ -439,7 +402,7 @@ class LogPanel(panel.Panel, threading.Thread): x, msg = draw_line(x, y, width, msg, *attr) if (y - orig_y + 1) >= CONFIG['features.log.max_lines_per_entry']: - break + break # filled up the maximum number of lines we're allowing for if msg: msg = ' ' + msg # indent the next line diff --git a/nyx/util/ui_tools.py b/nyx/util/ui_tools.py index 8d7edad..901676b 100644 --- a/nyx/util/ui_tools.py +++ b/nyx/util/ui_tools.py @@ -179,7 +179,7 @@ def get_printable(line, keep_newlines = True): return line -def draw_box(panel, top, left, width, height, attr=curses.A_NORMAL): +def draw_box(panel, top, left, width, height, *attributes): """ Draws a box in the panel with the given bounds. @@ -192,21 +192,30 @@ def draw_box(panel, top, left, width, height, attr=curses.A_NORMAL): attr - text attributes """ + format_attr = curses.A_NORMAL + + for attr in attributes: + if isinstance(attr, str): + format_attr |= get_color(attr) + else: + format_attr |= attr + # draws the top and bottom - panel.hline(top, left + 1, width - 2, attr) - panel.hline(top + height - 1, left + 1, width - 2, attr) + panel.hline(top, left + 1, width - 2, format_attr) + panel.hline(top + height - 1, left + 1, width - 2, format_attr) # draws the left and right sides - panel.vline(top + 1, left, height - 2, attr) - panel.vline(top + 1, left + width - 1, height - 2, attr) + panel.vline(top + 1, left, height - 2, format_attr) + panel.vline(top + 1, left + width - 1, height - 2, format_attr) # draws the corners - panel.addch(top, left, curses.ACS_ULCORNER, attr) - panel.addch(top, left + width - 1, curses.ACS_URCORNER, attr) - panel.addch(top + height - 1, left, curses.ACS_LLCORNER, attr) + panel.addch(top, left, curses.ACS_ULCORNER, format_attr) + panel.addch(top, left + width - 1, curses.ACS_URCORNER, format_attr) + panel.addch(top + height - 1, left, curses.ACS_LLCORNER, format_attr) + panel.addch(top + height - 1, left + width - 1, curses.ACS_LRCORNER, format_attr) def get_scroll_position(key, position, page_height, content_height, is_cursor = False):

1 0

[nyx/master] Drop deduplicated_log attribute
by atagar＠torproject.org 05 May '15

05 May '15

commit 008ab93d05eb005988066aea4381c27e817814ea Author: Damian Johnson <atagar(a)torproject.org> Date: Sun May 3 08:46:59 2015 -0700 Drop deduplicated_log attribute Easy bit of our draw() funtion to simplify. --- nyx/log_panel.py | 90 +++++++++++++++++++++++------------------------------- 1 file changed, 39 insertions(+), 51 deletions(-) diff --git a/nyx/log_panel.py b/nyx/log_panel.py index 590df6e..03f4478 100644 --- a/nyx/log_panel.py +++ b/nyx/log_panel.py @@ -84,8 +84,8 @@ class LogPanel(panel.Panel, threading.Thread): self._last_update = -1 # time the content was last revised self._halt = False # terminates thread if true - self._cond = threading.Condition() # used for pausing/resuming the thread - self.vals_lock = threading.RLock() + self._pause_condition = threading.Condition() + self._lock = threading.RLock() # cached parameters (invalidated if arguments for them change) # last set of events we've drawn with @@ -177,7 +177,7 @@ class LogPanel(panel.Panel, threading.Thread): try: if event_types != self._logged_event_types: - with self.vals_lock: + with self._lock: self._logged_event_types = nyx.util.log.listen_for_events(self._register_tor_event, event_types) self.redraw(True) except ValueError as exc: @@ -204,7 +204,7 @@ class LogPanel(panel.Panel, threading.Thread): Clears the contents of the event log. """ - with self.vals_lock: + with self._lock: self._logged_events = nyx.util.log.LogGroup(CONFIG['cache.log_panel.size'], group_by_day = CONFIG['features.log.showDateDividers']) self.redraw(True) @@ -230,17 +230,16 @@ class LogPanel(panel.Panel, threading.Thread): except OSError as exc: raise IOError("unable to make directory '%s'" % base_dir) - snapshot_file = open(path, 'w') - - with self.vals_lock: - try: - for entry in reversed(self._logged_events): - is_visible = self._filter.match(entry.display_message) + with self._lock: + with open(path, 'w') as snapshot_file: + try: + for entry in reversed(self._logged_events): + is_visible = self._filter.match(entry.display_message) - if is_visible: - snapshot_file.write(entry.display_message + '\n') - except Exception as exc: - raise exc + if is_visible: + snapshot_file.write(entry.display_message + '\n') + except Exception as exc: + raise IOError("unable to write to '%s': %s" % (path, exc)) def handle_key(self, key): if key.is_scroll(): @@ -248,11 +247,11 @@ class LogPanel(panel.Panel, threading.Thread): new_scroll = ui_tools.get_scroll_position(key, self.scroll, page_height, self.last_content_height) if self.scroll != new_scroll: - with self.vals_lock: + with self._lock: self.scroll = new_scroll self.redraw(True) elif key.match('u'): - with self.vals_lock: + with self._lock: self.set_duplicate_visability(not CONFIG['features.log.showDuplicateEntries']) self.redraw(True) elif key.match('c'): @@ -304,27 +303,22 @@ class LogPanel(panel.Panel, threading.Thread): ] def draw(self, width, height): - """ - Redraws message log. Entries stretch to use available space and may - contain up to two lines. Starts with newest entries. - """ - event_log = self.get_attr('_logged_events') - with self.vals_lock: + with self._lock: self._last_logged_events, self._last_update = event_log, time.time() event_log = list(event_log) # draws the top label if self.is_title_visible(): - comp = list(nyx.util.log.condense_runlevels(*self._logged_event_types)) + title_comp = list(nyx.util.log.condense_runlevels(*self._logged_event_types)) if self._filter.selection(): - comp.append('filter: %s' % self._filter.selection()) + title_comp.append('filter: %s' % self._filter.selection()) - comp_str = join(comp, ', ', width - 10) - title = 'Events (%s):' % comp_str if comp_str else 'Events:' + title_comp_str = join(title_comp, ', ', width - 10) + title = 'Events (%s):' % title_comp_str if title_comp_str else 'Events:' self.addstr(0, 0, title, curses.A_STANDOUT) @@ -347,27 +341,21 @@ class LogPanel(panel.Panel, threading.Thread): seen_first_date_divider = False divider_attr, duplicate_attr = (curses.A_BOLD, 'yellow'), (curses.A_BOLD, 'green') - # TODO: fix daybreak handling - # is_dates_shown = self.regex_filter is None and CONFIG['features.log.showDateDividers'] - # event_log = get_daybreaks(current_log, self.is_paused()) if is_dates_shown else current_log - - if not CONFIG['features.log.showDuplicateEntries']: - deduplicated_log = [] - - for entry in event_log: - if not entry.is_duplicate: - duplicate_count = len(entry.duplicates) if entry.duplicates else 0 - deduplicated_log.append((entry, duplicate_count)) - else: - deduplicated_log = [(entry, 0) for entry in event_log] - # determines if we have the minimum width to show date dividers show_daybreaks = width - divider_indent >= 3 - last_day = deduplicated_log[0][0].days_since() + last_day = event_log[0].days_since() if event_log else 0 + + for i in range(len(event_log)): + entry = event_log[i] + is_last = i == len(event_log) - 1 - while deduplicated_log: - entry, duplicate_count = deduplicated_log.pop(0) + if CONFIG['features.log.showDuplicateEntries']: + duplicate_count = 0 + elif entry.is_duplicate: + continue + else: + duplicate_count = len(entry.duplicates) if entry.duplicates else 0 if not self._filter.match(entry.display_message): continue # filter doesn't match log message - skip @@ -457,7 +445,7 @@ class LogPanel(panel.Panel, threading.Thread): # if this is the last line and there's room, then draw the bottom of the divider - if not deduplicated_log and seen_first_date_divider: + if is_last and seen_first_date_divider: if line_count < height and show_daybreaks: self.addch(line_count, divider_indent, curses.ACS_LLCORNER, *divider_attr) self.hline(line_count, divider_indent + 1, width - divider_indent - 2, *divider_attr) @@ -514,9 +502,9 @@ class LogPanel(panel.Panel, threading.Thread): sleep_time = max(0.05, max_log_update_rate - time_since_reset) if sleep_time: - with self._cond: + with self._pause_condition: if not self._halt: - self._cond.wait(sleep_time) + self._pause_condition.wait(sleep_time) else: last_day = current_day self.redraw(True) @@ -531,9 +519,9 @@ class LogPanel(panel.Panel, threading.Thread): Halts further resolutions and terminates the thread. """ - with self._cond: + with self._pause_condition: self._halt = True - self._cond.notifyAll() + self._pause_condition.notifyAll() def _register_tor_event(self, event): msg = ' '.join(str(event).split(' ')[1:]) @@ -555,12 +543,12 @@ class LogPanel(panel.Panel, threading.Thread): if event.type not in self._logged_event_types: return - with self.vals_lock: + with self._lock: self._logged_events.add(event) self._log_file.write(event.display_message) # notifies the display that it has new content if self._filter.match(event.display_message): - with self._cond: - self._cond.notifyAll() + with self._pause_condition: + self._pause_condition.notifyAll()

1 0

[nyx/master] Drop features.log.showDateDividers
by atagar＠torproject.org 05 May '15

05 May '15

commit de740bae89566eb4d436d2d8f902bed0ccafcd0e Author: Damian Johnson <atagar(a)torproject.org> Date: Mon May 4 21:35:01 2015 -0700 Drop features.log.showDateDividers Drop the ability to opt out of grouping by days. Never known this to be useful, and probably not worth accounting this in our new draw() function. Also fixing features.log.maxLinesPerEntry and features.logFile. They were documented as being camel case in our nyxrc.sample (like all options), but in the code it wasn't. --- nyx/log_panel.py | 15 +++++++-------- nyxrc.sample | 3 --- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/nyx/log_panel.py b/nyx/log_panel.py index 9685c77..300af83 100644 --- a/nyx/log_panel.py +++ b/nyx/log_panel.py @@ -22,7 +22,7 @@ from nyx.util import join, panel, tor_controller, ui_tools def conf_handler(key, value): - if key == 'features.log.max_lines_per_entry': + if key == 'features.log.maxLinesPerEntry': return max(1, value) elif key == 'features.log.prepopulateReadLimit': return max(0, value) @@ -33,10 +33,9 @@ def conf_handler(key, value): CONFIG = conf.config_dict('nyx', { - 'features.log_file': '', - 'features.log.showDateDividers': True, + 'features.logFile': '', 'features.log.showDuplicateEntries': False, - 'features.log.max_lines_per_entry': 6, + 'features.log.maxLinesPerEntry': 6, 'features.log.prepopulate': True, 'features.log.prepopulateReadLimit': 5000, 'features.log.maxRefreshRate': 300, @@ -73,8 +72,8 @@ class LogPanel(panel.Panel, threading.Thread): self.setDaemon(True) self._logged_event_types = nyx.util.log.listen_for_events(self._register_tor_event, logged_events) - self._logged_events = nyx.util.log.LogGroup(CONFIG['cache.log_panel.size'], group_by_day = CONFIG['features.log.showDateDividers']) - self._log_file = nyx.util.log.LogFileOutput(CONFIG['features.log_file']) + self._logged_events = nyx.util.log.LogGroup(CONFIG['cache.log_panel.size']) + self._log_file = nyx.util.log.LogFileOutput(CONFIG['features.logFile']) self._filter = nyx.util.log.LogFilters(initial_filters = CONFIG['features.log.regex']) self.set_pause_attr('_logged_events') @@ -200,7 +199,7 @@ class LogPanel(panel.Panel, threading.Thread): """ with self._lock: - self._logged_events = nyx.util.log.LogGroup(CONFIG['cache.log_panel.size'], group_by_day = CONFIG['features.log.showDateDividers']) + self._logged_events = nyx.util.log.LogGroup(CONFIG['cache.log_panel.size']) self.redraw(True) def save_snapshot(self, path): @@ -401,7 +400,7 @@ class LogPanel(panel.Panel, threading.Thread): while msg: x, msg = draw_line(x, y, width, msg, *attr) - if (y - orig_y + 1) >= CONFIG['features.log.max_lines_per_entry']: + if (y - orig_y + 1) >= CONFIG['features.log.maxLinesPerEntry']: break # filled up the maximum number of lines we're allowing for if msg: diff --git a/nyxrc.sample b/nyxrc.sample index fadc5df..44b0228 100644 --- a/nyxrc.sample +++ b/nyxrc.sample @@ -59,8 +59,6 @@ features.confirmQuit true # Paremters for the log panel # --------------------------- -# showDateDividers -# show borders with dates for entries from previous days # showDuplicateEntries # shows all log entries if true, otherwise collapses similar entries with an # indicator for how much is being hidden @@ -77,7 +75,6 @@ features.confirmQuit true # regex # preconfigured regular expression pattern, up to five will be loaded -features.log.showDateDividers true features.log.showDuplicateEntries false features.log.maxLinesPerEntry 6 features.log.prepopulate true

1 0

[nyx/master] Function for drawing a log message
by atagar＠torproject.org 05 May '15

05 May '15

commit eb090a9c2a9d985df4477136e682e89bc5905d04 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon May 4 13:28:58 2015 -0700 Function for drawing a log message Ick this draw function is a tricky, tangled mess! To be fair what it's trying to do isn't easy. Big chunk of this is rendering the content of a log message with line wrapping and deduplication. Rewritting this bit first in its own function. --- nyx/log_panel.py | 114 ++++++++++++++++++++++++------------------------------ 1 file changed, 51 insertions(+), 63 deletions(-) diff --git a/nyx/log_panel.py b/nyx/log_panel.py index 0a8e250..4211e09 100644 --- a/nyx/log_panel.py +++ b/nyx/log_panel.py @@ -331,7 +331,7 @@ class LogPanel(panel.Panel, threading.Thread): line_count = 1 - self._scroll seen_first_date_divider = False - divider_attr, duplicate_attr = (curses.A_BOLD, 'yellow'), (curses.A_BOLD, 'green') + divider_attr = (curses.A_BOLD, 'yellow') # determines if we have the minimum width to show date dividers @@ -342,15 +342,10 @@ class LogPanel(panel.Panel, threading.Thread): entry = event_log[i] is_last = i == len(event_log) - 1 - if CONFIG['features.log.showDuplicateEntries']: - duplicate_count = 0 - elif entry.is_duplicate: - continue - else: - duplicate_count = len(entry.duplicates) if entry.duplicates else 0 - - if not self._filter.match(entry.display_message): - continue # filter doesn't match log message - skip + if entry.is_duplicate and not CONFIG['features.log.showDuplicateEntries']: + continue # deduplicated message + elif not self._filter.match(entry.display_message): + continue # filter doesn't match log message # checks if we should be showing a divider with the date @@ -380,60 +375,13 @@ class LogPanel(panel.Panel, threading.Thread): seen_first_date_divider = True line_count += 1 - # entry contents to be displayed, tuples of the form: - # (msg, formatting, includeLinebreak) - - display_queue = [] - - msg_comp = entry.display_message.split('\n') - - for i in range(len(msg_comp)): - font = curses.A_BOLD if 'ERR' in entry.type else curses.A_NORMAL # emphasizes ERR messages - display_queue.append((msg_comp[i].strip(), (font, CONFIG['attr.log_color'].get(entry.type, 'white')), i != len(msg_comp) - 1)) - - if duplicate_count: - plural_label = 's' if duplicate_count > 1 else '' - duplicate_msg = ' [%i duplicate%s hidden]' % (duplicate_count, plural_label) - display_queue.append((duplicate_msg, duplicate_attr, False)) - - # TODO: a fix made line_offset unused, and probably broke max_entries_per_line... not sure if we care - - cursor_location, line_offset = msg_indent, 0 - max_entries_per_line = CONFIG['features.log.max_lines_per_entry'] - - while display_queue: - msg, format, include_break = display_queue.pop(0) - draw_line = line_count + line_offset - - if line_offset == max_entries_per_line: - break - - max_msg_size = width - cursor_location - 1 - - if len(msg) > max_msg_size: - # message is too long - break it up - if line_offset == max_entries_per_line - 1: - msg = str_tools.crop(msg, max_msg_size) - else: - msg, remainder = str_tools.crop(msg, max_msg_size, 4, 4, str_tools.Ending.HYPHEN, True) - display_queue.insert(0, (remainder.strip(), format, include_break)) - - include_break = True - - if draw_line < height and draw_line >= 1: - if seen_first_date_divider and width - divider_indent >= 3 and show_daybreaks: - self.addch(draw_line, divider_indent, curses.ACS_VLINE, *divider_attr) - self.addch(draw_line, width - 1, curses.ACS_VLINE, *divider_attr) - - self.addstr(draw_line, cursor_location, msg, *format) - - cursor_location += len(msg) + line_count_start = line_count + line_count = self._draw_entry(msg_indent, line_count, width, entry, height) - if include_break or not display_queue: - line_count += 1 - cursor_location = msg_indent + 2 # indent following lines - - line_count += line_offset + for y in range(line_count_start, line_count): + if seen_first_date_divider and width - divider_indent >= 3 and show_daybreaks: + self.addch(y, divider_indent, curses.ACS_VLINE, *divider_attr) + self.addch(y, width - 1, curses.ACS_VLINE, *divider_attr) # if this is the last line and there's room, then draw the bottom of the divider @@ -473,6 +421,46 @@ class LogPanel(panel.Panel, threading.Thread): log.debug('redrawing the log panel with the corrected content height (%s)' % force_redraw_reason) self.redraw(True) + def _draw_entry(self, x, y, width, entry, height): + """ + Presents a log entry with line wrapping. + """ + + def draw_line(x, y, width, msg, *attr): + msg, remaining_lines = msg.split('\n', 1) if ('\n' in msg) else (msg, '') + msg, cropped = str_tools.crop(msg, width - x - 1, min_crop = 4, ending = str_tools.Ending.HYPHEN, get_remainder = True) + x = self.addstr(y, x, msg, *attr) if y > 0 else 0 # draw unless it would cover the title + return x, (cropped + '\n' + remaining_lines).strip() + + def draw_msg(min_x, x, y, width, msg, *attr): + orig_y = y + + while msg: + x, msg = draw_line(x, y, width, msg, *attr) + + if (y - orig_y + 1) >= CONFIG['features.log.max_lines_per_entry']: + break + + if msg: + msg = ' ' + msg # indent the next line + x, y = min_x, y + 1 + + return x, y + + min_x, msg = x, entry.display_message + boldness = curses.A_BOLD if 'ERR' in entry.type else curses.A_NORMAL # emphasize ERR messages + color = CONFIG['attr.log_color'].get(entry.type, 'white') + + x, y = draw_msg(min_x, x, y, width, msg, boldness, color) + + if entry.duplicates and not CONFIG['features.log.showDuplicateEntries']: + duplicate_count = len(entry.duplicates) - 1 + plural = 's' if duplicate_count > 1 else '' + duplicate_msg = ' [%i duplicate%s hidden]' % (duplicate_count, plural) + x, y = draw_msg(min_x, x, y, width, duplicate_msg, curses.A_BOLD, 'green') + + return y + 1 + def run(self): """ Redraws the display, coalescing updates if events are rapidly logged (for

1 0

[nyx/master] Finish making log panel attributes private
by atagar＠torproject.org 05 May '15

05 May '15

commit e1795b755bbd3630f624c4c2448d7d78703ea61b Author: Damian Johnson <atagar(a)torproject.org> Date: Sun May 3 10:34:42 2015 -0700 Finish making log panel attributes private Been swapping them over as we rewrite sections, but now just renaming the rest. --- nyx/log_panel.py | 62 ++++++++++++++++++++++++------------------------------ 1 file changed, 27 insertions(+), 35 deletions(-) diff --git a/nyx/log_panel.py b/nyx/log_panel.py index 03f4478..0a8e250 100644 --- a/nyx/log_panel.py +++ b/nyx/log_panel.py @@ -79,18 +79,13 @@ class LogPanel(panel.Panel, threading.Thread): self.set_pause_attr('_logged_events') - self.last_content_height = 0 # height of the rendered content when last drawn - self.scroll = 0 + self._last_content_height = 0 # height of the rendered content when last drawn + self._scroll = 0 - self._last_update = -1 # time the content was last revised - self._halt = False # terminates thread if true + self._halt = False # terminates thread if true self._pause_condition = threading.Condition() self._lock = threading.RLock() - - # cached parameters (invalidated if arguments for them change) - # last set of events we've drawn with - - self._last_logged_events = [] + self._has_new_event = False # fetches past tor events from log file, if available @@ -116,7 +111,7 @@ class LogPanel(panel.Panel, threading.Thread): # leaving last_content_height as being too low causes initialization problems - self.last_content_height = len(self._logged_events) + self._last_content_height = len(self._logged_events) def set_duplicate_visability(self, is_visible): """ @@ -244,11 +239,11 @@ class LogPanel(panel.Panel, threading.Thread): def handle_key(self, key): if key.is_scroll(): page_height = self.get_preferred_size()[0] - 1 - new_scroll = ui_tools.get_scroll_position(key, self.scroll, page_height, self.last_content_height) + new_scroll = ui_tools.get_scroll_position(key, self._scroll, page_height, self._last_content_height) - if self.scroll != new_scroll: + if self._scroll != new_scroll: with self._lock: - self.scroll = new_scroll + self._scroll = new_scroll self.redraw(True) elif key.match('u'): with self._lock: @@ -303,11 +298,8 @@ class LogPanel(panel.Panel, threading.Thread): ] def draw(self, width, height): - event_log = self.get_attr('_logged_events') - with self._lock: - self._last_logged_events, self._last_update = event_log, time.time() - event_log = list(event_log) + event_log = list(self.get_attr('_logged_events')) # draws the top label @@ -324,20 +316,20 @@ class LogPanel(panel.Panel, threading.Thread): # restricts scroll location to valid bounds - self.scroll = max(0, min(self.scroll, self.last_content_height - height + 1)) + self._scroll = max(0, min(self._scroll, self._last_content_height - height + 1)) # draws left-hand scroll bar if content's longer than the height msg_indent, divider_indent = 1, 0 # offsets for scroll bar - is_scroll_bar_visible = self.last_content_height > height - 1 + is_scroll_bar_visible = self._last_content_height > height - 1 if is_scroll_bar_visible: msg_indent, divider_indent = 3, 2 - self.add_scroll_bar(self.scroll, self.scroll + height - 1, self.last_content_height, 1) + self.add_scroll_bar(self._scroll, self._scroll + height - 1, self._last_content_height, 1) # draws log entries - line_count = 1 - self.scroll + line_count = 1 - self._scroll seen_first_date_divider = False divider_attr, duplicate_attr = (curses.A_BOLD, 'yellow'), (curses.A_BOLD, 'green') @@ -459,13 +451,13 @@ class LogPanel(panel.Panel, threading.Thread): # - last_content_height was off by too much # - we're off the bottom of the page - new_content_height = line_count + self.scroll - 1 - content_height_delta = abs(self.last_content_height - new_content_height) + new_content_height = line_count + self._scroll - 1 + content_height_delta = abs(self._last_content_height - new_content_height) force_redraw, force_redraw_reason = True, '' if content_height_delta >= CONTENT_HEIGHT_REDRAW_THRESHOLD: force_redraw_reason = 'estimate was off by %i' % content_height_delta - elif new_content_height > height and self.scroll + height - 1 > new_content_height: + elif new_content_height > height and self._scroll + height - 1 > new_content_height: force_redraw_reason = 'scrolled off the bottom of the page' elif not is_scroll_bar_visible and new_content_height > height - 1: force_redraw_reason = "scroll bar wasn't previously visible" @@ -474,7 +466,8 @@ class LogPanel(panel.Panel, threading.Thread): else: force_redraw = False - self.last_content_height = new_content_height + self._last_content_height = new_content_height + self._has_new_event = False if force_redraw: log.debug('redrawing the log panel with the corrected content height (%s)' % force_redraw_reason) @@ -487,16 +480,16 @@ class LogPanel(panel.Panel, threading.Thread): responsive if additions are less frequent. """ - last_day = nyx.util.log.days_since(time.time()) # used to determine if the date has changed + last_ran, last_day = -1, nyx.util.log.days_since(time.time()) while not self._halt: current_day = nyx.util.log.days_since(time.time()) - time_since_reset = time.time() - self._last_update + time_since_reset = time.time() - last_ran max_log_update_rate = CONFIG['features.log.maxRefreshRate'] / 1000.0 sleep_time = 0 - if (self._logged_events == self._last_logged_events and last_day == current_day) or self.is_paused(): + if (not self._has_new_event and last_day == current_day) or self.is_paused(): sleep_time = 5 elif time_since_reset < max_log_update_rate: sleep_time = max(0.05, max_log_update_rate - time_since_reset) @@ -505,18 +498,15 @@ class LogPanel(panel.Panel, threading.Thread): with self._pause_condition: if not self._halt: self._pause_condition.wait(sleep_time) - else: - last_day = current_day - self.redraw(True) - # makes sure that we register this as an update, otherwise lacking the - # curses lock can cause a busy wait here + continue - self._last_update = time.time() + last_ran, last_day = time.time(), current_day + self.redraw(True) def stop(self): """ - Halts further resolutions and terminates the thread. + Halts further updates and terminates the thread. """ with self._pause_condition: @@ -550,5 +540,7 @@ class LogPanel(panel.Panel, threading.Thread): # notifies the display that it has new content if self._filter.match(event.display_message): + self._has_new_event = True + with self._pause_condition: self._pause_condition.notifyAll()

1 0

[nyx/master] Use absolute names for nyx.util.log functions
by atagar＠torproject.org 05 May '15

05 May '15

commit 2ebaf9cbf00b23d9d5646504bb19dc0e0405a4b2 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat May 2 13:56:44 2015 -0700 Use absolute names for nyx.util.log functions Now that so many are over there importing the names is a pretty big import line. Lets just use the absolute names instead. --- nyx/log_panel.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/nyx/log_panel.py b/nyx/log_panel.py index fe3e39e..590df6e 100644 --- a/nyx/log_panel.py +++ b/nyx/log_panel.py @@ -16,11 +16,9 @@ from stem.util import conf, log, str_tools import nyx.arguments import nyx.popups +import nyx.util.log from nyx.util import join, panel, tor_controller, ui_tools -from nyx.util.log import LogFileOutput, LogGroup, LogEntry, LogFilters, read_tor_log, condense_runlevels, listen_for_events, days_since, log_file_path - -ENTRY_INDENT = 2 # spaces an entry's message is indented after the first line def conf_handler(key, value): @@ -74,10 +72,10 @@ class LogPanel(panel.Panel, threading.Thread): threading.Thread.__init__(self) self.setDaemon(True) - self._logged_event_types = listen_for_events(self._register_tor_event, logged_events) - self._logged_events = LogGroup(CONFIG['cache.log_panel.size'], group_by_day = CONFIG['features.log.showDateDividers']) - self._log_file = LogFileOutput(CONFIG['features.log_file']) - self._filter = LogFilters(initial_filters = CONFIG['features.log.regex']) + self._logged_event_types = nyx.util.log.listen_for_events(self._register_tor_event, logged_events) + self._logged_events = nyx.util.log.LogGroup(CONFIG['cache.log_panel.size'], group_by_day = CONFIG['features.log.showDateDividers']) + self._log_file = nyx.util.log.LogFileOutput(CONFIG['features.log_file']) + self._filter = nyx.util.log.LogFilters(initial_filters = CONFIG['features.log.regex']) self.set_pause_attr('_logged_events') @@ -97,11 +95,11 @@ class LogPanel(panel.Panel, threading.Thread): # fetches past tor events from log file, if available if CONFIG['features.log.prepopulate']: - log_location = log_file_path(tor_controller()) + log_location = nyx.util.log.log_file_path(tor_controller()) if log_location: try: - for entry in reversed(list(read_tor_log(log_location, CONFIG['features.log.prepopulateReadLimit']))): + for entry in reversed(list(nyx.util.log.read_tor_log(log_location, CONFIG['features.log.prepopulateReadLimit']))): if entry.type in self._logged_event_types: self._logged_events.add(entry) except IOError as exc: @@ -180,7 +178,7 @@ class LogPanel(panel.Panel, threading.Thread): try: if event_types != self._logged_event_types: with self.vals_lock: - self._logged_event_types = listen_for_events(self._register_tor_event, event_types) + self._logged_event_types = nyx.util.log.listen_for_events(self._register_tor_event, event_types) self.redraw(True) except ValueError as exc: nyx.popups.show_msg('Invalid flags: %s' % str(exc), 2) @@ -207,7 +205,7 @@ class LogPanel(panel.Panel, threading.Thread): """ with self.vals_lock: - self._logged_events = LogGroup(CONFIG['cache.log_panel.size'], group_by_day = CONFIG['features.log.showDateDividers']) + self._logged_events = nyx.util.log.LogGroup(CONFIG['cache.log_panel.size'], group_by_day = CONFIG['features.log.showDateDividers']) self.redraw(True) def save_snapshot(self, path): @@ -320,7 +318,7 @@ class LogPanel(panel.Panel, threading.Thread): # draws the top label if self.is_title_visible(): - comp = list(condense_runlevels(*self._logged_event_types)) + comp = list(nyx.util.log.condense_runlevels(*self._logged_event_types)) if self._filter.selection(): comp.append('filter: %s' % self._filter.selection()) @@ -453,7 +451,7 @@ class LogPanel(panel.Panel, threading.Thread): if include_break or not display_queue: line_count += 1 - cursor_location = msg_indent + ENTRY_INDENT + cursor_location = msg_indent + 2 # indent following lines line_count += line_offset @@ -501,10 +499,10 @@ class LogPanel(panel.Panel, threading.Thread): responsive if additions are less frequent. """ - last_day = days_since(time.time()) # used to determine if the date has changed + last_day = nyx.util.log.days_since(time.time()) # used to determine if the date has changed while not self._halt: - current_day = days_since(time.time()) + current_day = nyx.util.log.days_since(time.time()) time_since_reset = time.time() - self._last_update max_log_update_rate = CONFIG['features.log.maxRefreshRate'] / 1000.0 @@ -545,13 +543,13 @@ class LogPanel(panel.Panel, threading.Thread): elif isinstance(event, stem.response.events.LogEvent): msg = event.message - self._register_event(LogEntry(event.arrived_at, event.type, msg)) + self._register_event(nyx.util.log.LogEntry(event.arrived_at, event.type, msg)) def _register_nyx_event(self, record): if record.levelname == 'WARNING': record.levelname = 'WARN' - self._register_event(LogEntry(int(record.created), 'NYX_%s' % record.levelname, record.msg)) + self._register_event(nyx.util.log.LogEntry(int(record.created), 'NYX_%s' % record.levelname, record.msg)) def _register_event(self, event): if event.type not in self._logged_event_types:

1 0

[nyx/master] Finish rewriting the log panel
by atagar＠torproject.org 05 May '15

05 May '15

commit 454803d2516cf53c91a9f0068003761168fcbcf7 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon May 4 22:35:38 2015 -0700 Finish rewriting the log panel Not completely perfect, but damn close. Happy enough with it to finally merge. :P --- nyx/graph_panel.py | 3 +-- nyx/header_panel.py | 5 ++-- nyx/log_panel.py | 74 ++++++++++++++++++++------------------------------- 3 files changed, 32 insertions(+), 50 deletions(-) diff --git a/nyx/graph_panel.py b/nyx/graph_panel.py index 3cc60bd..cf43492 100644 --- a/nyx/graph_panel.py +++ b/nyx/graph_panel.py @@ -20,10 +20,9 @@ import nyx.controller import nyx.popups import nyx.util.tracker -from nyx.util import join, msg, panel, tor_controller - from stem.control import EventType, Listener from stem.util import conf, enum, log, str_tools, system +from nyx.util import join, msg, panel, tor_controller GraphStat = enum.Enum(('BANDWIDTH', 'bandwidth'), ('CONNECTIONS', 'connections'), ('SYSTEM_RESOURCES', 'resources')) Interval = enum.Enum(('EACH_SECOND', 'each second'), ('FIVE_SECONDS', '5 seconds'), ('THIRTY_SECONDS', '30 seconds'), ('MINUTELY', 'minutely'), ('FIFTEEN_MINUTE', '15 minute'), ('THIRTY_MINUTE', '30 minute'), ('HOURLY', 'hourly'), ('DAILY', 'daily')) diff --git a/nyx/header_panel.py b/nyx/header_panel.py index df63174..4eb0a84 100644 --- a/nyx/header_panel.py +++ b/nyx/header_panel.py @@ -10,14 +10,13 @@ import time import curses import threading +import stem + import nyx.controller import nyx.popups -import stem - from stem.control import Listener from stem.util import conf, log, proc, str_tools, system - from nyx.util import msg, tor_controller, panel, tracker MIN_DUAL_COL_WIDTH = 141 # minimum width where we'll show two columns diff --git a/nyx/log_panel.py b/nyx/log_panel.py index 300af83..626a42f 100644 --- a/nyx/log_panel.py +++ b/nyx/log_panel.py @@ -9,15 +9,13 @@ import time import curses import threading -import stem import stem.response.events -from stem.util import conf, log, str_tools - import nyx.arguments import nyx.popups import nyx.util.log +from stem.util import conf, log, str_tools from nyx.util import join, panel, tor_controller, ui_tools @@ -45,6 +43,8 @@ CONFIG = conf.config_dict('nyx', { 'attr.log_color': {}, }, conf_handler) +TIMEZONE_OFFSET = time.altzone if time.localtime()[8] else time.timezone + # The height of the drawn content is estimated based on the last time we redrew # the panel. It's chiefly used for scrolling and the bar indicating its # position. Letting the estimate be too inaccurate results in a display bug, so @@ -55,14 +55,14 @@ CONTENT_HEIGHT_REDRAW_THRESHOLD = 3 # Log buffer so we start collecting stem/nyx events when imported. This is used # to make our LogPanel when curses initializes. -stem_logger = stem.util.log.get_logger() +stem_logger = log.get_logger() NYX_LOGGER = log.LogBuffer(log.Runlevel.DEBUG, yield_records = True) stem_logger.addHandler(NYX_LOGGER) class LogPanel(panel.Panel, threading.Thread): """ - Listens for and displays tor, nyx, and stem events. This can prepopulate + Listens for and displays tor, nyx, and stem events. This prepopulates from tor's log file if it exists. """ @@ -71,19 +71,16 @@ class LogPanel(panel.Panel, threading.Thread): threading.Thread.__init__(self) self.setDaemon(True) + self._logged_events = nyx.util.log.LogGroup(CONFIG['cache.log_panel.size'], group_by_day = True) self._logged_event_types = nyx.util.log.listen_for_events(self._register_tor_event, logged_events) - self._logged_events = nyx.util.log.LogGroup(CONFIG['cache.log_panel.size']) self._log_file = nyx.util.log.LogFileOutput(CONFIG['features.logFile']) self._filter = nyx.util.log.LogFilters(initial_filters = CONFIG['features.log.regex']) self.set_pause_attr('_logged_events') - self._last_content_height = 0 # height of the rendered content when last drawn - self._scroll = 0 - self._halt = False # terminates thread if true - self._pause_condition = threading.Condition() self._lock = threading.RLock() + self._pause_condition = threading.Condition() self._has_new_event = False # fetches past tor events from log file, if available @@ -101,24 +98,22 @@ class LogPanel(panel.Panel, threading.Thread): except ValueError as exc: log.info(str(exc)) - # stop logging to NYX_LOGGER, adding its event backlog and future ones + self._last_content_height = len(self._logged_events) # height of the rendered content when last drawn + self._scroll = 0 + + # merge NYX_LOGGER into us, and listen for its future events for event in NYX_LOGGER: self._register_nyx_event(event) NYX_LOGGER.emit = self._register_nyx_event - # leaving last_content_height as being too low causes initialization problems - - self._last_content_height = len(self._logged_events) - def set_duplicate_visability(self, is_visible): """ Sets if duplicate log entries are collaped or expanded. - Arguments: - is_visible - if true all log entries are shown, otherwise they're - deduplicated + :param bool is_visible: if **True** all log entries are shown, otherwise + they're deduplicated """ nyx_config = conf.get_config('nyx') @@ -166,10 +161,10 @@ class LogPanel(panel.Panel, threading.Thread): user_input = nyx.popups.input_prompt('Events to log: ') if user_input: - user_input = user_input.replace(' ', '') # strips spaces - event_types = nyx.arguments.expand_events(user_input) - try: + user_input = user_input.replace(' ', '') # strip spaces + event_types = nyx.arguments.expand_events(user_input) + if event_types != self._logged_event_types: with self._lock: self._logged_event_types = nyx.util.log.listen_for_events(self._register_tor_event, event_types) @@ -191,7 +186,7 @@ class LogPanel(panel.Panel, threading.Thread): self.save_snapshot(path_input) nyx.popups.show_msg('Saved: %s' % path_input, 2) except IOError as exc: - nyx.popups.show_msg('Unable to save snapshot: %s' % exc.strerror, 2) + nyx.popups.show_msg('Unable to save snapshot: %s' % exc, 2) def clear(self): """ @@ -199,17 +194,17 @@ class LogPanel(panel.Panel, threading.Thread): """ with self._lock: - self._logged_events = nyx.util.log.LogGroup(CONFIG['cache.log_panel.size']) + self._logged_events = nyx.util.log.LogGroup(CONFIG['cache.log_panel.size'], group_by_day = True) self.redraw(True) def save_snapshot(self, path): """ Saves the log events currently being displayed to the given path. This - takes filers into account. This overwrites the file if it already exists, - and raises an IOError if there's a problem. + takes filers into account. This overwrites the file if it already exists. + + :param str path: path where to save the log snapshot - Arguments: - path - path where to save the log snapshot + :raises: **IOError** if unsuccessful """ path = os.path.abspath(os.path.expanduser(path)) @@ -227,10 +222,8 @@ class LogPanel(panel.Panel, threading.Thread): with self._lock: with open(path, 'w') as snapshot_file: try: - for entry in reversed(self._logged_events): - is_visible = self._filter.match(entry.display_message) - - if is_visible: + for entry in reversed(list(self._logged_events)): + if self._filter.match(entry.display_message): snapshot_file.write(entry.display_message + '\n') except Exception as exc: raise IOError("unable to write to '%s': %s" % (path, exc)) @@ -255,19 +248,10 @@ class LogPanel(panel.Panel, threading.Thread): if key_press.match('c'): self.clear() elif key.match('f'): - # Provides menu to pick regular expression filters or adding new ones: - # for syntax see: http://docs.python.org/library/re.html#regular-expression-syntax - - options = ['None'] + self._filter.latest_selections() + ['New...'] - old_selection = 0 if not self._filter.selection() else 1 - - # does all activity under a curses lock to prevent redraws when adding - # new filters - with panel.CURSES_LOCK: - selection = nyx.popups.show_menu('Log Filter:', options, old_selection) - - # applies new setting + initial_selection = 1 if self._filter.selection() else 0 + options = ['None'] + self._filter.latest_selections() + ['New...'] + selection = nyx.popups.show_menu('Log Filter:', options, initial_selection) if selection == 0: self._filter.select(None) @@ -430,10 +414,10 @@ class LogPanel(panel.Panel, threading.Thread): responsive if additions are less frequent. """ - last_ran, last_day = -1, nyx.util.log.days_since(time.time()) + last_ran, last_day = -1, int((time.time() - TIMEZONE_OFFSET) / 86400) while not self._halt: - current_day = nyx.util.log.days_since(time.time()) + current_day = int((time.time() - TIMEZONE_OFFSET) / 86400) time_since_reset = time.time() - last_ran max_log_update_rate = CONFIG['features.log.maxRefreshRate'] / 1000.0

1 0

[nyx/master] Rewrite log panel
by atagar＠torproject.org 05 May '15

05 May '15

commit 1ef21851fabb44d4df83cde2df64e490f31848f7 Merge: 4517746 454803d Author: Damian Johnson <atagar(a)torproject.org> Date: Mon May 4 22:36:43 2015 -0700 Rewrite log panel Overhaul our log panel. Cleaner code, decent test coverage, but probably neatest for users is that our deduplication logic is no longer painfully inefficient. Previously it was a O(n^2) operation essentially every time we logged an event (so pretty much O(n^3). This was so bad the panel turned off deduplication when it became too slow. New version is O(n^2) over the lifetime of our process, and we can make it linear with just a bit more work. nyx/config/attributes.cfg | 18 + nyx/config/dedup.cfg | 68 +- nyx/config/strings.cfg | 1 + nyx/controller.py | 10 +- nyx/graph_panel.py | 3 +- nyx/header_panel.py | 5 +- nyx/log_panel.py | 1309 +++++------------------------ nyx/menu/actions.py | 6 +- nyx/util/__init__.py | 9 +- nyx/util/log.py | 452 ++++++++++ nyx/util/panel.py | 36 +- nyx/util/ui_tools.py | 27 +- nyxrc.sample | 7 - run_tests.py | 10 +- test/util/__init__.py | 2 +- test/util/log/__init__.py | 8 + test/util/log/condense_runlevels.py | 13 + test/util/log/data/daybreak_deduplication | 37 + test/util/log/data/malformed_date | 21 + test/util/log/data/malformed_line | 21 + test/util/log/data/malformed_runlevel | 21 + test/util/log/data/multiple_tor_instances | 33 + test/util/log/data/tor_log | 21 + test/util/log/log_entry.py | 27 + test/util/log/log_group.py | 146 ++++ test/util/log/read_tor_log.py | 59 ++ 26 files changed, 1202 insertions(+), 1168 deletions(-)

1 0

[tor-browser-spec/master] Address Georg's first round of comments.
by mikeperry＠torproject.org 05 May '15

05 May '15

commit 20c36ab5fa6d1cfe2023fbd5d254afa64cf5208f Author: Mike Perry <mikeperry-git(a)torproject.org> Date: Mon May 4 19:30:38 2015 -0700 Address Georg's first round of comments. Primarily removing mention of IP address linkability, and clarifying the WebWorker blob isolation section areas. --- design-doc/design.xml | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/design-doc/design.xml b/design-doc/design.xml index f7ef5dc..3d4f18e 100644 --- a/design-doc/design.xml +++ b/design-doc/design.xml @@ -1259,12 +1259,12 @@ False Start</ulink> via the Firefox Pref <command>security.ssl.enable_false_start</command>. </para> </listitem> - <listitem>IP address, Tor circuit, and HTTP Keep-Alive linkability + <listitem>Tor circuit and HTTP connection linkability <para> -IP addresses, Tor circuits, and HTTP connections from a third party in one URL -bar origin MUST NOT be reused for that same third party in another URL bar -origin. +Tor circuits and HTTP connections from a third party in one URL bar origin +MUST NOT be reused for that same third party in another URL bar origin. + </para> <para> @@ -1275,11 +1275,10 @@ component that <ulink linkend="https://gitweb.torproject.org/torbutton.git/tree/src/components/domain-isol…">sets the SOCKS username and password for each request</ulink>. The Tor client has logic to prevent connections with different SOCKS usernames and passwords from -using the same Tor circuit, which provides us with IP address unlinkability. -Firefox has existing logic to ensure that connections with SOCKS proxies do not -re-use existing HTTP Keep-Alive connections unless the proxy settings match. -We extended this logic to cover SOCKS username and password authentication, -providing us with HTTP Keep-Alive unlinkability. +using the same Tor circuit. Firefox has existing logic to ensure that connections with +SOCKS proxies do not re-use existing HTTP Keep-Alive connections unless the +proxy settings match. We extended this logic to cover SOCKS username and +password authentication, providing us with HTTP Keep-Alive unlinkability. </para> </listitem> @@ -1324,7 +1323,9 @@ URIs created with URL.createObjectURL MUST be limited in scope to the first party URL bar domain that created them. We provide this isolation in Tor Browser via a <ulink url="https://gitweb.torproject.org/tor-browser.git/commit/?h=tor-browser-31.6.0e…">direct -patch to Firefox</ulink> and disable URL.createObjectURL in a worker context as a stopgap. +patch to Firefox</ulink> and disable URL.createObjectURL in the WebWorker +context as a stopgap, due to an edge case with enforcing this isolation in +WebWorkers. </para> </listitem>

1 0

[tor-browser-spec/master] Describe user behavior as a fingerprinting source.
by mikeperry＠torproject.org 05 May '15

05 May '15

commit d8b78e0729a1e5d16244812fbc86b9fa9f29f5b0 Author: Mike Perry <mikeperry-git(a)torproject.org> Date: Mon May 4 20:14:38 2015 -0700 Describe user behavior as a fingerprinting source. Also generalize TCP Port to TCP Port and Local Network. --- design-doc/design.xml | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/design-doc/design.xml b/design-doc/design.xml index 3d4f18e..fbec073 100644 --- a/design-doc/design.xml +++ b/design-doc/design.xml @@ -1464,8 +1464,10 @@ severe, and how to study the efficacy of defenses properly. <title>Sources of Fingerprinting Issues</title> <para> -All fingerprinting issues arise from one of four primary sources. In order -from most severe to least severe, these sources are: +All fingerprinting issues arise from one of four primary sources in the +browser. Additionally, user behavior itself provides one more source of +potential fingerprinting. Listed in order from most severe to least severe in +terms of the amount of information they reveal, these sources are: </para> <orderedlist> @@ -1487,13 +1489,15 @@ do so only on a per-site basis via site permissions, to avoid linkability. <listitem><command>Device and Hardware Characteristics</command> <para> -Device and hardware characteristics can be determined in three ways: they can be -reported explicitly by the browser, they can be inferred through API behavior, -or they can be extracted through statistical measurements of system -performance. We are most concerned with the cases where this information is -either directly reported or can be determined via a single use of an API or -feature, and prefer to place such APIs either behind site permissions, or -disable them entirely. +Device and hardware characteristics can be determined in three ways: they can +be reported explicitly by the browser, they can be inferred through browser +functionality, or they can be extracted through statistical measurements of +system performance. We are most concerned with the cases where this +information is either directly reported or can be determined via a single use +of an API or feature, and prefer to place such APIs either behind site +permissions, alter their functionality to prevent exposing the most variable +aspects of these characteristics, or disable them entirely. + </para> <para> @@ -1522,6 +1526,20 @@ specific version of a system can be inferred. </para> </listitem> + <listitem><command>User Behavior</command> + <para> + +While somewhat outside the scope of browser fingerprinting, for completeness +it is important to mention that users themselves theoretically might be +fingerprinted through their behavior while interacting with a website. This +behavior includes as keystrokes, mouse movements, click speed, and writing +style. Basic vectors such as keystroke and mouse usage fingerprinting can be +mitigated by altering Javascript's notion of time. More advanced issues like +writing style fingerprinting are the domain of <ulink +url="https://github.com/psal/anonymouth">other tools</ulink>. + + </para> + </listitem> <listitem><command>Browser Vendor and Version Differences</command> <para> @@ -1633,7 +1651,7 @@ image data, pure white image data is returned to the Javascript APIs. <para> </para> </listitem> - <listitem>Open TCP Port Fingerprinting + <listitem>Open TCP Port and Local Network Fingerprinting <para> In Firefox, by using either WebSockets or XHR, it is possible for remote

1 0