September 2014 - tor-commits - lists.torproject.org

[arm/master] Using new behavior to fetch our own descriptor
by atagar＠torproject.org 08 Sep '14

08 Sep '14

commit 5636ea27b05476fb5ed10443dcd3cb4d86b8aa7f Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Sep 7 14:21:15 2014 -0700 Using new behavior to fetch our own descriptor Stem now returns our own descriptor if no fingerprint is provided, so taking advantage of that. --- arm/header_panel.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/arm/header_panel.py b/arm/header_panel.py index 0278285..164e50d 100644 --- a/arm/header_panel.py +++ b/arm/header_panel.py @@ -402,8 +402,6 @@ class Sampling(object): def __init__(self, last_sampling = None): controller = tor_controller() - pid = controller.get_pid(None) - fingerprint = controller.get_info('fingerprint', None) or_listeners = controller.get_listeners(Listener.OR, []) fd_limit = controller.get_info('process/descriptor-limit', '-1') @@ -416,7 +414,7 @@ class Sampling(object): self.retrieved = time.time() self.arm_total_cpu_time = sum(os.times()[:3]) - self.fingerprint = fingerprint if fingerprint else 'Unknown' + self.fingerprint = controller.get_info('fingerprint', 'Unknown') self.nickname = controller.get_conf('Nickname', '') self.or_address = or_listeners[0][0] if or_listeners else controller.get_info('address', 'Unknown') self.or_port = or_listeners[0][1] if or_listeners else '' @@ -435,17 +433,17 @@ class Sampling(object): self.auth_color = 'red' if self.auth_type == 'open' else 'green' self.exit_policy = controller.get_exit_policy(None) - self.flags = getattr(controller.get_network_status(fingerprint, None), 'flags', []) + self.flags = getattr(controller.get_network_status(default = None), 'flags', []) self.version = str(controller.get_version('Unknown')).split()[0] self.version_status = controller.get_info('status/version/current', 'Unknown') self.version_color = CONFIG['attr.version_status_colors'].get(self.version_status, 'white') - self.pid = pid if pid else '' - self.start_time = stem.util.system.start_time(pid) + self.pid = controller.get_pid('') + self.start_time = stem.util.system.start_time(self.pid) self.fd_limit = int(fd_limit) if fd_limit.isdigit() else None try: - self.fd_used = stem.util.proc.file_descriptors_used(pid) + self.fd_used = stem.util.proc.file_descriptors_used(self.pid) except IOError: self.fd_used = None

1 0

[arm/master] Exception when running 'run_arm --help'
by atagar＠torproject.org 08 Sep '14

08 Sep '14

commit 120d751cb8117cf3b593e536803fc22ce1676b64 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Sep 7 15:06:35 2014 -0700 Exception when running 'run_arm --help' Stacktrace due to having two 'config' attributes. Traceback (most recent call last): File "./run_arm", line 47, in <module> main() File "./run_arm", line 16, in main arm.starter.main() File "/home/atagar/Desktop/arm/stem/util/conf.py", line 288, in wrapped return func(*args, config = config, **kwargs) File "/home/atagar/Desktop/arm/arm/starter.py", line 43, in main print arm.arguments.get_help() File "/home/atagar/Desktop/arm/arm/arguments.py", line 142, in get_help event_flags = msg('misc.event_types'), File "/home/atagar/Desktop/arm/stem/util/conf.py", line 288, in wrapped return func(*args, config = config, **kwargs) TypeError: msg() got multiple values for keyword argument 'config' --- arm/arguments.py | 2 +- arm/config/strings.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arm/arguments.py b/arm/arguments.py index 136971e..d7820dd 100644 --- a/arm/arguments.py +++ b/arm/arguments.py @@ -137,7 +137,7 @@ def get_help(): address = DEFAULT_ARGS['control_address'], port = DEFAULT_ARGS['control_port'], socket = DEFAULT_ARGS['control_socket'], - config = DEFAULT_ARGS['config'], + config_path = DEFAULT_ARGS['config'], events = DEFAULT_ARGS['logged_events'], event_flags = msg('misc.event_types'), ) diff --git a/arm/config/strings.cfg b/arm/config/strings.cfg index 25970d0..f20d277 100644 --- a/arm/config/strings.cfg +++ b/arm/config/strings.cfg @@ -76,7 +76,7 @@ msg.usage.help_output | -s, --socket SOCKET_PATH attach using unix domain socket if present, | SOCKET_PATH defaults to: {socket} | -c, --config CONFIG_PATH loaded configuration options, CONFIG_PATH -| defaults to: {config} +| defaults to: {config_path} | -d, --debug LOG_PATH writes all arm logs to the given location | -l, --log EVENT_FLAGS event types to be logged (default: {events}) |{event_flags}

1 0

[arm/master] First stab at reconnection support
by atagar＠torproject.org 08 Sep '14

08 Sep '14

commit 934cab1949e0fef1229cd5e0b6e920c4466d45f1 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Sep 8 00:14:46 2014 -0700 First stab at reconnection support This really shouldn't be hard. While writing Stem I took reconnectability into account from the beginning so it should be as simple as re-connecting and re-authenticating. However, something failing badly. Unfortunately this is proving tricky to troubleshoot so think I'm gonna move on for now, then loop back when more of arm is rewritten. Hopefully this will make investigating easier. --- arm/header_panel.py | 80 +++++++++++++++++++++++---------------------------- 1 file changed, 36 insertions(+), 44 deletions(-) diff --git a/arm/header_panel.py b/arm/header_panel.py index 08e6282..49ec6dc 100644 --- a/arm/header_panel.py +++ b/arm/header_panel.py @@ -12,7 +12,9 @@ import threading import arm.controller import arm.popups -from stem.control import Listener, Signal +import stem + +from stem.control import Listener from stem.util import conf, log, proc, str_tools, system from util import msg, tor_controller, panel, tracker @@ -70,7 +72,12 @@ class HeaderPanel(panel.Panel, threading.Thread): Requests a new identity and provides a visual queue. """ - tor_controller().signal(Signal.NEWNYM) + controller = tor_controller() + + if not controller.is_newnym_available(): + return + + controller.signal(stem.Signal.NEWNYM) # If we're wide then the newnym label in this panel will give an # indication that the signal was sent. Otherwise use a msg. @@ -81,51 +88,36 @@ class HeaderPanel(panel.Panel, threading.Thread): def handle_key(self, key): is_keystroke_consumed = True - if key in (ord('n'), ord('N')) and tor_controller().is_newnym_available(): + if key in (ord('n'), ord('N')): self.send_newnym() elif key in (ord('r'), ord('R')) and not self._vals.is_connected: - # oldSocket = tor_tools.get_conn().get_controller().get_socket() - # - # controller = None - # allowPortConnection, allowSocketConnection, _ = starter.allowConnectionTypes() - # - # if os.path.exists(CONFIG["startup.interface.socket"]) and allowSocketConnection: - # try: - # # TODO: um... what about passwords? - # controller = Controller.from_socket_file(CONFIG["startup.interface.socket"]) - # controller.authenticate() - # except (IOError, stem.SocketError), exc: - # controller = None - # - # if not allowPortConnection: - # arm.popups.show_msg("Unable to reconnect (%s)" % exc, 3) - # elif not allowPortConnection: - # arm.popups.show_msg("Unable to reconnect (socket '%s' doesn't exist)" % CONFIG["startup.interface.socket"], 3) - # - # if not controller and allowPortConnection: - # # TODO: This has diverged from starter.py's connection, for instance it - # # doesn't account for relative cookie paths or multiple authentication - # # methods. We can't use the starter.py's connection function directly - # # due to password prompts, but we could certainly make this mess more - # # manageable. + # TODO: This is borked. Not quite sure why but our attempt to call + # PROTOCOLINFO fails with a socket error, followed by completely freezing + # arm. This is exposing two bugs... # - # try: - # ctlAddr, ctl_port = CONFIG["startup.interface.ip_address"], CONFIG["startup.interface.port"] - # controller = Controller.from_port(ctlAddr, ctl_port) - # - # try: - # controller.authenticate() - # except stem.connection.MissingPassword: - # controller.authenticate(authValue) # already got the password above - # except Exception, exc: - # controller = None - # - # if controller: - # tor_tools.get_conn().init(controller) - # log.notice("Reconnected to Tor's control port") - # arm.popups.show_msg("Tor reconnected", 1) + # * This should be working. That's a stem issue. + # * Our interface shouldn't be locking up. That's an arm issue. + + return True + + controller = tor_controller() + + try: + controller.connect() - pass + try: + controller.authenticate() + except stem.connection.MissingPassword: + password = cli.popups.input_prompt('Controller Password: ') + + if password: + controller.authenticate(password) + + log.notice("Reconnected to Tor's control port") + arm.popups.show_msg('Tor reconnected', 1) + except Exception as exc: + arm.popups.show_msg('Unable to reconnect (%s)' % exc, 3) + controller.close() else: is_keystroke_consumed = False @@ -443,9 +435,9 @@ class Sampling(object): self.fd_used = None tor_resources = tracker.get_resource_tracker().get_value() + self.arm_total_cpu_time = sum(os.times()[:3]) self.tor_cpu = '%0.1f' % (100 * tor_resources.cpu_sample) self.arm_cpu = '%0.1f' % (100 * self._get_cpu_percentage(last_sampling)) - self.arm_total_cpu_time = sum(os.times()[:3]) self.memory = str_tools.size_label(tor_resources.memory_bytes) if tor_resources.memory_bytes > 0 else 0 self.memory_percent = '%0.1f' % (100 * tor_resources.memory_percent)

1 0

[arm/master] General header panel cleanup
by atagar＠torproject.org 08 Sep '14

08 Sep '14

commit c0d5567957e0d7f5f2065270bcac354a1034fa39 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Sep 7 15:34:00 2014 -0700 General header panel cleanup Just some rearrangement and minor tweaks. --- arm/config/strings.cfg | 3 ++ arm/header_panel.py | 101 +++++++++++++++++++++++------------------------- 2 files changed, 52 insertions(+), 52 deletions(-) diff --git a/arm/config/strings.cfg b/arm/config/strings.cfg index f20d277..f44e168 100644 --- a/arm/config/strings.cfg +++ b/arm/config/strings.cfg @@ -19,6 +19,9 @@ msg.config.nothing_loaded No armrc loaded, using defaults. You can customize arm msg.debug.saving_to_path Saving a debug log to {path}, please check it for sensitive information before sharing it. msg.debug.unable_to_write_file Unable to write to our debug log file ({path}): {error} +msg.misc.fd_used_at_sixty_percent Tor's file descriptor usage is at {percentage}%. +msg.misc.fd_used_at_ninety_percent Tor's file descriptor usage is at {percentage}%. If you run out Tor will be unable to continue functioning. + msg.setup.arm_is_running_as_root Arm is currently running with root permissions. This isn't a good idea, nor should it be necessary. Try starting arm with "sudo -u {tor_user} arm" instead. msg.setup.chroot_doesnt_exist The chroot path set in your config ({path}) doesn't exist. msg.setup.set_freebsd_chroot Adjusting paths to account for Tor running in a FreeBSD jail at: {path} diff --git a/arm/header_panel.py b/arm/header_panel.py index bc06089..08e6282 100644 --- a/arm/header_panel.py +++ b/arm/header_panel.py @@ -9,20 +9,13 @@ import time import curses import threading -import stem -import stem.util.proc -import stem.util.str_tools -import stem.util.system - import arm.controller import arm.popups -import arm.starter -import arm.util.tracker -from stem.control import Listener -from stem.util import conf, log, str_tools +from stem.control import Listener, Signal +from stem.util import conf, log, proc, str_tools, system -from util import panel, tor_controller +from util import msg, tor_controller, panel, tracker MIN_DUAL_COL_WIDTH = 141 # minimum width where we'll show two columns SHOW_FD_THRESHOLD = 60 # show file descriptor usage if usage is over this percentage @@ -77,7 +70,7 @@ class HeaderPanel(panel.Panel, threading.Thread): Requests a new identity and provides a visual queue. """ - tor_controller().signal(stem.Signal.NEWNYM) + tor_controller().signal(Signal.NEWNYM) # If we're wide then the newnym label in this panel will give an # indication that the signal was sent. Otherwise use a msg. @@ -156,19 +149,19 @@ class HeaderPanel(panel.Panel, threading.Thread): if is_wide: self._draw_resource_usage(left_width, 0, right_width, vals) - else: - self._draw_resource_usage(0, 2, left_width, vals) - if vals.is_relay: - if is_wide: + if vals.is_relay: self._draw_fingerprint_and_fd_usage(left_width, 1, right_width, vals) self._draw_flags(0, 2, left_width, vals) self._draw_exit_policy(left_width, 2, right_width, vals) - else: + elif vals.is_connected: + self._draw_newnym_option(left_width, 1, right_width, vals) + else: + self._draw_resource_usage(0, 2, left_width, vals) + + if vals.is_relay: self._draw_fingerprint_and_fd_usage(0, 3, left_width, vals) self._draw_flags(0, 4, left_width, vals) - elif is_wide and vals.is_connected: - self._draw_newnym_option(left_width, 1, right_width, vals) def _draw_platform_section(self, x, y, width, vals): """ @@ -207,20 +200,22 @@ class HeaderPanel(panel.Panel, threading.Thread): if not vals.is_relay: x = self.addstr(y, x, 'Relaying Disabled', 'cyan') else: - x = self.addstr(y, x, vals.format('{nickname} - {or_address}:{or_port}')) + x = self.addstr(y, x, vals.format('{nickname} - {address}:{or_port}')) if vals.dir_port != '0': x = self.addstr(y, x, vals.format(', Dir Port: {dir_port}')) - if vals.control_port == '0': - self.addstr(y, x, vals.format(', Control Socket: {socket_path}')) - else: + if vals.control_port: if width >= x + 19 + len(vals.control_port) + len(vals.auth_type): + auth_color = 'red' if vals.auth_type == 'open' else 'green' + x = self.addstr(y, x, ', Control Port (') - x = self.addstr(y, x, vals.auth_type, vals.auth_color) + x = self.addstr(y, x, vals.auth_type, auth_color) self.addstr(y, x, vals.format('): {control_port}')) else: self.addstr(y, x, vals.format(', Control Port: {control_port}')) + elif vals.socket_path: + self.addstr(y, x, vals.format(', Control Socket: {socket_path}')) def _draw_disconnected(self, x, y, width, vals): """ @@ -247,7 +242,7 @@ class HeaderPanel(panel.Panel, threading.Thread): else: now = time.time() - uptime = stem.util.str_tools.short_time_label(now - vals.start_time) + uptime = str_tools.short_time_label(now - vals.start_time) else: uptime = '' @@ -405,27 +400,29 @@ class Sampling(object): def __init__(self, last_sampling = None): controller = tor_controller() - or_listeners = controller.get_listeners(Listener.OR, []) - fd_limit = controller.get_info('process/descriptor-limit', '-1') - - uname_vals = os.uname() - tor_resources = arm.util.tracker.get_resource_tracker().get_value() - + self.retrieved = time.time() self.is_connected = controller.is_alive() self.connection_time = controller.connection_time() self.last_heartbeat = time.strftime('%H:%M %m/%d/%Y', time.localtime(controller.get_latest_heartbeat())) - self.retrieved = time.time() - self.arm_total_cpu_time = sum(os.times()[:3]) self.fingerprint = controller.get_info('fingerprint', 'Unknown') self.nickname = controller.get_conf('Nickname', '') - self.or_address = or_listeners[0][0] if or_listeners else controller.get_info('address', 'Unknown') + self.newnym_wait = controller.get_newnym_wait() + self.exit_policy = controller.get_exit_policy(None) + self.flags = getattr(controller.get_network_status(default = None), 'flags', []) + + self.version = str(controller.get_version('Unknown')).split()[0] + self.version_status = controller.get_info('status/version/current', 'Unknown') + self.version_color = CONFIG['attr.version_status_colors'].get(self.version_status, 'white') + + or_listeners = controller.get_listeners(Listener.OR, []) + control_listeners = controller.get_listeners(Listener.CONTROL, []) + self.address = or_listeners[0][0] if (or_listeners and or_listeners[0][0] != '0.0.0.0') else controller.get_info('address', 'Unknown') self.or_port = or_listeners[0][1] if or_listeners else '' self.dir_port = controller.get_conf('DirPort', '0') - self.control_port = controller.get_conf('ControlPort', '0') - self.socket_path = controller.get_conf('ControlSocket', '') + self.control_port = str(control_listeners[0][1]) if control_listeners else None + self.socket_path = controller.get_conf('ControlSocket', None) self.is_relay = bool(self.or_port) - self.newnym_wait = controller.get_newnym_wait() if controller.get_conf('HashedControlPassword', None): self.auth_type = 'password' @@ -434,45 +431,45 @@ class Sampling(object): else: self.auth_type = 'open' - self.auth_color = 'red' if self.auth_type == 'open' else 'green' - self.exit_policy = controller.get_exit_policy(None) - self.flags = getattr(controller.get_network_status(default = None), 'flags', []) - self.version = str(controller.get_version('Unknown')).split()[0] - self.version_status = controller.get_info('status/version/current', 'Unknown') - self.version_color = CONFIG['attr.version_status_colors'].get(self.version_status, 'white') - self.pid = controller.get_pid('') - self.start_time = stem.util.system.start_time(self.pid) + self.start_time = system.start_time(self.pid) + + fd_limit = controller.get_info('process/descriptor-limit', '-1') self.fd_limit = int(fd_limit) if fd_limit.isdigit() else None try: - self.fd_used = stem.util.proc.file_descriptors_used(self.pid) + self.fd_used = proc.file_descriptors_used(self.pid) except IOError: self.fd_used = None + tor_resources = tracker.get_resource_tracker().get_value() self.tor_cpu = '%0.1f' % (100 * tor_resources.cpu_sample) self.arm_cpu = '%0.1f' % (100 * self._get_cpu_percentage(last_sampling)) - self.memory = stem.util.str_tools.size_label(tor_resources.memory_bytes) if tor_resources.memory_bytes > 0 else 0 + self.arm_total_cpu_time = sum(os.times()[:3]) + self.memory = str_tools.size_label(tor_resources.memory_bytes) if tor_resources.memory_bytes > 0 else 0 self.memory_percent = '%0.1f' % (100 * tor_resources.memory_percent) + + uname_vals = os.uname() self.hostname = uname_vals[1] self.platform = '%s %s' % (uname_vals[0], uname_vals[2]) # [platform name] [version] if self.fd_used and self.fd_limit: fd_percent = 100 * self.fd_used / self.fd_limit - msg = "Tor's file descriptor usage is at %i%%." % fd_percent if fd_percent >= 90: - msg += ' If you run out Tor will be unable to continue functioning.' - log.log_once('fd_used_at_ninety_percent', log.WARN, msg) + log_msg = msg('misc.fd_used_at_ninety_percent', percentage = fd_percent) + log.log_once('fd_used_at_ninety_percent', log.WARN, log_msg) + log.DEDUPLICATION_MESSAGE_IDS.add('fd_used_at_sixty_percent') elif fd_percent >= 60: - log.log_once('fd_used_at_sixty_percent', log.NOTICE, msg) + log_msg = msg('misc.fd_used_at_sixty_percent', percentage = fd_percent) + log.log_once('fd_used_at_sixty_percent', log.NOTICE, log_msg) - def format(self, msg, crop_width = None): + def format(self, message, crop_width = None): """ Applies our attributes to the given string. """ - formatted_msg = msg.format(**self.__dict__) + formatted_msg = message.format(**self.__dict__) if crop_width: formatted_msg = str_tools.crop(formatted_msg, crop_width)

1 0

[stem/master] Fix for prior correction of name_by_pid()
by atagar＠torproject.org 08 Sep '14

08 Sep '14

commit 11835ff247b43ea2c01ee3f100e655032606b262 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Sep 7 21:20:21 2014 -0700 Fix for prior correction of name_by_pid() Oops! Mistook which local the ps call was setting... File "/usr/lib/python2.7/threading.py", line 504, in run self.__target(*self.__args, **self.__kwargs) File "/home/atagar/Desktop/arm/arm/util/tracker.py", line 398, in _tor_status_listener tor_cmd = system.name_by_pid(tor_pid) if tor_pid else None File "/home/atagar/Desktop/arm/stem/util/system.py", line 277, in name_by_pid if results and len(results) == 2 and results[0] == 'COMMAND': UnboundLocalError: local variable 'results' referenced before assignment --- stem/util/system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stem/util/system.py b/stem/util/system.py index bcd6c0d..da02843 100644 --- a/stem/util/system.py +++ b/stem/util/system.py @@ -272,7 +272,7 @@ def name_by_pid(pid): try: results = call(GET_NAME_BY_PID_PS % pid) except OSError: - pass + results = None if results and len(results) == 2 and results[0] == 'COMMAND': process_name = results[1].strip()

1 0

[stem/master] Uncaught exception when name_by_pid() fails to call ps
by atagar＠torproject.org 08 Sep '14

08 Sep '14

commit efd1c86073ad04ec3c6d79c97dd3db5d3255fdf6 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Sep 7 21:10:35 2014 -0700 Uncaught exception when name_by_pid() fails to call ps Our name_by_pid() function is documented as returning None if it can't deterine the process name, but we accidently exposed a possible OSException via our ps call. Ran into this during arm development... File "/home/atagar/Desktop/arm/arm/util/tracker.py", line 398, in _tor_status_listener tor_cmd = system.name_by_pid(tor_pid) if tor_pid else None File "/home/atagar/Desktop/arm/stem/util/system.py", line 272, in name_by_pid results = call(GET_NAME_BY_PID_PS % pid) File "/home/atagar/Desktop/arm/stem/util/system.py", line 921, in call raise exc OSError: ps -p 30834 -o comm returned exit status 1 --- stem/util/system.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/stem/util/system.py b/stem/util/system.py index adfc802..bcd6c0d 100644 --- a/stem/util/system.py +++ b/stem/util/system.py @@ -269,7 +269,10 @@ def name_by_pid(pid): # vim if not process_name: - results = call(GET_NAME_BY_PID_PS % pid) + try: + results = call(GET_NAME_BY_PID_PS % pid) + except OSError: + pass if results and len(results) == 2 and results[0] == 'COMMAND': process_name = results[1].strip()

1 0

[stem/master] Better handling for default exit policy suffix
by atagar＠torproject.org 07 Sep '14

07 Sep '14

commit e0aeefb8c721b285fa23e1097cf8be45b86aa538 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Sep 7 11:50:39 2014 -0700 Better handling for default exit policy suffix Methods similar to the private entries to better handle the policy entries tor adds to the suffix by default if we lack an accept-all or reject-all rule. --- stem/exit_policy.py | 66 ++++++++++++++++++++++++++++++++++++++- test/unit/exit_policy/policy.py | 33 +++++++++++++++++--- 2 files changed, 93 insertions(+), 6 deletions(-) diff --git a/stem/exit_policy.py b/stem/exit_policy.py index 1f72b60..06f4d83 100644 --- a/stem/exit_policy.py +++ b/stem/exit_policy.py @@ -30,7 +30,9 @@ exiting to a destination is permissible or not. For instance... |- can_exit_to - check if exiting to this destination is allowed or not |- is_exiting_allowed - check if any exiting is allowed |- summary - provides a short label, similar to a microdescriptor + |- is_default - checks if policy ends with the defaultly appended suffix |- strip_private - provides a copy of the policy without 'private' entries + |- strip_default - provides a copy of the policy without the default suffix |- __str__ - string representation +- __iter__ - ExitPolicyRule entries that this contains @@ -44,6 +46,7 @@ exiting to a destination is permissible or not. For instance... |- get_mask - provides the address representation of our mask |- get_masked_bits - provides the bit representation of our mask |- is_private - flag indicating if this was expanded from a 'private' keyword + |- is_default - flag indicating if this was part of the default end of a policy +- __str__ - string representation for this rule get_config_policy - provides the ExitPolicy based on torrc rules @@ -200,6 +203,19 @@ def _flag_private_rules(rules): rule._is_private = True +def _flag_default_rules(rules): + """ + Determine if part of our policy ends with the defaultly appended suffix. + """ + + if len(rules) >= len(DEFAULT_POLICY_RULES): + rules_suffix = tuple(rules[-len(DEFAULT_POLICY_RULES):]) + + if rules_suffix == DEFAULT_POLICY_RULES: + for rule in rules_suffix: + rule._is_default_suffix = True + + class ExitPolicy(object): """ Policy for the destinations that a relay allows or denies exiting to. This @@ -365,6 +381,19 @@ class ExitPolicy(object): return (label_prefix + ', '.join(display_ranges)).strip() + def is_default(self): + """ + Checks if we have the default policy suffix. + + :returns: **True** if we have the default policy suffix, **False** otherwise + """ + + for rule in self._get_rules(): + if rule.is_default(): + return True + + return False + def strip_private(self): """ Provides a copy of this policy without 'private' policy entries. @@ -374,6 +403,15 @@ class ExitPolicy(object): return ExitPolicy(*[rule for rule in self._get_rules() if not rule.is_private()]) + def strip_default(self): + """ + Provides a copy of this policy without the default policy suffix. + + :returns: **ExitPolicy** without default rules + """ + + return ExitPolicy(*[rule for rule in self._get_rules() if not rule.is_default()]) + def _get_rules(self): if self._rules is None: rules = [] @@ -421,12 +459,16 @@ class ExitPolicy(object): rules = [ExitPolicyRule('reject *:*')] _flag_private_rules(rules) + _flag_default_rules(rules) self._rules = rules self._input_rules = None return self._rules + def __len__(self): + return len(self._get_rules()) + def __iter__(self): for rule in self._get_rules(): yield rule @@ -610,7 +652,7 @@ class ExitPolicyRule(object): # keyword or tor's default policy suffix. self._is_private = False - self._is_default_suffix = False # TODO: implement + self._is_default_suffix = False def is_address_wildcard(self): """ @@ -754,6 +796,13 @@ class ExitPolicyRule(object): return self._is_private + def is_default(self): + """ + True if this rule was part of the default end of a policy, False otherwise. + """ + + return self._is_default_suffix + @lru_cache() def __str__(self): """ @@ -979,3 +1028,18 @@ class MicroExitPolicyRule(ExitPolicyRule): self._hash = my_hash return self._hash + + +DEFAULT_POLICY_RULES = tuple([ExitPolicyRule(rule) for rule in ( + 'reject *:25', + 'reject *:119', + 'reject *:135-139', + 'reject *:445', + 'reject *:563', + 'reject *:1214', + 'reject *:4661-4666', + 'reject *:6346-6429', + 'reject *:6699', + 'reject *:6881-6999', + 'accept *:*', +)]) diff --git a/test/unit/exit_policy/policy.py b/test/unit/exit_policy/policy.py index b4a35e5..a1ee390 100644 --- a/test/unit/exit_policy/policy.py +++ b/test/unit/exit_policy/policy.py @@ -6,6 +6,7 @@ import pickle import unittest from stem.exit_policy import ( + DEFAULT_POLICY_RULES, get_config_policy, ExitPolicy, MicroExitPolicy, @@ -96,6 +97,18 @@ class TestExitPolicy(unittest.TestCase): policy = ExitPolicy('reject *:80-65535', 'accept *:1-65533', 'reject *:*') self.assertEquals('accept 1-79', policy.summary()) + def test_non_private_non_default_policy(self): + policy = get_config_policy('reject *:80-65535, accept *:1-65533, reject *:*') + + for rule in policy: + self.assertFalse(rule.is_private()) + self.assertFalse(rule.is_default()) + + self.assertFalse(policy.is_default()) + + self.assertEqual(policy, policy.strip_private()) + self.assertEqual(policy, policy.strip_default()) + def test_all_private_policy(self): for port in ('*', '80', '1-1024'): private_policy = get_config_policy('reject private:%s' % port) @@ -110,13 +123,14 @@ class TestExitPolicy(unittest.TestCase): private_policy = get_config_policy('accept private:*') self.assertEqual(ExitPolicy(), private_policy.strip_private()) - def test_all_non_private_policy(self): - nonprivate_policy = get_config_policy('reject *:80-65535, accept *:1-65533, reject *:*') + def test_all_default_policy(self): + policy = ExitPolicy(*DEFAULT_POLICY_RULES) - for rule in nonprivate_policy: - self.assertFalse(rule.is_private()) + for rule in policy: + self.assertTrue(rule.is_default()) - self.assertEqual(nonprivate_policy, nonprivate_policy.strip_private()) + self.assertTrue(policy.is_default()) + self.assertEqual(ExitPolicy(), policy.strip_default()) def test_mixed_private_policy(self): policy = get_config_policy('accept *:80, reject private:1-65533, accept *:*') @@ -126,6 +140,15 @@ class TestExitPolicy(unittest.TestCase): self.assertEqual(get_config_policy('accept *:80, accept *:*'), policy.strip_private()) + def test_mixed_default_policy(self): + policy = ExitPolicy('accept *:80', 'accept 127.0.0.1:1-65533', *DEFAULT_POLICY_RULES) + + for rule in policy: + # only accept-all and reject rules are the default ones + self.assertTrue(rule.is_accept != rule.is_default() or (rule.is_accept and rule.is_address_wildcard() and rule.is_port_wildcard())) + + self.assertEqual(get_config_policy('accept *:80, accept 127.0.0.1:1-65533'), policy.strip_default()) + def test_str(self): # sanity test for our __str__ method

1 0

[stem/master] Better ExitPolicy handling for the 'private' keyword and default suffix
by atagar＠torproject.org 07 Sep '14

07 Sep '14

commit 09940b6f3c0fa2f883cf07a144d7e319511b8d0e Merge: 54de59a e0aeefb Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Sep 7 12:02:30 2014 -0700 Better ExitPolicy handling for the 'private' keyword and default suffix While working on ticket #10067 and arm I realized that it was clunky dealing with a couple exit policy oddities... * Policies tor's 'private' keyword expands to. These prefix our policy by default and includes our public address, so it makes all exit policies unique. Often we want to strip this off to get what the user *really* specified. * If the policy lacks an accept-all or reject-all rule at the end then tor appends a default policy. This is for... https://trac.torproject.org/projects/tor/ticket/10107 With it our script in... https://trac.torproject.org/projects/tor/ticket/10067#comment:9 ... becomes *much* simpler... from stem.descriptor import remote policy_counts = {} for desc in remote.DescriptorDownloader().get_server_descriptors(): policy = str(desc.exit_policy.strip_private().strip_default()) if desc.exit_policy.is_default(): policy += ', default' if policy else 'default' policy_counts[policy] = policy_counts.setdefault(policy, 0) + 1 # exit policies sorted by their count counts = sorted(policy_counts.values(), reverse = True) for count in counts: for policy, policy_count in policy_counts.items(): if count == policy_count: print "%i %s" % (count, policy) docs/change_log.rst | 1 + stem/control.py | 7 +- stem/exit_policy.py | 172 +++++++++++++++++++++++++++++++++++++-- test/unit/exit_policy/policy.py | 56 ++++++++++++- 4 files changed, 222 insertions(+), 14 deletions(-)

1 0

[stem/master] ExitPolicy's 'private' keyword handling didn't account for our public address
by atagar＠torproject.org 07 Sep '14

07 Sep '14

commit c5e5d1ac2df52631d542d03d2f5c35db6a84f225 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Sep 5 08:34:43 2014 -0700 ExitPolicy's 'private' keyword handling didn't account for our public address Our Controller included a rule for our public ip address, but the exit_policy module didn't. Moving this functionality down to that. --- stem/control.py | 7 +------ stem/exit_policy.py | 35 +++++++++++++++++++++++++++-------- test/unit/exit_policy/policy.py | 3 ++- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/stem/control.py b/stem/control.py index 58dd119..1b33659 100644 --- a/stem/control.py +++ b/stem/control.py @@ -1033,17 +1033,12 @@ class Controller(BaseController): if self.get_conf('ExitPolicyRejectPrivate') == '1': policy.append('reject private:*') - public_addr = self.get_info('address', None) - - if public_addr: - policy.append('reject %s:*' % public_addr) - for policy_line in self.get_conf('ExitPolicy', multiple = True): policy += policy_line.split(',') policy += self.get_info('exit-policy/default').split(',') - config_policy = stem.exit_policy.get_config_policy(policy) + config_policy = stem.exit_policy.get_config_policy(policy, self.get_info('address')) self._set_cache({'exit_policy': config_policy}) return config_policy diff --git a/stem/exit_policy.py b/stem/exit_policy.py index 2cade1d..86722af 100644 --- a/stem/exit_policy.py +++ b/stem/exit_policy.py @@ -61,6 +61,9 @@ exiting to a destination is permissible or not. For instance... ============ =========== """ +from __future__ import absolute_import + +import socket import zlib import stem.prereq @@ -92,7 +95,7 @@ PRIVATE_ADDRESSES = ( ) -def get_config_policy(rules): +def get_config_policy(rules, ip_address = None): """ Converts an ExitPolicy found in a torrc to a proper exit pattern. This accounts for... @@ -101,12 +104,17 @@ def get_config_policy(rules): * the 'private' keyword :param str,list rules: comma separated rules or list to be converted + :param str ip_address: this relay's IP address for the 'private' policy if + it's present, this defaults to the local address :returns: :class:`~stem.exit_policy.ExitPolicy` reflected by the rules :raises: **ValueError** if input isn't a valid tor exit policy """ + if ip_address and not (stem.util.connection.is_valid_ipv4_address(ip_address) or stem.util.connection.is_valid_ipv6_address(ip_address)): + raise ValueError("%s isn't a valid IP address" % ip_address) + if isinstance(rules, (bytes, unicode)): rules = rules.split(',') @@ -125,7 +133,10 @@ def get_config_policy(rules): acceptance = rule.split(' ', 1)[0] port = rule.split(':', 1)[1] - for private_addr in PRIVATE_ADDRESSES: + if ip_address is None: + ip_address = socket.gethostbyname(socket.gethostname()) + + for private_addr in PRIVATE_ADDRESSES + (ip_address,): result.append(ExitPolicyRule("%s %s:%s" % (acceptance, private_addr, port))) else: result.append(ExitPolicyRule(rule)) @@ -149,7 +160,7 @@ def _flag_private_rules(rules): matches = [] for i, rule in enumerate(rules): - if i + len(PRIVATE_ADDRESSES) > len(rules): + if i + len(PRIVATE_ADDRESSES) + 1 > len(rules): break rule_str = '%s/%s' % (rule.address, rule.get_masked_bits()) @@ -161,21 +172,29 @@ def _flag_private_rules(rules): # To match the private policy the following must all be true... # # * series of addresses and bit masks match PRIVATE_ADDRESSES - # * all these rules have the same port range and acceptance - # * all these rules must be either accept or reject entries + # * all rules have the same port range and acceptance + # * all rules have the same acceptance (all accept or reject entries) + + rule_set = rules[start_index:start_index + len(PRIVATE_ADDRESSES) + 1] + is_match = True - rule_set = rules[start_index:start_index + len(PRIVATE_ADDRESSES)] min_port, max_port = rule_set[0].min_port, rule_set[0].max_port is_accept = rule_set[0].is_accept - is_match = True - for i, rule in enumerate(rule_set): + for i, rule in enumerate(rule_set[:-1]): rule_str = '%s/%s' % (rule.address, rule.get_masked_bits()) if rule_str != PRIVATE_ADDRESSES[i] or rule.min_port != min_port or rule.max_port != max_port or rule.is_accept != is_accept: is_match = False break + # The last rule is for the relay's public address, so it's dynamic. + + last_rule = rule_set[-1] + + if last_rule.is_address_wildcard() or last_rule.min_port != min_port or last_rule.max_port != max_port or last_rule.is_accept != is_accept: + is_match = False + if is_match: for rule in rule_set: rule._is_private = True diff --git a/test/unit/exit_policy/policy.py b/test/unit/exit_policy/policy.py index d00954d..b4a35e5 100644 --- a/test/unit/exit_policy/policy.py +++ b/test/unit/exit_policy/policy.py @@ -225,6 +225,7 @@ class TestExitPolicy(unittest.TestCase): 'reject 192.168.0.0/16:*', 'reject 10.0.0.0/8:*', 'reject 172.16.0.0/12:*', + 'reject 12.34.56.78:*', ), 'accept *:80, reject *': ExitPolicy( 'accept *:80', @@ -237,7 +238,7 @@ class TestExitPolicy(unittest.TestCase): } for test_input, expected in test_inputs.items(): - self.assertEqual(expected, get_config_policy(test_input)) + self.assertEqual(expected, get_config_policy(test_input, '12.34.56.78')) test_inputs = ( 'blarg',

1 0

[stem/master] Proper hash function for ExitPolicy
by atagar＠torproject.org 07 Sep '14

07 Sep '14

commit d76ced9fbc1408472de5a08b15789bbb1bccc6c0 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Sep 5 08:52:37 2014 -0700 Proper hash function for ExitPolicy We skipped on implementing hash() out of concern of a circular dependency from @lru_cache on _get_rules(). But _get_rules() no longer uses @lru_cache there due to pickleability so we can now implement hash(). --- stem/exit_policy.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/stem/exit_policy.py b/stem/exit_policy.py index 86722af..1f72b60 100644 --- a/stem/exit_policy.py +++ b/stem/exit_policy.py @@ -235,6 +235,7 @@ class ExitPolicy(object): self._input_rules = rules self._rules = None + self._hash = None # Result when no rules apply. According to the spec policies default to 'is # allowed', but our microdescriptor policy subclass might want to change @@ -435,12 +436,16 @@ class ExitPolicy(object): return ', '.join([str(rule) for rule in self._get_rules()]) def __hash__(self): - # TODO: It would be nice to provide a real hash function, but doing so is - # tricky due to how we lazily load the rules. Like equality checks a proper - # hash function would need to call _get_rules(), but that's behind - # @lru_cache which calls hash() forming a circular dependency. + if self._hash is None: + my_hash = 0 + + for rule in self._get_rules(): + my_hash *= 1024 + my_hash += hash(rule) + + self._hash = my_hash - return id(self) + return self._hash def __eq__(self, other): if isinstance(other, ExitPolicy):

1 0