February 2013 - tor-commits - lists.torproject.org

[translation/torbirdy] Update translations for torbirdy
by translation＠torproject.org 04 Feb '13

04 Feb '13

commit 1b7147d5b04e6ec874b77cc3e597801532d646a3 Author: Translation commit bot <translation(a)torproject.org> Date: Mon Feb 4 09:16:25 2013 +0000 Update translations for torbirdy --- eu/torbirdy.dtd | 2 +- eu/torbirdy.properties | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/eu/torbirdy.dtd b/eu/torbirdy.dtd index fec85b0..a234f0a 100644 --- a/eu/torbirdy.dtd +++ b/eu/torbirdy.dtd @@ -43,5 +43,5 @@ <!ENTITY torbirdy.panel.usetor.… [View More]

1 0

[metrics-tasks/master] Revise stem parsing (thanks atagar) (#7241).
by karsten＠torproject.org 04 Feb '13

04 Feb '13

commit c58607d29ac304d08d4d51cf0bd597ebd9779df3 Author: peer <peer(a)lavabit.com> Date: Sun Jan 20 03:37:46 2013 +0000 Revise stem parsing (thanks atagar) (#7241). --- task-7241/first_pass.py | 15 ++++++--------- 1 files changed, 6 insertions(+), 9 deletions(-) diff --git a/task-7241/first_pass.py b/task-7241/first_pass.py index f3f9707..b453b84 100644 --- a/task-7241/first_pass.py +++ b/task-7241/first_pass.py @@ -1,7 +1,7 @@ import sys from datetime import datetime, … [View More]

1 0

[metrics-tasks/master] Revise calculation script following code review (thanks atagar) (#7241).
by karsten＠torproject.org 04 Feb '13

04 Feb '13

commit 1625bbe6cb482d8ec599362fefdd99cf06a1c66c Author: peer <peer(a)lavabit.com> Date: Tue Jan 22 11:53:57 2013 +0000 Revise calculation script following code review (thanks atagar) (#7241). --- task-7241/first_pass.py | 138 +++++++++++++++++++++------------------------- 1 files changed, 63 insertions(+), 75 deletions(-) diff --git a/task-7241/first_pass.py b/task-7241/first_pass.py index b453b84..457a4f7 100644 --- a/task-7241/first_pass.py +++ b/task-7241/first_pass.py @@ -1,… [View More]106 +1,94 @@ -import sys -from datetime import datetime, timedelta +# calculate frac_relays, frac_cw to compare consensus documents over time -from stem.descriptor import parse_file +# let Y be the base document and X be some hours before the base document +# frac_relays = count(intersection(Y, X)) / count(Y) +# frac_cw = sum(cw(Y) over intersection(Y,X)) / sum(cw(Y)) -# http://stackoverflow.com/questions/82831/how-do-i-check-if-a-file-exists-us… -def file_check(file_path): - try: - with open(file_path) as f: - return True - except IOError: - return False +import os +from datetime import datetime, timedelta +from stem.descriptor import parse_file +# generate expected consensus filepath from time def filepath_from_time(cur_datetime): - consensus_path = 'consensuses-' - consensus_path += cur_datetime.strftime('%Y-%m') - consensus_path += '/' - consensus_path += cur_datetime.strftime('%d') - consensus_path += '/' - consensus_path += cur_datetime.strftime('%Y-%m-%d-%H-%M-%S') - consensus_path += '-consensus' - - return consensus_path + return os.path.join( + 'consensuses-%s' % cur_datetime.strftime('%Y-%m'), + cur_datetime.strftime('%d'), + '%s-consensus' % cur_datetime.strftime('%Y-%m-%d-%H-%M-%S'), + ) -def filename_from_time(cur_datetime): - consensus_filename = cur_datetime.strftime('%Y-%m-%d-%H-%M-%S') - consensus_filename += '-consensus' - - return consensus_filename +# router bw storage by fingerprint +router_data = {} +# unit time interval time_interval = timedelta(0, 60*60) # one hour +# interval multipliers for analysis: 1 hour to 7 days +time_interval_list = [1,2,3,4,5,6,12,24,36,48,72,96,120,144,168] # hours + # base consensuses for examination initial_time_info_bound = datetime(2012, 1, 1) # inclusive final_time_info_bound = datetime(2013, 1, 1) # exclusive -router_data = {} - # data range for consensuses initial_time_data_bound = datetime(2011, 12, 1) # inclusive final_time_data_bound = datetime(2013, 1, 1) # exclusive # load information -cur_datetime = initial_time_data_bound - time_interval -while cur_datetime < final_time_data_bound - time_interval: - cur_datetime += time_interval - +cur_datetime = initial_time_data_bound +while cur_datetime < final_time_data_bound: cur_filepath = filepath_from_time(cur_datetime) - cur_filename = filename_from_time(cur_datetime) - - if file_check(cur_filepath) == True: - routers = {} + cur_filename = os.path.basename(cur_filepath) + try: with open(cur_filepath) as consensus_file: - for router in parse_file(consensus_file): - routers[router.fingerprint] = router.bandwidth - - router_data[cur_filename] = routers - -# interval multipliers -time_interval_list = [1,2,3,4,5,6,12,24,36,48,72,96,120,144,168] # hours + router_data[cur_filename] = dict([(r.fingerprint, r.bandwidth) + for r in parse_file(consensus_file)]) + except IOError: + pass # file does not exist (possible situation) and iterate -# iterate over base consensuses -cur_datetime = initial_time_info_bound - time_interval -while cur_datetime < final_time_info_bound - time_interval: + # next file to read cur_datetime += time_interval +# iterate over base consensuses for frac_relays, frac_cw +cur_datetime = initial_time_info_bound +while cur_datetime < final_time_info_bound: cur_filepath = filepath_from_time(cur_datetime) # current - cur_filename = filename_from_time(cur_datetime) # current + cur_filename = os.path.basename(cur_filepath) # current - if file_check(cur_filepath) == True: + # find base data, if data exists + if cur_filename in router_data: base_routers = router_data[cur_filename] - base_router_count = 0 - base_router_bandwidth = 0 - for fingerprint in router_data[cur_filename].keys(): - base_router_count += 1 - base_router_bandwidth += router_data[cur_filename][fingerprint] - - for comparison_time_interval_multiplier in time_interval_list: - comparison_time_interval = timedelta(0, comparison_time_interval_multiplier*60*60) - comparison_datetime = cur_datetime - comparison_time_interval - - comparison_filepath = filepath_from_time(comparison_datetime) # comparison - comparison_filename = filename_from_time(comparison_datetime) # comparison - - if file_check(comparison_filepath) == True: - comparison_router_count = 0 - comparison_router_bandwidth = 0 - comparison_router_overlap_bandwidth = 0 - base_router_overlap_bandwidth = 0 - comparison_router_overlap_count = 0 - - for fingerprint in router_data[comparison_filename].keys(): - comparison_router_count += 1 - comparison_router_bandwidth += router_data[comparison_filename][fingerprint] + base_router_count = len(router_data[cur_filename]) + base_router_bw = sum(router_data[cur_filename].values()) + + # for each analysis analysis interval, find comparison locator + for time_interval_multiplier in time_interval_list: + comp_time_interval = time_interval_multiplier*time_interval + comp_datetime = cur_datetime - comp_time_interval + + comp_filepath = filepath_from_time(comp_datetime) # comp + comp_filename = os.path.basename(comp_filepath) # comp + # find comparison data, if data exists + if comp_filename in router_data: + router_overlap_count = 0 + base_router_overlap_bw = 0 + + # determine intersection(Y,X) and sum cw over intersection(Y,X) + for fingerprint in router_data[comp_filename]: if fingerprint in base_routers: - base_router_overlap_bandwidth += base_routers[fingerprint] - comparison_router_overlap_count += 1 - comparison_router_overlap_bandwidth += router_data[comparison_filename][fingerprint] + router_overlap_count += 1 + base_router_overlap_bw += base_routers[fingerprint] + + # determine ratios + frac_relays = float(router_overlap_count)/float(base_router_count) + frac_cw = float(base_router_overlap_bw)/float(base_router_bw) - frac_relays = float(comparison_router_overlap_count)/float(base_router_count) - frac_cw = float(base_router_overlap_bandwidth)/float(base_router_bandwidth) + # output + print '%s,%d,%f,%f,%d,%d,%s' % (cur_filename, time_interval_multiplier, + frac_relays, frac_cw, cur_datetime.month, cur_datetime.day, + cur_datetime.strftime('%w')) - print '%s,%d,%f,%f,%d,%d,%s' % (cur_filename, comparison_time_interval_multiplier, frac_relays, frac_cw, cur_datetime.month, cur_datetime.day, cur_datetime.strftime('%w')) + # next base consensus + cur_datetime += time_interval [View Less]

1 0

[stem/master] Allowing for IPv4 'a' lines in router status entries
by atagar＠torproject.org 04 Feb '13

04 Feb '13

commit c6a9cde0f4eb5c627f5ba41f9d263be0c5854ae8 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Feb 3 22:14:20 2013 -0800 Allowing for IPv4 'a' lines in router status entries Karsten reports on ticket #8036 that IPv4 addresses are indeed allowed on a router status entry's 'a' line. This is a little unfortunate since it means a less friendly attribute but not a big whoop. --- stem/descriptor/router_status_entry.py | 34 +++++++++++++++----------- … [View More]stem/descriptor/server_descriptor.py | 5 +-- test/unit/descriptor/router_status_entry.py | 19 ++++++++------- 3 files changed, 32 insertions(+), 26 deletions(-) diff --git a/stem/descriptor/router_status_entry.py b/stem/descriptor/router_status_entry.py index 2cd6632..d22dfa3 100644 --- a/stem/descriptor/router_status_entry.py +++ b/stem/descriptor/router_status_entry.py @@ -298,8 +298,8 @@ class RouterStatusEntryV3(RouterStatusEntry): Information about an individual router stored within a version 3 network status document. - :var dict addresses_v6: **\*** relay's IPv6 OR addresses, this is a mapping - of IPv6 addresses to a tuple with the ports it accepts + :var list addresses_v6: **\*** relay's OR addresses, this is a tuple listing + of the form (address (**str**), port (**int**), is_ipv6 (**bool**)) :var str digest: **\*** router's digest :var int bandwidth: bandwidth claimed by the relay (in kb/s) @@ -318,7 +318,7 @@ class RouterStatusEntryV3(RouterStatusEntry): """ def __init__(self, content, validate = True, document = None): - self.addresses_v6 = {} + self.addresses_v6 = [] self.digest = None self.bandwidth = None @@ -521,21 +521,27 @@ def _parse_a_line(desc, value, validate): raise ValueError("%s 'a' line must be of the form '[address]:[ports]': a %s" % (desc._name(), value)) - address, port_label = value.rsplit(':', 1) + address, ports = value.rsplit(':', 1) + is_ipv6 = address.startswith("[") and address.endswith("]") - if validate and not stem.util.connection.is_valid_ipv6_address(address, allow_brackets = True): - raise ValueError("%s 'a' line must start with an IPv6 address: a %s" % (desc._name(), value)) + if is_ipv6: + address = address[1:-1] # remove brackets - address = address.lstrip('[').rstrip(']') - ports = [] + if not ((not is_ipv6 and stem.util.connection.is_valid_ip_address(address)) or + (is_ipv6 and stem.util.connection.is_valid_ipv6_address(address))): + if not validate: + return + else: + raise ValueError("%s 'a' line must start with an IPv6 address: a %s" % (desc._name(), value)) - for port in port_label.split(','): - if stem.util.connection.is_valid_port(port): - ports.append(int(port)) - elif validate: - raise ValueError("%s 'a' line had an invalid port (%s): a %s" % (desc._name(), port, value)) + for port in ports.split(','): + if not stem.util.connection.is_valid_port(port): + if not validate: + continue + else: + raise ValueError("%s 'a' line had an invalid port (%s): a %s" % (desc._name(), port, value)) - desc.addresses_v6[address] = tuple(ports) + desc.addresses_v6.append((address, int(port), is_ipv6)) def _parse_s_line(desc, value, validate): diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 95a4ef0..be63ddf 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -514,8 +514,7 @@ class ServerDescriptor(stem.descriptor.Descriptor): else: raise ValueError("or-address line missing a colon: %s" % line) - div = entry.rfind(":") - address, ports = entry[:div], entry[div + 1:] + address, ports = entry.rsplit(':', 1) is_ipv6 = address.startswith("[") and address.endswith("]") if is_ipv6: @@ -531,7 +530,7 @@ class ServerDescriptor(stem.descriptor.Descriptor): for port in ports.split(","): if not stem.util.connection.is_valid_port(port): if not validate: - break + continue else: raise ValueError("or-address line has malformed ports: %s" % line) diff --git a/test/unit/descriptor/router_status_entry.py b/test/unit/descriptor/router_status_entry.py index b9d3649..b994f23 100644 --- a/test/unit/descriptor/router_status_entry.py +++ b/test/unit/descriptor/router_status_entry.py @@ -312,10 +312,11 @@ class TestRouterStatusEntry(unittest.TestCase): """ test_values = { - "[2607:fcd0:daaa:101::602c:bd62]:443": { - '2607:fcd0:daaa:101::602c:bd62': (443,)}, - "[2607:fcd0:daaa:101::602c:bd62]:80,443": { - '2607:fcd0:daaa:101::602c:bd62': (80, 443)}, + "[2607:fcd0:daaa:101::602c:bd62]:443": [ + ('2607:fcd0:daaa:101::602c:bd62', 443, True)], + "[2607:fcd0:daaa:101::602c:bd62]:80,443": [ + ('2607:fcd0:daaa:101::602c:bd62', 80, True), + ('2607:fcd0:daaa:101::602c:bd62', 443, True)] } for a_line, expected in test_values.items(): @@ -328,10 +329,11 @@ class TestRouterStatusEntry(unittest.TestCase): content += "\na [2607:fcd0:daaa:101::602c:bd62]:80,443" content += "\na [1148:fcd0:daaa:101::602c:bd62]:80" - expected = { - '2607:fcd0:daaa:101::602c:bd62': (80, 443), - '1148:fcd0:daaa:101::602c:bd62': (80,), - } + expected = [ + ('2607:fcd0:daaa:101::602c:bd62', 80, True), + ('2607:fcd0:daaa:101::602c:bd62', 443, True), + ('1148:fcd0:daaa:101::602c:bd62', 80, True), + ] entry = RouterStatusEntryV3(content) self.assertEquals(expected, entry.addresses_v6) @@ -340,7 +342,6 @@ class TestRouterStatusEntry(unittest.TestCase): test_values = ( "", - "127.0.0.1:80", "[1148:fcd0:daaa:101::602c:bd62]:80000", ) [View Less]

1 0

[stem/master] Try to make minor descriptor versions clearer.
by atagar＠torproject.org 04 Feb '13

04 Feb '13

commit b8baf77c13154ccb6c43299a36979d812ca7b8be Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Feb 4 06:17:28 2013 +0100 Try to make minor descriptor versions clearer. --- stem/descriptor/__init__.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index fafdd8a..669485c 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -87,7 +87,8 @@ def parse_file(… [View More]

1 0

[stem/master] Using port lists for addresses_v6 rather than ranges
by atagar＠torproject.org 04 Feb '13

04 Feb '13

commit 6c99a28e83490537615de9388484c574aa1b85dd Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Feb 3 20:43:28 2013 -0800 Using port lists for addresses_v6 rather than ranges Huh, I wonder where I got the idea that 'a' lines had port ranges. Dropping that. According to the spec the 'a' lines should be parsed in the same way as 'or-address'. However, I suspect that the spec is a little off here - checking if it can contain IPv4 addresses... … [View More]Caught by Karsten on... https://trac.torproject.org/8036 --- stem/descriptor/router_status_entry.py | 25 +++++++++---------------- test/unit/descriptor/router_status_entry.py | 11 ++++------- 2 files changed, 13 insertions(+), 23 deletions(-) diff --git a/stem/descriptor/router_status_entry.py b/stem/descriptor/router_status_entry.py index ce5a7db..2cd6632 100644 --- a/stem/descriptor/router_status_entry.py +++ b/stem/descriptor/router_status_entry.py @@ -299,7 +299,7 @@ class RouterStatusEntryV3(RouterStatusEntry): status document. :var dict addresses_v6: **\*** relay's IPv6 OR addresses, this is a mapping - of IPv6 addresses to a listing of [(min port, max port)...] it accepts + of IPv6 addresses to a tuple with the ports it accepts :var str digest: **\*** router's digest :var int bandwidth: bandwidth claimed by the relay (in kb/s) @@ -521,28 +521,21 @@ def _parse_a_line(desc, value, validate): raise ValueError("%s 'a' line must be of the form '[address]:[ports]': a %s" % (desc._name(), value)) - address, ports = value.rsplit(':', 1) + address, port_label = value.rsplit(':', 1) if validate and not stem.util.connection.is_valid_ipv6_address(address, allow_brackets = True): raise ValueError("%s 'a' line must start with an IPv6 address: a %s" % (desc._name(), value)) address = address.lstrip('[').rstrip(']') + ports = [] - for port_entry in ports.split(','): - if '-' in port_entry: - min_port, max_port = port_entry.split('-', 1) - else: - min_port = max_port = port_entry - - if not stem.util.connection.is_valid_port(min_port) or \ - not stem.util.connection.is_valid_port(max_port): - - if not validate: - continue - - raise ValueError("%s 'a' line had an invalid port range (%s): a %s" % (desc._name(), port_entry, value)) + for port in port_label.split(','): + if stem.util.connection.is_valid_port(port): + ports.append(int(port)) + elif validate: + raise ValueError("%s 'a' line had an invalid port (%s): a %s" % (desc._name(), port, value)) - desc.addresses_v6.setdefault(address, []).append((int(min_port), int(max_port))) + desc.addresses_v6[address] = tuple(ports) def _parse_s_line(desc, value, validate): diff --git a/test/unit/descriptor/router_status_entry.py b/test/unit/descriptor/router_status_entry.py index 21795f2..b9d3649 100644 --- a/test/unit/descriptor/router_status_entry.py +++ b/test/unit/descriptor/router_status_entry.py @@ -313,11 +313,9 @@ class TestRouterStatusEntry(unittest.TestCase): test_values = { "[2607:fcd0:daaa:101::602c:bd62]:443": { - '2607:fcd0:daaa:101::602c:bd62': [(443, 443)]}, + '2607:fcd0:daaa:101::602c:bd62': (443,)}, "[2607:fcd0:daaa:101::602c:bd62]:80,443": { - '2607:fcd0:daaa:101::602c:bd62': [(80, 80), (443, 443)]}, - "[2607:fcd0:daaa:101::602c:bd62]:443-512": { - '2607:fcd0:daaa:101::602c:bd62': [(443, 512)]}, + '2607:fcd0:daaa:101::602c:bd62': (80, 443)}, } for a_line, expected in test_values.items(): @@ -328,12 +326,11 @@ class TestRouterStatusEntry(unittest.TestCase): content = get_router_status_entry_v3(content = True) content += "\na [2607:fcd0:daaa:101::602c:bd62]:80,443" - content += "\na [2607:fcd0:daaa:101::602c:bd62]:512-600" content += "\na [1148:fcd0:daaa:101::602c:bd62]:80" expected = { - '2607:fcd0:daaa:101::602c:bd62': [(80, 80), (443, 443), (512, 600)], - '1148:fcd0:daaa:101::602c:bd62': [(80, 80)], + '2607:fcd0:daaa:101::602c:bd62': (80, 443), + '1148:fcd0:daaa:101::602c:bd62': (80,), } entry = RouterStatusEntryV3(content) [View Less]

1 0

[stem/master] Renaming check_whitespace.py to static_checks.py
by atagar＠torproject.org 04 Feb '13

04 Feb '13

commit d44018a5566fe5c92a326a413768bdd8462c425e Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Feb 3 20:01:35 2013 -0800 Renaming check_whitespace.py to static_checks.py The check_whitespace.py module no longer... well, checks whitespace. Rather, it has become a dumping ground for all of the static checks that we do. Renaming it to something more appropriate. --- run_tests.py | 22 ++-- test/__init__.py | 2 +- test/… [View More]check_whitespace.py | 229 ---------------------------------------------- test/static_checks.py | 229 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 241 insertions(+), 241 deletions(-) diff --git a/run_tests.py b/run_tests.py index feab3a7..1154870 100755 --- a/run_tests.py +++ b/run_tests.py @@ -19,9 +19,9 @@ import stem.util.enum from stem.util import log, system, term -import test.check_whitespace import test.output import test.runner +import test.static_checks import test.unit.connection.authentication import test.unit.control.controller import test.unit.descriptor.export @@ -276,7 +276,7 @@ def _clean_orphaned_pyc(): orphaned_pyc = [] for base_dir in ('stem', 'test', 'run_tests.py'): - for pyc_path in test.check_whitespace._get_files_with_suffix(base_dir, ".pyc"): + for pyc_path in test.static_checks._get_files_with_suffix(base_dir, ".pyc"): # If we're running python 3 then the *.pyc files are no longer bundled # with the *.py. Rather, they're in a __pycache__ directory. # @@ -350,9 +350,9 @@ def _python3_setup(python3_destination): def _print_style_issues(): base_path = os.path.sep.join(__file__.split(os.path.sep)[:-1]).lstrip("./") - style_issues = test.check_whitespace.get_issues(os.path.join(base_path, "stem")) - style_issues.update(test.check_whitespace.get_issues(os.path.join(base_path, "test"))) - style_issues.update(test.check_whitespace.get_issues(os.path.join(base_path, "run_tests.py"))) + style_issues = test.static_checks.get_issues(os.path.join(base_path, "stem")) + style_issues.update(test.static_checks.get_issues(os.path.join(base_path, "test"))) + style_issues.update(test.static_checks.get_issues(os.path.join(base_path, "run_tests.py"))) # If we're doing some sort of testing (unit or integ) and pyflakes is # available then use it. Its static checks are pretty quick so there's not @@ -360,17 +360,17 @@ def _print_style_issues(): if CONFIG["argument.unit"] or CONFIG["argument.integ"]: if system.is_available("pyflakes"): - style_issues.update(test.check_whitespace.pyflakes_issues(os.path.join(base_path, "stem"))) - style_issues.update(test.check_whitespace.pyflakes_issues(os.path.join(base_path, "test"))) - style_issues.update(test.check_whitespace.pyflakes_issues(os.path.join(base_path, "run_tests.py"))) + style_issues.update(test.static_checks.pyflakes_issues(os.path.join(base_path, "stem"))) + style_issues.update(test.static_checks.pyflakes_issues(os.path.join(base_path, "test"))) + style_issues.update(test.static_checks.pyflakes_issues(os.path.join(base_path, "run_tests.py"))) else: test.output.print_line("Static error checking requires pyflakes. Please install it from ...\n http://pypi.python.org/pypi/pyflakes\n", *ERROR_ATTR) if CONFIG["argument.style"]: if system.is_available("pep8"): - style_issues.update(test.check_whitespace.pep8_issues(os.path.join(base_path, "stem"))) - style_issues.update(test.check_whitespace.pep8_issues(os.path.join(base_path, "test"))) - style_issues.update(test.check_whitespace.pep8_issues(os.path.join(base_path, "run_tests.py"))) + style_issues.update(test.static_checks.pep8_issues(os.path.join(base_path, "stem"))) + style_issues.update(test.static_checks.pep8_issues(os.path.join(base_path, "test"))) + style_issues.update(test.static_checks.pep8_issues(os.path.join(base_path, "run_tests.py"))) else: test.output.print_line("Style checks require pep8. Please install it from...\n http://pypi.python.org/pypi/pep8\n", *ERROR_ATTR) diff --git a/test/__init__.py b/test/__init__.py index 43fdd7f..098ce65 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -3,12 +3,12 @@ Unit and integration tests for the stem library. """ __all__ = [ - "check_whitespace", "mocking", "network", "output", "prompt", "runner", + "static_checks", "tutorial", "utils", ] diff --git a/test/check_whitespace.py b/test/check_whitespace.py deleted file mode 100644 index 2f758b8..0000000 --- a/test/check_whitespace.py +++ /dev/null @@ -1,229 +0,0 @@ -""" -Performs a check that our python source code follows its whitespace conventions -which are... - -* two space indentations -* tabs are the root of all evil and should be shot on sight -* standard newlines (\\n), not windows (\\r\\n) nor classic mac (\\r) - -This also checks for 2.5 compatibility issues (yea, they're not whitespace but -it's so much easier to do here...): - -* checks that anything using the 'with' keyword has... - from __future__ import with_statement -""" - -from __future__ import with_statement - -import re -import os - -from stem.util import conf, system - -# if ran directly then run over everything one level up -DEFAULT_TARGET = os.path.sep.join(__file__.split(os.path.sep)[:-1]) - -# mapping of files to the issues that should be ignored -PYFLAKES_IGNORE = None - -CONFIG = conf.config_dict("test", { - "pyflakes.ignore": [], - "integ.test_directory": "./test/data", -}) - - -def pep8_issues(base_path = DEFAULT_TARGET): - """ - Checks for stylistic issues that are an issue according to the parts of PEP8 - we conform to. - - :param str base_path: directory to be iterated over - - :returns: dict of the form ``path => [(line_number, message)...]`` - """ - - # The pep8 command give output of the form... - # - # FILE:LINE:CHARACTER ISSUE - # - # ... for instance... - # - # ./test/mocking.py:868:31: E225 missing whitespace around operator - # - # Ignoring the following compliance issues. - # - # * E251 no spaces around keyword / parameter equals - # - # This one I dislike a great deal. It makes keyword arguments different - # from assignments which looks... aweful. I'm not sure what PEP8's author - # was on when he wrote this one but it's stupid. - # - # Someone else can change this if they really care. - # - # * E501 line is over 79 characters - # - # We're no longer on TTY terminals. Overly constraining line length makes - # things far less readable, encouraging bad practices like abbreviated - # variable names. - # - # If the code fits on my tiny netbook screen then it's narrow enough. - # - # * E111 and E121 four space indentations - # - # Ahhh, indentation. The holy war that'll never die. Sticking with two - # space indentations since it leads to shorter lines. - # - # * E127 continuation line over-indented for visual indent - # - # Pep8 only works with this one if we have four space indents (its - # detection is based on multiples of four). - - ignored_issues = "E111,E121,E501,E251,E127" - - issues = {} - pep8_output = system.call("pep8 --ignore %s %s" % (ignored_issues, base_path)) - - for line in pep8_output: - line_match = re.match("^(.*):(\d+):(\d+): (.*)$", line) - - if line_match: - path, line, _, issue = line_match.groups() - - if not _is_test_data(path): - issues.setdefault(path, []).append((int(line), issue)) - - return issues - - -def pyflakes_issues(base_path = DEFAULT_TARGET): - """ - Checks for issues via pyflakes. False positives can be whitelisted via our - test configuration. - - :param str base_path: directory to be iterated over - - :returns: dict of the form ``path => [(line_number, message)...]`` - """ - - global PYFLAKES_IGNORE - - if PYFLAKES_IGNORE is None: - pyflakes_ignore = {} - - for line in CONFIG["pyflakes.ignore"]: - path, issue = line.split("=>") - pyflakes_ignore.setdefault(path.strip(), []).append(issue.strip()) - - PYFLAKES_IGNORE = pyflakes_ignore - - # Pyflakes issues are of the form... - # - # FILE:LINE: ISSUE - # - # ... for instance... - # - # stem/prereq.py:73: 'long_to_bytes' imported but unused - # stem/control.py:957: undefined name 'entry' - - issues = {} - pyflakes_output = system.call("pyflakes %s" % base_path) - - for line in pyflakes_output: - line_match = re.match("^(.*):(\d+): (.*)$", line) - - if line_match: - path, line, issue = line_match.groups() - - if not _is_test_data(path) and not issue in PYFLAKES_IGNORE.get(path, []): - issues.setdefault(path, []).append((int(line), issue)) - - return issues - - -def get_issues(base_path = DEFAULT_TARGET): - """ - Checks python source code in the given directory for whitespace issues. - - :param str base_path: directory to be iterated over - - :returns: dict of the form ``path => [(line_number, message)...]`` - """ - - # TODO: This does not check that block indentations are two spaces because - # differentiating source from string blocks ("""foo""") is more of a pita - # than I want to deal with right now. - - issues = {} - - for file_path in _get_files_with_suffix(base_path): - if _is_test_data(file_path): - continue - - with open(file_path) as f: - file_contents = f.read() - - lines, file_issues, prev_indent = file_contents.split("\n"), [], 0 - has_with_import, given_with_warning = False, False - is_block_comment = False - - for index, line in enumerate(lines): - whitespace, content = re.match("^(\s*)(.*)$", line).groups() - - if '"""' in content: - is_block_comment = not is_block_comment - - if content == "from __future__ import with_statement": - has_with_import = True - elif content.startswith("with ") and content.endswith(":") \ - and not has_with_import and not given_with_warning and not is_block_comment: - file_issues.append((index + 1, "missing 'with' import (from __future__ import with_statement)")) - given_with_warning = True - - if "\t" in whitespace: - file_issues.append((index + 1, "indentation has a tab")) - elif "\r" in content: - file_issues.append((index + 1, "contains a windows newline")) - elif content != content.rstrip(): - file_issues.append((index + 1, "line has trailing whitespace")) - - if file_issues: - issues[file_path] = file_issues - - return issues - - -def _is_test_data(path): - return os.path.normpath(path).startswith(os.path.normpath(CONFIG["integ.test_directory"])) - - -def _get_files_with_suffix(base_path, suffix = ".py"): - """ - Iterates over files in a given directory, providing filenames with a certain - suffix. - - :param str base_path: directory to be iterated over - :param str suffix: filename suffix to look for - - :returns: iterator that yields the absolute path for files with the given suffix - """ - - if os.path.isfile(base_path): - if base_path.endswith(suffix): - yield base_path - else: - for root, _, files in os.walk(base_path): - for filename in files: - if filename.endswith(suffix): - yield os.path.join(root, filename) - -if __name__ == '__main__': - issues = get_issues() - - for file_path in issues: - print file_path - - for line_number, msg in issues[file_path]: - line_count = "%-4s" % line_number - print " line %s %s" % (line_count, msg) - - print diff --git a/test/static_checks.py b/test/static_checks.py new file mode 100644 index 0000000..2f758b8 --- /dev/null +++ b/test/static_checks.py @@ -0,0 +1,229 @@ +""" +Performs a check that our python source code follows its whitespace conventions +which are... + +* two space indentations +* tabs are the root of all evil and should be shot on sight +* standard newlines (\\n), not windows (\\r\\n) nor classic mac (\\r) + +This also checks for 2.5 compatibility issues (yea, they're not whitespace but +it's so much easier to do here...): + +* checks that anything using the 'with' keyword has... + from __future__ import with_statement +""" + +from __future__ import with_statement + +import re +import os + +from stem.util import conf, system + +# if ran directly then run over everything one level up +DEFAULT_TARGET = os.path.sep.join(__file__.split(os.path.sep)[:-1]) + +# mapping of files to the issues that should be ignored +PYFLAKES_IGNORE = None + +CONFIG = conf.config_dict("test", { + "pyflakes.ignore": [], + "integ.test_directory": "./test/data", +}) + + +def pep8_issues(base_path = DEFAULT_TARGET): + """ + Checks for stylistic issues that are an issue according to the parts of PEP8 + we conform to. + + :param str base_path: directory to be iterated over + + :returns: dict of the form ``path => [(line_number, message)...]`` + """ + + # The pep8 command give output of the form... + # + # FILE:LINE:CHARACTER ISSUE + # + # ... for instance... + # + # ./test/mocking.py:868:31: E225 missing whitespace around operator + # + # Ignoring the following compliance issues. + # + # * E251 no spaces around keyword / parameter equals + # + # This one I dislike a great deal. It makes keyword arguments different + # from assignments which looks... aweful. I'm not sure what PEP8's author + # was on when he wrote this one but it's stupid. + # + # Someone else can change this if they really care. + # + # * E501 line is over 79 characters + # + # We're no longer on TTY terminals. Overly constraining line length makes + # things far less readable, encouraging bad practices like abbreviated + # variable names. + # + # If the code fits on my tiny netbook screen then it's narrow enough. + # + # * E111 and E121 four space indentations + # + # Ahhh, indentation. The holy war that'll never die. Sticking with two + # space indentations since it leads to shorter lines. + # + # * E127 continuation line over-indented for visual indent + # + # Pep8 only works with this one if we have four space indents (its + # detection is based on multiples of four). + + ignored_issues = "E111,E121,E501,E251,E127" + + issues = {} + pep8_output = system.call("pep8 --ignore %s %s" % (ignored_issues, base_path)) + + for line in pep8_output: + line_match = re.match("^(.*):(\d+):(\d+): (.*)$", line) + + if line_match: + path, line, _, issue = line_match.groups() + + if not _is_test_data(path): + issues.setdefault(path, []).append((int(line), issue)) + + return issues + + +def pyflakes_issues(base_path = DEFAULT_TARGET): + """ + Checks for issues via pyflakes. False positives can be whitelisted via our + test configuration. + + :param str base_path: directory to be iterated over + + :returns: dict of the form ``path => [(line_number, message)...]`` + """ + + global PYFLAKES_IGNORE + + if PYFLAKES_IGNORE is None: + pyflakes_ignore = {} + + for line in CONFIG["pyflakes.ignore"]: + path, issue = line.split("=>") + pyflakes_ignore.setdefault(path.strip(), []).append(issue.strip()) + + PYFLAKES_IGNORE = pyflakes_ignore + + # Pyflakes issues are of the form... + # + # FILE:LINE: ISSUE + # + # ... for instance... + # + # stem/prereq.py:73: 'long_to_bytes' imported but unused + # stem/control.py:957: undefined name 'entry' + + issues = {} + pyflakes_output = system.call("pyflakes %s" % base_path) + + for line in pyflakes_output: + line_match = re.match("^(.*):(\d+): (.*)$", line) + + if line_match: + path, line, issue = line_match.groups() + + if not _is_test_data(path) and not issue in PYFLAKES_IGNORE.get(path, []): + issues.setdefault(path, []).append((int(line), issue)) + + return issues + + +def get_issues(base_path = DEFAULT_TARGET): + """ + Checks python source code in the given directory for whitespace issues. + + :param str base_path: directory to be iterated over + + :returns: dict of the form ``path => [(line_number, message)...]`` + """ + + # TODO: This does not check that block indentations are two spaces because + # differentiating source from string blocks ("""foo""") is more of a pita + # than I want to deal with right now. + + issues = {} + + for file_path in _get_files_with_suffix(base_path): + if _is_test_data(file_path): + continue + + with open(file_path) as f: + file_contents = f.read() + + lines, file_issues, prev_indent = file_contents.split("\n"), [], 0 + has_with_import, given_with_warning = False, False + is_block_comment = False + + for index, line in enumerate(lines): + whitespace, content = re.match("^(\s*)(.*)$", line).groups() + + if '"""' in content: + is_block_comment = not is_block_comment + + if content == "from __future__ import with_statement": + has_with_import = True + elif content.startswith("with ") and content.endswith(":") \ + and not has_with_import and not given_with_warning and not is_block_comment: + file_issues.append((index + 1, "missing 'with' import (from __future__ import with_statement)")) + given_with_warning = True + + if "\t" in whitespace: + file_issues.append((index + 1, "indentation has a tab")) + elif "\r" in content: + file_issues.append((index + 1, "contains a windows newline")) + elif content != content.rstrip(): + file_issues.append((index + 1, "line has trailing whitespace")) + + if file_issues: + issues[file_path] = file_issues + + return issues + + +def _is_test_data(path): + return os.path.normpath(path).startswith(os.path.normpath(CONFIG["integ.test_directory"])) + + +def _get_files_with_suffix(base_path, suffix = ".py"): + """ + Iterates over files in a given directory, providing filenames with a certain + suffix. + + :param str base_path: directory to be iterated over + :param str suffix: filename suffix to look for + + :returns: iterator that yields the absolute path for files with the given suffix + """ + + if os.path.isfile(base_path): + if base_path.endswith(suffix): + yield base_path + else: + for root, _, files in os.walk(base_path): + for filename in files: + if filename.endswith(suffix): + yield os.path.join(root, filename) + +if __name__ == '__main__': + issues = get_issues() + + for file_path in issues: + print file_path + + for line_number, msg in issues[file_path]: + line_count = "%-4s" % line_number + print " line %s %s" % (line_count, msg) + + print [View Less]

1 0

[stem/master] Providing alternative methods for parsing a NetworkStatusDocument
by atagar＠torproject.org 03 Feb '13

03 Feb '13

commit ea0b73a5aa221fadafc2ba718a0ef42e151e5ad6 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Feb 3 13:00:56 2013 -0800 Providing alternative methods for parsing a NetworkStatusDocument Adding support in both the DescriptorReader and parse_file() function for three ways of parsing network status documents... a. Provide the router status entries (ie. the current behavior). b. Provide the document itself with the router status entries that … [View More]it contains. This has the biggest cost in terms of upfront parsing time and memory usage, but provides the caller with everything they might want. c. Provide the document but skip reading the router status entries. A handy option of you just care about the document's header/footer. Now that we have these capability I'm further simplifying the descriptor API a bit. The network status docs encouraged users to use the NetworkStatusDocument constructors to achieve option 'b' above, but now that it's in the reader and parse_file() there's no reason for them to do that. Users should now *always* use either the DescriptorReader or parse_file(). If they don't then they're off the reservation. --- stem/descriptor/__init__.py | 49 ++++++++--- stem/descriptor/networkstatus.py | 94 +++++++++------------ stem/descriptor/reader.py | 9 ++- test/settings.cfg | 2 +- test/unit/descriptor/networkstatus/document_v3.py | 23 +++++ 5 files changed, 108 insertions(+), 69 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index f99e3ca..fafdd8a 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -11,6 +11,24 @@ Package for parsing and processing descriptor data. |- get_path - location of the descriptor on disk if it came from a file |- get_unrecognized_lines - unparsed descriptor content +- __str__ - string that the descriptor was made from + +.. data:: DocumentHandler (enum) + + Ways in which we can parse a + :class:`~stem.descriptor.networkstatus.NetworkStatusDocument`. + + Both **ENTRIES** and **BARE_DOCUMENT** have a 'thin' document, which doesn't + have a populated **routers** attribute. This allows for lower memory usage + and upfront runtime. However, if read time and memory aren't a concern then + **DOCUMENT** can provide you with a fully populated document. + + =================== =========== + DocumentHandler Description + =================== =========== + **ENTRIES** Iterates over the contained :class:`~stem.descriptor.router_status_entry.RouterStatusEntry`. Each has a reference to the bare document it came from (through its **document** attribute). + **DOCUMENT** :class:`~stem.descriptor.networkstatus.NetworkStatusDocument` with the :class:`~stem.descriptor.router_status_entry.RouterStatusEntry` it contains (through its **routers** attribute). + **BARE_DOCUMENT** :class:`~stem.descriptor.networkstatus.NetworkStatusDocument` **without** a reference to its contents (the :class:`~stem.descriptor.router_status_entry.RouterStatusEntry` are unread). + =================== =========== """ __all__ = [ @@ -28,6 +46,7 @@ import os import re import stem.prereq +import stem.util.enum import stem.util.str_tools try: @@ -42,8 +61,14 @@ KEYWORD_LINE = re.compile("^([%s]+)(?:[%s]+(.*))?$" % (KEYWORD_CHAR, WHITESPACE) PGP_BLOCK_START = re.compile("^-----BEGIN ([%s%s]+)-----$" % (KEYWORD_CHAR, WHITESPACE)) PGP_BLOCK_END = "-----END %s-----" +DocumentHandler = stem.util.enum.UppercaseEnum( + "ENTRIES", + "DOCUMENT", + "BARE_DOCUMENT", +) + -def parse_file(descriptor_file, descriptor_type = None, path = None, validate = True): +def parse_file(descriptor_file, descriptor_type = None, path = None, validate = True, document_handler = DocumentHandler.ENTRIES): """ Simple function to read the descriptor contents from a file, providing an iterator for its :class:`~stem.descriptor.__init__.Descriptor` contents. @@ -84,7 +109,7 @@ def parse_file(descriptor_file, descriptor_type = None, path = None, validate = ========================================= ===== If you're using **python 3** then beware that the open() function defaults to - using **text mode**. **Binary mode** is strongly suggested because it's both + using text mode. **Binary mode** is strongly suggested because it's both faster (by my testing by about 33x) and doesn't do universal newline translation which can make us misparse the document. @@ -97,6 +122,8 @@ def parse_file(descriptor_file, descriptor_type = None, path = None, validate = :param str path: absolute path to the file's location on disk :param bool validate: checks the validity of the descriptor's content if **True**, skips these checks otherwise + :param stem.descriptor.__init__.DocumentHandler document_handler: method in + which to parse :class:`~stem.descriptor.networkstatus.NetworkStatusDocument` :returns: iterator for :class:`~stem.descriptor.__init__.Descriptor` instances in the file @@ -134,14 +161,14 @@ def parse_file(descriptor_file, descriptor_type = None, path = None, validate = if descriptor_type_match: desc_type, major_version, minor_version = descriptor_type_match.groups() - file_parser = lambda f: _parse_metrics_file(desc_type, int(major_version), int(minor_version), f, validate) + file_parser = lambda f: _parse_metrics_file(desc_type, int(major_version), int(minor_version), f, validate, document_handler) else: raise ValueError("The descriptor_type must be of the form '<type> <major_version>.<minor_version>'") elif metrics_header_match: # Metrics descriptor handling desc_type, major_version, minor_version = metrics_header_match.groups() - file_parser = lambda f: _parse_metrics_file(desc_type, int(major_version), int(minor_version), f, validate) + file_parser = lambda f: _parse_metrics_file(desc_type, int(major_version), int(minor_version), f, validate, document_handler) else: # Cached descriptor handling. These contain multiple descriptors per file. @@ -150,9 +177,9 @@ def parse_file(descriptor_file, descriptor_type = None, path = None, validate = elif filename == "cached-extrainfo": file_parser = lambda f: stem.descriptor.extrainfo_descriptor._parse_file(f, validate = validate) elif filename == "cached-consensus": - file_parser = lambda f: stem.descriptor.networkstatus._parse_file(f, validate = validate) + file_parser = lambda f: stem.descriptor.networkstatus._parse_file(f, validate = validate, document_handler = document_handler) elif filename == "cached-microdesc-consensus": - file_parser = lambda f: stem.descriptor.networkstatus._parse_file(f, is_microdescriptor = True, validate = validate) + file_parser = lambda f: stem.descriptor.networkstatus._parse_file(f, is_microdescriptor = True, validate = validate, document_handler = document_handler) if file_parser: for desc in file_parser(descriptor_file): @@ -168,7 +195,7 @@ def parse_file(descriptor_file, descriptor_type = None, path = None, validate = raise TypeError("Unable to determine the descriptor's type. filename: '%s', first line: '%s'" % (filename, first_line)) -def _parse_metrics_file(descriptor_type, major_version, minor_version, descriptor_file, validate): +def _parse_metrics_file(descriptor_type, major_version, minor_version, descriptor_file, validate, document_handler): # Parses descriptor files from metrics, yielding individual descriptors. This # throws a TypeError if the descriptor_type or version isn't recognized. import stem.descriptor.server_descriptor @@ -193,24 +220,24 @@ def _parse_metrics_file(descriptor_type, major_version, minor_version, descripto elif descriptor_type == "network-status-2" and major_version == 1: document_type = stem.descriptor.networkstatus.NetworkStatusDocumentV2 - for desc in stem.descriptor.networkstatus._parse_file(descriptor_file, document_type, validate = validate): + for desc in stem.descriptor.networkstatus._parse_file(descriptor_file, document_type, validate = validate, document_handler = document_handler): yield desc elif descriptor_type == "dir-key-certificate-3" and major_version == 1: yield stem.descriptor.networkstatus.KeyCertificate(descriptor_file.read(), validate = validate) elif descriptor_type in ("network-status-consensus-3", "network-status-vote-3") and major_version == 1: document_type = stem.descriptor.networkstatus.NetworkStatusDocumentV3 - for desc in stem.descriptor.networkstatus._parse_file(descriptor_file, document_type, validate = validate): + for desc in stem.descriptor.networkstatus._parse_file(descriptor_file, document_type, validate = validate, document_handler = document_handler): yield desc elif descriptor_type == "network-status-microdesc-consensus-3" and major_version == 1: document_type = stem.descriptor.networkstatus.NetworkStatusDocumentV3 - for desc in stem.descriptor.networkstatus._parse_file(descriptor_file, document_type, is_microdescriptor = True, validate = validate): + for desc in stem.descriptor.networkstatus._parse_file(descriptor_file, document_type, is_microdescriptor = True, validate = validate, document_handler = document_handler): yield desc elif descriptor_type == "bridge-network-status" and major_version == 1: document_type = stem.descriptor.networkstatus.BridgeNetworkStatusDocument - for desc in stem.descriptor.networkstatus._parse_file(descriptor_file, document_type, validate = validate): + for desc in stem.descriptor.networkstatus._parse_file(descriptor_file, document_type, validate = validate, document_handler = document_handler): yield desc else: raise TypeError("Unrecognized metrics descriptor format. type: '%s', version: '%i.%i'" % (descriptor_type, major_version, minor_version)) diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index 0d60de1..9f3730c 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -15,43 +15,18 @@ dir-spec. Documents can be obtained from a few sources... Of these, the router status entry section can be quite large (on the order of hundreds of kilobytes). As such we provide a couple of methods for reading -network status documents... - -* :class:`stem.descriptor.networkstatus.NetworkStatusDocumentV3` constructor - -If read time and memory aren't a concern then you can simply use the document -constructor. Router entries are assigned to its 'routers' attribute... - -:: - - from stem.descriptor.networkstatus import NetworkStatusDocumentV3 - - # Reads the full consensus into memory twice (both for the parsed and - # unparsed contents). - - consensus_file = open('.tor/cached-consensus', 'r') - consensus = NetworkStatusDocumentV3(consensus_file.read()) - consensus_file.close() - - for router in consensus.routers: - print router.nickname - -* :func:`stem.descriptor.parse_file` - -Alternatively, the :func:`~stem.descriptor.parse_file` function provides an -iterator for a document's routers. Those routers refer to a 'thin' document, -which doesn't have a 'routers' attribute. This allows for lower memory usage -and upfront runtime. +network status documents through :func:`~stem.descriptor.__init__.parse_file`. +For more information see :func:`~stem.descriptor.__init__.DocumentHandler`... :: - from stem.descriptor import parse_file + from stem.descriptor import parse_file, DocumentHandler - with open('.tor/cached-consensus', 'r') as consensus_file: + with open('.tor/cached-consensus', 'rb') as consensus_file: # Processes the routers as we read them in. The routers refer to a document # with an unset 'routers' attribute. - for router in parse_file(consensus_file, 'network-status-consensus-3 1.0'): + for router in parse_file(consensus_file, 'network-status-consensus-3 1.0', document_handler = DocumentHandler.ENTRIES): print router.nickname **Module Overview:** @@ -167,7 +142,7 @@ BANDWIDTH_WEIGHT_ENTRIES = ( ) -def _parse_file(document_file, document_type = None, validate = True, is_microdescriptor = False): +def _parse_file(document_file, document_type = None, validate = True, is_microdescriptor = False, document_handler = stem.descriptor.DocumentHandler.ENTRIES): """ Parses a network status and iterates over the RouterStatusEntry in it. The document that these instances reference have an empty 'routers' attribute to @@ -179,6 +154,8 @@ def _parse_file(document_file, document_type = None, validate = True, is_microde **True**, skips these checks otherwise :param bool is_microdescriptor: **True** if this is for a microdescriptor consensus, **False** otherwise + :param stem.descriptor.__init__.DocumentHandler document_handler: method in + which to parse :class:`~stem.descriptor.networkstatus.NetworkStatusDocument` :returns: :class:`stem.descriptor.networkstatus.NetworkStatusDocument` object @@ -193,17 +170,6 @@ def _parse_file(document_file, document_type = None, validate = True, is_microde if document_type is None: document_type = NetworkStatusDocumentV3 - # getting the document without the routers section - - header = stem.descriptor._read_until_keywords((ROUTERS_START, FOOTER_START, V2_FOOTER_START), document_file) - - routers_start = document_file.tell() - stem.descriptor._read_until_keywords((FOOTER_START, V2_FOOTER_START), document_file, skip = True) - routers_end = document_file.tell() - - footer = document_file.readlines() - document_content = "".join(header + footer) - if document_type == NetworkStatusDocumentV2: document_type = NetworkStatusDocumentV2 router_type = stem.descriptor.router_status_entry.RouterStatusEntryV2 @@ -218,18 +184,38 @@ def _parse_file(document_file, document_type = None, validate = True, is_microde else: raise ValueError("Document type %i isn't recognized (only able to parse v2, v3, and bridge)" % document_type) - desc_iterator = stem.descriptor.router_status_entry._parse_file( - document_file, - validate, - entry_class = router_type, - entry_keyword = ROUTERS_START, - start_position = routers_start, - end_position = routers_end, - extra_args = (document_type(document_content, validate),), - ) - - for desc in desc_iterator: - yield desc + if document_handler == stem.descriptor.DocumentHandler.DOCUMENT: + yield document_type(document_file.read(), validate) + return + + # getting the document without the routers section + + header = stem.descriptor._read_until_keywords((ROUTERS_START, FOOTER_START, V2_FOOTER_START), document_file) + + routers_start = document_file.tell() + stem.descriptor._read_until_keywords((FOOTER_START, V2_FOOTER_START), document_file, skip = True) + routers_end = document_file.tell() + + footer = document_file.readlines() + document_content = "".join(header + footer) + + if document_handler == stem.descriptor.DocumentHandler.BARE_DOCUMENT: + yield document_type(document_content, validate) + elif document_handler == stem.descriptor.DocumentHandler.ENTRIES: + desc_iterator = stem.descriptor.router_status_entry._parse_file( + document_file, + validate, + entry_class = router_type, + entry_keyword = ROUTERS_START, + start_position = routers_start, + end_position = routers_end, + extra_args = (document_type(document_content, validate),), + ) + + for desc in desc_iterator: + yield desc + else: + raise ValueError("Unrecognized document_handler: %s" % document_handler) class NetworkStatusDocument(stem.descriptor.Descriptor): diff --git a/stem/descriptor/reader.py b/stem/descriptor/reader.py index f00e0aa..21ab049 100644 --- a/stem/descriptor/reader.py +++ b/stem/descriptor/reader.py @@ -255,9 +255,11 @@ class DescriptorReader(object): be read, this is unbounded if zero :param str persistence_path: if set we will load and save processed file listings from this path, errors are ignored + :param stem.descriptor.__init__.DocumentHandler document_handler: method in + which to parse :class:`~stem.descriptor.networkstatus.NetworkStatusDocument` """ - def __init__(self, target, validate = True, follow_links = False, buffer_size = 100, persistence_path = None): + def __init__(self, target, validate = True, follow_links = False, buffer_size = 100, persistence_path = None, document_handler = stem.descriptor.DocumentHandler.ENTRIES): if isinstance(target, str): self._targets = [target] else: @@ -266,6 +268,7 @@ class DescriptorReader(object): self._validate = validate self._follow_links = follow_links self._persistence_path = persistence_path + self._document_handler = document_handler self._read_listeners = [] self._skip_listeners = [] self._processed_files = {} @@ -514,7 +517,7 @@ class DescriptorReader(object): self._notify_read_listeners(target) with open(target, 'rb') as target_file: - for desc in stem.descriptor.parse_file(target_file, validate = self._validate, path = target): + for desc in stem.descriptor.parse_file(target_file, validate = self._validate, path = target, document_handler = self._document_handler): if self._is_stopped.isSet(): return @@ -542,7 +545,7 @@ class DescriptorReader(object): if tar_entry.isfile(): entry = tar_file.extractfile(tar_entry) - for desc in stem.descriptor.parse_file(entry, validate = self._validate, path = target): + for desc in stem.descriptor.parse_file(entry, validate = self._validate, path = target, document_handler = self._document_handler): if self._is_stopped.isSet(): return diff --git a/test/settings.cfg b/test/settings.cfg index 6ad4b59..a5abc91 100644 --- a/test/settings.cfg +++ b/test/settings.cfg @@ -157,7 +157,7 @@ target.torrc RUN_PTRACE => PORT, PTRACE pyflakes.ignore stem/prereq.py => 'RSA' imported but unused pyflakes.ignore stem/prereq.py => 'asn1' imported but unused pyflakes.ignore stem/prereq.py => 'long_to_bytes' imported but unused -pyflakes.ignore stem/descriptor/__init__.py => redefinition of unused 'OrderedDict' from line 35 +pyflakes.ignore stem/descriptor/__init__.py => redefinition of unused 'OrderedDict' from line 54 pyflakes.ignore stem/util/str_tools.py => redefinition of function '_to_bytes' from line 54 pyflakes.ignore stem/util/str_tools.py => redefinition of function '_to_unicode' from line 60 pyflakes.ignore test/mocking.py => undefined name 'builtins' diff --git a/test/unit/descriptor/networkstatus/document_v3.py b/test/unit/descriptor/networkstatus/document_v3.py index b1cc74e..74c7fe4 100644 --- a/test/unit/descriptor/networkstatus/document_v3.py +++ b/test/unit/descriptor/networkstatus/document_v3.py @@ -130,6 +130,29 @@ class TestNetworkStatusDocument(unittest.TestCase): for router in stem.descriptor.parse_file(consensus_file, 'network-status-consensus-3 1.0'): self.assertEqual('caerSidi', router.nickname) + def test_handlers(self): + """ + Try parsing a document with DocumentHandler.DOCUMENT and + DocumentHandler.BARE_DOCUMENT. + """ + + # Simple sanity check that they provide the right type, and that the + # document includes or excludes the router status entries as appropriate. + + entry1 = get_router_status_entry_v3({'s': "Fast"}) + entry2 = get_router_status_entry_v3({'s': "Valid"}) + content = get_network_status_document_v3(routers = (entry1, entry2), content = True) + + descriptors = list(stem.descriptor.parse_file(StringIO.StringIO(content), 'network-status-consensus-3 1.0', document_handler = stem.descriptor.DocumentHandler.DOCUMENT)) + self.assertEqual(1, len(descriptors)) + self.assertTrue(isinstance(descriptors[0], NetworkStatusDocumentV3)) + self.assertEqual(2, len(descriptors[0].routers)) + + descriptors = list(stem.descriptor.parse_file(StringIO.StringIO(content), 'network-status-consensus-3 1.0', document_handler = stem.descriptor.DocumentHandler.BARE_DOCUMENT)) + self.assertEqual(1, len(descriptors)) + self.assertTrue(isinstance(descriptors[0], NetworkStatusDocumentV3)) + self.assertEqual(0, len(descriptors[0].routers)) + def test_parse_file(self): """ Try parsing a document via the _parse_file() function. [View Less]

1 0

[stem/master] More succinct python 3 warning for parse_file()
by atagar＠torproject.org 03 Feb '13

03 Feb '13

commit 30146d77243f7a25d54019d9d2641be258c444f8 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Feb 3 11:12:57 2013 -0800 More succinct python 3 warning for parse_file() Read speeds and universal newline translation are both addressed by reading in binary mode. There's no need to have a separate warning for each. --- stem/descriptor/__init__.py | 18 ++++-------------- 1 files changed, 4 insertions(+), 14 deletions(-) diff --git a/stem/descriptor/… [View More]

1 0

[stem/master] Dropping Version.meets_requirements() in favour of comparisons
by atagar＠torproject.org 03 Feb '13

03 Feb '13

commit 479f535614081f097ee2e39ecde70ffaf658ba25 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Feb 3 11:29:53 2013 -0800 Dropping Version.meets_requirements() in favour of comparisons Once upon a time you checked your requirements via simple comparisons... if my_version >= requirement: ... do stuff... I reluctantly changed this to a meets_requirements() method when we added the VersionRequirements class since it was no longer … [View More]simple comparisons the __cmp__ method could handle. However, now that we're using rich comparison operators we can go back to the nicer style of comparisons. Apologies for any confusion this back-and-forth has caused. --- stem/control.py | 6 +- stem/version.py | 58 +++++++++++++--------------- test/integ/connection/authentication.py | 4 +- test/integ/control/controller.py | 2 +- test/integ/response/protocolinfo.py | 2 +- test/unit/version.py | 62 +++++++++++++++--------------- 6 files changed, 65 insertions(+), 69 deletions(-) diff --git a/stem/control.py b/stem/control.py index 8708783..5c7a54e 100644 --- a/stem/control.py +++ b/stem/control.py @@ -1435,7 +1435,7 @@ class Controller(BaseController): for event_type in events: event_version = stem.response.events.EVENT_TYPE_TO_CLASS[event_type]._VERSION_ADDED - if not self.get_version().meets_requirements(event_version): + if self.get_version() < event_version: raise stem.InvalidRequest(552, "%s event requires Tor version %s or later" % (event_type, event_version)) for event_type in events: @@ -1558,7 +1558,7 @@ class Controller(BaseController): if defaulted_version: our_version = self.get_version(None) - if our_version and our_version.meets_requirements(defaulted_version): + if our_version and our_version >= defaulted_version: self._enabled_features.append(feature) return feature in self._enabled_features @@ -1728,7 +1728,7 @@ class Controller(BaseController): if path is None and circuit_id == '0': path_opt_version = stem.version.Requirement.EXTENDCIRCUIT_PATH_OPTIONAL - if not self.get_version().meets_requirements(path_opt_version): + if not self.get_version() >= path_opt_version: raise stem.InvalidRequest(512, "EXTENDCIRCUIT requires the path prior to version %s" % path_opt_version) args = [circuit_id] diff --git a/stem/version.py b/stem/version.py index a7c816c..00e0769 100644 --- a/stem/version.py +++ b/stem/version.py @@ -8,7 +8,7 @@ easily parsed and compared, for instance... >>> my_version = get_system_tor_version() >>> print my_version 0.2.1.30 - >>> my_version.meets_requirements(Requirement.CONTROL_SOCKET) + >>> my_version >= Requirement.CONTROL_SOCKET True **Module Overview:** @@ -18,12 +18,6 @@ easily parsed and compared, for instance... get_system_tor_version - gets the version of our system's tor installation Version - Tor versioning information - +- meets_requirements - checks if this version meets the given requirements - - VersionRequirements - Series of version requirements - |- greater_than - adds rule that matches if we're greater than a version - |- less_than - adds rule that matches if we're less than a version - +- in_range - adds rule that matches if we're within a given version range .. data:: Requirement (enum) @@ -160,24 +154,6 @@ class Version(object): else: raise ValueError("'%s' isn't a properly formatted tor version" % version_str) - def meets_requirements(self, requirements): - """ - Checks if this version meets the requirements for a given feature. We can - be compared to either a :class:`~stem.version.Version` or - :class:`~stem.version.VersionRequirements`. - - :param requirements: requirements to be checked for - """ - - if isinstance(requirements, Version): - return self >= requirements - else: - for rule in requirements.rules: - if rule(self): - return True - - return False - def __str__(self): """ Provides the string used to construct the version. @@ -219,14 +195,34 @@ class Version(object): def __eq__(self, other): return self._compare(other, lambda s, o: s == o) - def __lt__(self, other): - return self._compare(other, lambda s, o: s < o) + def __gt__(self, other): + """ + Checks if this version meets the requirements for a given feature. We can + be compared to either a :class:`~stem.version.Version` or + :class:`~stem.version._VersionRequirements`. + """ + + if isinstance(other, _VersionRequirements): + for rule in other.rules: + if rule(self): + return True + + return False + + return self._compare(other, lambda s, o: s > o) + + def __ge__(self, other): + if isinstance(other, _VersionRequirements): + for rule in other.rules: + if rule(self): + return True + + return False - def __le__(self, other): - return self._compare(other, lambda s, o: s <= o) + return self._compare(other, lambda s, o: s >= o) -class VersionRequirements(object): +class _VersionRequirements(object): """ Series of version constraints that can be compared to. For instance, this allows for comparisons like 'if I'm greater than version X in the 0.2.2 @@ -283,7 +279,7 @@ class VersionRequirements(object): self.rules.append(new_rule) -safecookie_req = VersionRequirements() +safecookie_req = _VersionRequirements() safecookie_req.in_range(Version("0.2.2.36"), Version("0.2.3.0")) safecookie_req.greater_than(Version("0.2.3.13")) diff --git a/test/integ/connection/authentication.py b/test/integ/connection/authentication.py index b8833df..d8c0a8a 100644 --- a/test/integ/connection/authentication.py +++ b/test/integ/connection/authentication.py @@ -43,7 +43,7 @@ def _can_authenticate(auth_type): tor_options = runner.get_options() password_auth = test.runner.Torrc.PASSWORD in tor_options cookie_auth = test.runner.Torrc.COOKIE in tor_options - safecookie_auth = cookie_auth and runner.get_tor_version().meets_requirements(stem.version.Requirement.AUTH_SAFECOOKIE) + safecookie_auth = cookie_auth and runner.get_tor_version() >= stem.version.Requirement.AUTH_SAFECOOKIE if not password_auth and not cookie_auth: # open socket, anything but safecookie will work @@ -104,7 +104,7 @@ class TestAuthenticate(unittest.TestCase): self.cookie_auth_methods = [stem.connection.AuthMethod.COOKIE] tor_version = test.runner.get_runner().get_tor_version() - if tor_version.meets_requirements(stem.version.Requirement.AUTH_SAFECOOKIE): + if tor_version >= stem.version.Requirement.AUTH_SAFECOOKIE: self.cookie_auth_methods.append(stem.connection.AuthMethod.SAFECOOKIE) def test_authenticate_general_socket(self): diff --git a/test/integ/control/controller.py b/test/integ/control/controller.py index 15168a4..3cabcd9 100644 --- a/test/integ/control/controller.py +++ b/test/integ/control/controller.py @@ -369,7 +369,7 @@ class TestController(unittest.TestCase): if test.runner.Torrc.COOKIE in tor_options: auth_methods.append(stem.response.protocolinfo.AuthMethod.COOKIE) - if tor_version.meets_requirements(stem.version.Requirement.AUTH_SAFECOOKIE): + if tor_version >= stem.version.Requirement.AUTH_SAFECOOKIE: auth_methods.append(stem.response.protocolinfo.AuthMethod.SAFECOOKIE) if test.runner.Torrc.PASSWORD in tor_options: diff --git a/test/integ/response/protocolinfo.py b/test/integ/response/protocolinfo.py index 8756c4e..6f3e79a 100644 --- a/test/integ/response/protocolinfo.py +++ b/test/integ/response/protocolinfo.py @@ -138,7 +138,7 @@ class TestProtocolInfo(unittest.TestCase): if test.runner.Torrc.COOKIE in tor_options: auth_methods.append(stem.response.protocolinfo.AuthMethod.COOKIE) - if tor_version.meets_requirements(stem.version.Requirement.AUTH_SAFECOOKIE): + if tor_version >= stem.version.Requirement.AUTH_SAFECOOKIE: auth_methods.append(stem.response.protocolinfo.AuthMethod.SAFECOOKIE) chroot_path = runner.get_chroot() diff --git a/test/unit/version.py b/test/unit/version.py index 068492a..8d15dc0 100644 --- a/test/unit/version.py +++ b/test/unit/version.py @@ -142,62 +142,62 @@ class TestVersion(unittest.TestCase): Checks a VersionRequirements with a single greater_than rule. """ - requirements = stem.version.VersionRequirements() + requirements = stem.version._VersionRequirements() requirements.greater_than(Version("0.2.2.36")) - self.assertTrue(Version("0.2.2.36").meets_requirements(requirements)) - self.assertTrue(Version("0.2.2.37").meets_requirements(requirements)) - self.assertTrue(Version("0.2.3.36").meets_requirements(requirements)) - self.assertFalse(Version("0.2.2.35").meets_requirements(requirements)) - self.assertFalse(Version("0.2.1.38").meets_requirements(requirements)) + self.assertTrue(Version("0.2.2.36") >= requirements) + self.assertTrue(Version("0.2.2.37") >= requirements) + self.assertTrue(Version("0.2.3.36") >= requirements) + self.assertFalse(Version("0.2.2.35") >= requirements) + self.assertFalse(Version("0.2.1.38") >= requirements) - requirements = stem.version.VersionRequirements() + requirements = stem.version._VersionRequirements() requirements.greater_than(Version("0.2.2.36"), False) - self.assertFalse(Version("0.2.2.35").meets_requirements(requirements)) - self.assertFalse(Version("0.2.2.36").meets_requirements(requirements)) - self.assertTrue(Version("0.2.2.37").meets_requirements(requirements)) + self.assertFalse(Version("0.2.2.35") >= requirements) + self.assertFalse(Version("0.2.2.36") >= requirements) + self.assertTrue(Version("0.2.2.37") >= requirements) def test_requirements_less_than(self): """ Checks a VersionRequirements with a single less_than rule. """ - requirements = stem.version.VersionRequirements() + requirements = stem.version._VersionRequirements() requirements.less_than(Version("0.2.2.36")) - self.assertTrue(Version("0.2.2.36").meets_requirements(requirements)) - self.assertTrue(Version("0.2.2.35").meets_requirements(requirements)) - self.assertTrue(Version("0.2.1.38").meets_requirements(requirements)) - self.assertFalse(Version("0.2.2.37").meets_requirements(requirements)) - self.assertFalse(Version("0.2.3.36").meets_requirements(requirements)) + self.assertTrue(Version("0.2.2.36") >= requirements) + self.assertTrue(Version("0.2.2.35") >= requirements) + self.assertTrue(Version("0.2.1.38") >= requirements) + self.assertFalse(Version("0.2.2.37") >= requirements) + self.assertFalse(Version("0.2.3.36") >= requirements) - requirements = stem.version.VersionRequirements() + requirements = stem.version._VersionRequirements() requirements.less_than(Version("0.2.2.36"), False) - self.assertFalse(Version("0.2.2.37").meets_requirements(requirements)) - self.assertFalse(Version("0.2.2.36").meets_requirements(requirements)) - self.assertTrue(Version("0.2.2.35").meets_requirements(requirements)) + self.assertFalse(Version("0.2.2.37") >= requirements) + self.assertFalse(Version("0.2.2.36") >= requirements) + self.assertTrue(Version("0.2.2.35") >= requirements) def test_requirements_in_range(self): """ Checks a VersionRequirements with a single in_range rule. """ - requirements = stem.version.VersionRequirements() + requirements = stem.version._VersionRequirements() requirements.in_range(Version("0.2.2.36"), Version("0.2.2.38")) - self.assertFalse(Version("0.2.2.35").meets_requirements(requirements)) - self.assertTrue(Version("0.2.2.36").meets_requirements(requirements)) - self.assertTrue(Version("0.2.2.37").meets_requirements(requirements)) - self.assertFalse(Version("0.2.2.38").meets_requirements(requirements)) + self.assertFalse(Version("0.2.2.35") >= requirements) + self.assertTrue(Version("0.2.2.36") >= requirements) + self.assertTrue(Version("0.2.2.37") >= requirements) + self.assertFalse(Version("0.2.2.38") >= requirements) # rule for 'anything in the 0.2.2.x series' - requirements = stem.version.VersionRequirements() + requirements = stem.version._VersionRequirements() requirements.in_range(Version("0.2.2.0"), Version("0.2.3.0")) for index in xrange(0, 100): - self.assertTrue(Version("0.2.2.%i" % index).meets_requirements(requirements)) + self.assertTrue(Version("0.2.2.%i" % index) >= requirements) def test_requirements_multiple_rules(self): """ @@ -205,15 +205,15 @@ class TestVersion(unittest.TestCase): """ # rule to say 'anything but the 0.2.2.x series' - requirements = stem.version.VersionRequirements() + requirements = stem.version._VersionRequirements() requirements.greater_than(Version("0.2.3.0")) requirements.less_than(Version("0.2.2.0"), False) - self.assertTrue(Version("0.2.3.0").meets_requirements(requirements)) - self.assertFalse(Version("0.2.2.0").meets_requirements(requirements)) + self.assertTrue(Version("0.2.3.0") >= requirements) + self.assertFalse(Version("0.2.2.0") >= requirements) for index in xrange(0, 100): - self.assertFalse(Version("0.2.2.%i" % index).meets_requirements(requirements)) + self.assertFalse(Version("0.2.2.%i" % index) >= requirements) def assert_versions_match(self, version, major, minor, micro, patch, status, extra): """ [View Less]

1 0