- tor-commits - lists.torproject.org

[ooni-probe/master] Add more detailed error handling to runner.py
by art＠torproject.org 26 Dec '12

26 Dec '12

commit 51e9ff3db497f1494a45dbb7525576f8eb1cd278 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Dec 26 16:30:34 2012 +0100 Add more detailed error handling to runner.py --- ooni/runner.py | 12 +++++++++++- 1 files changed, 11 insertions(+), 1 deletions(-) diff --git a/ooni/runner.py b/ooni/runner.py index 7787992..d064756 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -177,7 +177,12 @@ def runTestCasesWithInput(test_cases, test_input, yaml_reporter, return yaml_reporter.testDone(test_instance, test_name) d1 = oonib_reporter.testDone(test_instance, test_name) d2 = yaml_reporter.testDone(test_instance, test_name) - return defer.DeferredList([d1, d2]) + dl = defer.DeferredList([d1, d2]) + @dl.addErrback + def reportingFailed(failure): + log.err("Error in reporting %s" % test_name) + log.exception(failure) + return dl def test_done(result, test_instance, test_name): log.msg("Finished running %s" % test_name) @@ -232,6 +237,11 @@ def runTestCasesWithInput(test_cases, test_input, yaml_reporter, test_methods_d = defer.DeferredList(dl) test_methods_d.addCallback(tests_done, test_cases[0][0]) + @test_methods_d.addErrback + def deferredListFailed(failure): + log.err("Error Test Method Deferred List") + log.exception(failure) + return test_methods_d def runTestCasesWithInputUnit(test_cases, input_unit, yaml_reporter,

1 0

r25970: {website} add vieth-server mirror. (website/trunk)
by Andrew Lewman 26 Dec '12

26 Dec '12

Author: phobos Date: 2012-12-26 15:29:58 +0000 (Wed, 26 Dec 2012) New Revision: 25970 Modified: website/trunk/update-mirrors.pl Log: add vieth-server mirror. Modified: website/trunk/update-mirrors.pl =================================================================== --- website/trunk/update-mirrors.pl 2012-12-26 15:27:14 UTC (rev 25969) +++ website/trunk/update-mirrors.pl 2012-12-26 15:29:58 UTC (rev 25970) @@ -1178,6 +1178,24 @@ httpsDistMirror => "", rsyncDistMirror => "", hiddenServiceMirror => "", + }, + mirror083 => { + adminContact => "", + orgName => "", + isoCC => "DE", + subRegion => "", + region => "DE", + ipv4 => "True", + ipv6 => "False", + loadBalanced => "No", + httpWebsiteMirror => "http://mirror.vieth-server.de/torproject.org", + httpsWebsiteMirror => "", + rsyncWebsiteMirror => "", + ftpWebsiteMirror => "", + httpDistMirror => "http://mirror.vieth-server.de/torproject.org/dist/", + httpsDistMirror => "", + rsyncDistMirror => "", + hiddenServiceMirror => "", } );

1 0

r25969: {website} update the tor.li mirror (website/trunk)
by Andrew Lewman 26 Dec '12

26 Dec '12

Author: phobos Date: 2012-12-26 15:27:14 +0000 (Wed, 26 Dec 2012) New Revision: 25969 Modified: website/trunk/update-mirrors.pl Log: update the tor.li mirror Modified: website/trunk/update-mirrors.pl =================================================================== --- website/trunk/update-mirrors.pl 2012-12-26 00:51:41 UTC (rev 25968) +++ website/trunk/update-mirrors.pl 2012-12-26 15:27:14 UTC (rev 25969) @@ -1153,11 +1153,11 @@ ipv6 => "False", loadBalanced => "No", httpWebsiteMirror => "http://tor.li/", - httpsWebsiteMirror => "", + httpsWebsiteMirror => "https://tor.li/", rsyncWebsiteMirror => "", ftpWebsiteMirror => "", - httpDistMirror => "", - httpsDistMirror => "", + httpDistMirror => "http://tor.li/dist/", + httpsDistMirror => "https://tor.li/dist/", rsyncDistMirror => "", hiddenServiceMirror => "", },

1 0

[tor/master] Rate-limit "No circuits are opened" message to once-per-hour
by nickm＠torproject.org 26 Dec '12

26 Dec '12

commit 127cb39ffcacbcd99633671e92d299ef53bbac46 Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Dec 26 10:05:45 2012 -0500 Rate-limit "No circuits are opened" message to once-per-hour mr-4 reports on #7799 that he was seeing it several times per second, which suggests that things had gone very wrong. This isn't a real fix, but it should make Tor usable till we can figure out the real issue. --- changes/bug7799.ratelim | 6 ++++++ src/or/circuituse.c | 11 ++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/changes/bug7799.ratelim b/changes/bug7799.ratelim new file mode 100644 index 0000000..cb7742c --- /dev/null +++ b/changes/bug7799.ratelim @@ -0,0 +1,6 @@ + o Minor bugfixes: + - Rate-limit the "No circuits are opened. Relaxed timeout for a + circuit with channel state open..." message to once per hour to + keep it from filling the notice logs. Mitigates bug 7799 but does + not fix the underlying cause. Bugfix on 0.2.4.7-alpha. + diff --git a/src/or/circuituse.c b/src/or/circuituse.c index 57ecef1..c3495b4 100644 --- a/src/or/circuituse.c +++ b/src/or/circuituse.c @@ -478,13 +478,18 @@ circuit_expire_building(void) } continue; } else { - log_notice(LD_CIRC, + static ratelim_t relax_timeout_limit = RATELIM_INIT(3600); + char *m; + if ((m = rate_limit_log(&relax_timeout_limit, approx_time()))) { + log_notice(LD_CIRC, "No circuits are opened. Relaxed timeout for " "a circuit with channel state %s to %ldms. " "However, it appears the circuit has timed out anyway. " - "%d guards are live. ", + "%d guards are live. %s", channel_state_to_string(victim->n_chan->state), - (long)circ_times.close_ms, num_live_entry_guards(0)); + (long)circ_times.close_ms, num_live_entry_guards(0), m); + tor_free(m); + } } }

1 0

[ooni-probe/master] Refactor to raise exceptions instead of checking return code
by art＠torproject.org 26 Dec '12

26 Dec '12

commit 6c616ba759c6c2ac4d14b8a63f95084b0ec17ad7 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Dec 26 08:12:51 2012 +0100 Refactor to raise exceptions instead of checking return code --- scripts/archive_reports.py | 27 +++++++++++++++------------ 1 files changed, 15 insertions(+), 12 deletions(-) diff --git a/scripts/archive_reports.py b/scripts/archive_reports.py index f48fa08..8246822 100755 --- a/scripts/archive_reports.py +++ b/scripts/archive_reports.py @@ -36,13 +36,15 @@ def filter_reports_by_age(report): except (InvalidTimestampFormat, ValueError): return False +class InvalidReportField(Exception): + pass + def validate_fields(fields): log.debug("Report fields are: %s" % fields) # check report version if fields['test_version'] not in valid_test_versions: - log.err("Report submitted with invalid report version!") - return False + raise InvalidReportField('test_version') # check report CC #XXX: confirm what value we use for default CC and whether @@ -50,33 +52,28 @@ def validate_fields(fields): if fields['probe_cc'] is None: fields['probe_cc'] = default_probe_cc if not re.match('[A-Z\?]{2,4}', fields['probe_cc'].upper()): - log.err("Report submitted with invalid CC!") - return False + raise InvalidReportField('probe_cc') # check report ASN if fields['probe_asn'] is None: fields['probe_asn'] = 'AS0' if not re.match('^AS[0-9]{1,10}', fields['probe_asn'].upper()): - log.err("Report submitted with invalid AS Number!") - return False + raise InvalidReportField('probe_asn') # check report timestamp try: datetime_ts = datetime.fromtimestamp(fields['start_time']) datetime_str = timestamp(datetime_ts) except InvalidTimestampFormat: - log.err("Report submitted with invalid timestamp!") - return False + raise InvalidReportField('start_time') # check report IP try: IPAddress(fields['probe_ip']) except ValueError: - log.err("Report submitted with invalid IP Address!") - return False + raise InvalidReportField('probe_ip') # all looks good! - return True def get_report_header_fields(report_header): required_fields = ['probe_asn', 'probe_cc', 'probe_ip', 'start_time', @@ -141,7 +138,13 @@ for report in reports_to_archive: yamloo = yaml.safe_load_all(f) report_header = yamloo.next() fields = get_report_header_fields(report_header) - if not validate_fields(fields): + try: + validate_fields(fields) + except InvalidReportField, field_name: + log.err("Report %s contains invalid field called %s" % (report, field_name)) + continue + except: + log.err("An unhandled error occurred while processing %s" % report) continue # get a target filename or fail

1 0

[ooni-probe/master] Add report archival script
by art＠torproject.org 26 Dec '12

26 Dec '12

commit cf8a8cc3a7a5798025f2c7d33bed3f8c8824f1af Author: aagbsn <aagbsn(a)extc.org> Date: Mon Dec 24 04:15:54 2012 +0000 Add report archival script --- scripts/archive_reports.py | 166 ++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 166 insertions(+), 0 deletions(-) diff --git a/scripts/archive_reports.py b/scripts/archive_reports.py new file mode 100755 index 0000000..f48fa08 --- /dev/null +++ b/scripts/archive_reports.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python +import yaml +import sys +import glob +import fcntl +import os +import re +from ipaddr import IPAddress +from datetime import timedelta +from datetime import datetime +from ooni.otime import fromTimestamp, timestamp +from ooni.otime import InvalidTimestampFormat, utcDateNow +from ooni.utils import log + +############################################################################### +# You can set some config options here # +############################################################################### +report_age = 1 # hours +report_archive_dir = '/home/user/oonib/reports/archived' +report_source_dir = '/home/user/oonib/reports' +valid_test_versions = ['0.1', '0.1.1', '0.4', '0.1.3'] +default_probe_cc = '??' +target_permission = 0444 +path_permission = 0755 +retry_attempts = 100 +############################################################################### + +now = utcDateNow() +delta = timedelta(hours=report_age) + +def filter_reports_by_age(report): + try: + ts,__,__ = os.path.basename(report).split('_') + if now - fromTimestamp(ts) > delta: + return True + except (InvalidTimestampFormat, ValueError): + return False + +def validate_fields(fields): + log.debug("Report fields are: %s" % fields) + + # check report version + if fields['test_version'] not in valid_test_versions: + log.err("Report submitted with invalid report version!") + return False + + # check report CC + #XXX: confirm what value we use for default CC and whether + # or not we should support > 2 character CC + if fields['probe_cc'] is None: + fields['probe_cc'] = default_probe_cc + if not re.match('[A-Z\?]{2,4}', fields['probe_cc'].upper()): + log.err("Report submitted with invalid CC!") + return False + + # check report ASN + if fields['probe_asn'] is None: + fields['probe_asn'] = 'AS0' + if not re.match('^AS[0-9]{1,10}', fields['probe_asn'].upper()): + log.err("Report submitted with invalid AS Number!") + return False + + # check report timestamp + try: + datetime_ts = datetime.fromtimestamp(fields['start_time']) + datetime_str = timestamp(datetime_ts) + except InvalidTimestampFormat: + log.err("Report submitted with invalid timestamp!") + return False + + # check report IP + try: + IPAddress(fields['probe_ip']) + except ValueError: + log.err("Report submitted with invalid IP Address!") + return False + + # all looks good! + return True + +def get_report_header_fields(report_header): + required_fields = ['probe_asn', 'probe_cc', 'probe_ip', 'start_time', + 'test_name', 'test_version'] + try: + return dict([(k,report_header[k]) for k in required_fields ]) + except KeyError: + return None + +def get_target_or_fail(fields, report): + # set the target filename + reportFormatVersion = fields['test_version'] + CC = fields['probe_cc'] + # XXX: wouldn't hurt to check timestamp for sanity again? + dateInISO8601Format,__,__ = os.path.basename(report).split('_') + probeASNumber = fields['probe_asn'] + + # make sure path reportFormatVersion/CC exists + path = os.path.abspath(report_archive_dir) + for component in [reportFormatVersion, CC]: + path = os.path.join(path, component) + if not os.path.isdir(path): + try: + os.mkdir(path, path_permission) + log.debug("mkdir path: %s" % path) + except OSError: + return None + + # if the target file already exists, try to find another filename + filename = "%s_%s.yamloo" % (dateInISO8601Format, probeASNumber) + target = os.path.join(path, filename) + + # try to get a unique filename. os.open as used below requires + # that the file not already exist + naming_attempts = 1 + while os.path.exists(target) and naming_attempts < retry_attempts: + filename = "%s_%s.%d.yamloo" % (dateInISO8601Format, probeASNumber, + naming_attempts) + target = os.path.join(path, filename) + naming_attempts = naming_attempts + 1 + + if naming_attempts >= retry_attempts: + log.err("Failed getting unique filename %d times; skipping" % i) + return None + return target + +# grab list of reports +reports = glob.glob(report_source_dir+'/*') +reports_to_archive = filter(filter_reports_by_age, reports) + +# iterate over the reports to archive +for report in reports_to_archive: + log.debug("Parsing report: %s" % report) + try: + #XXX: verify that os.fdopen works as expected + f = os.fdopen(os.open(report, os.O_RDONLY|os.O_EXCL|os.O_NONBLOCK)) + except IOError: + log.err("Unable to get exclusive lock on %s; skipping" % report) + continue + + # parse the header and validate it + yamloo = yaml.safe_load_all(f) + report_header = yamloo.next() + fields = get_report_header_fields(report_header) + if not validate_fields(fields): + continue + + # get a target filename or fail + target = get_target_or_fail(fields, report) + if not target: + continue + + log.debug("target: %s" % target) + + try: + #XXX: My system does not have os.O_EXLOCK. Verify this works as is. + g = os.fdopen(os.open(target, os.O_CREAT|os.O_EXCL|os.O_NONBLOCK)) + + os.rename(report, target) + os.chmod(target, target_permission) + f.close() + g.close() + + except IOError: + # unable to lock the file... still held open? + log.err("Failed to lock target file. Possible race condition!") + continue

1 0

[ooni-probe/master] Merge branch 'archive_reports'
by art＠torproject.org 26 Dec '12

26 Dec '12

commit 796611aabf5a93d0674cd050704d39bd63ad95cf Merge: c2abe62 6c616ba Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Dec 26 08:13:41 2012 +0100 Merge branch 'archive_reports' * archive_reports: Refactor to raise exceptions instead of checking return code Add report archival script scripts/archive_reports.py | 169 ++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 169 insertions(+), 0 deletions(-)

1 0

[tor/master] Fix a possibly-unused-var warning. Thank you, GCC.
by nickm＠torproject.org 26 Dec '12

26 Dec '12

commit 2e9be92cd7a3ca58d73f3bd127f97c08c21210e0 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Dec 25 23:37:41 2012 -0500 Fix a possibly-unused-var warning. Thank you, GCC. --- src/or/directory.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/src/or/directory.c b/src/or/directory.c index d774dc0..042b2dc 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -365,7 +365,7 @@ static const routerstatus_t * directory_pick_generic_dirserver(dirinfo_type_t type, int pds_flags, uint8_t dir_purpose) { - const routerstatus_t *rs; + const routerstatus_t *rs = NULL; const or_options_t *options = get_options(); if (options->UseBridges)

1 0

[tor/master] Prop 209: Add in hidserv path bias counts for usage.
by nickm＠torproject.org 26 Dec '12

26 Dec '12

commit aa0e6e2c0337c5ddfc9f64ed593d8a59b3c4cb44 Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Sat Nov 17 17:51:27 2012 -0800 Prop 209: Add in hidserv path bias counts for usage. --- src/or/connection_edge.c | 4 ++++ src/or/or.h | 5 ++--- src/or/rendclient.c | 3 +++ src/or/rendservice.c | 3 +++ 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index cb2afe1..a654b61 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -2454,6 +2454,10 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) assert_circuit_ok(circ); connection_exit_connect(n_stream); + + /* For path bias: This circuit was used successfully */ + origin_circ->any_streams_succeeded = 1; + tor_free(address); return 0; } diff --git a/src/or/or.h b/src/or/or.h index f26fc39..8501235 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -2873,9 +2873,8 @@ typedef struct origin_circuit_t { unsigned int isolation_any_streams_attached : 1; /** - * Did any SOCKS streams actually succeed on this circuit? - * - * XXX: We probably also need to set this for intro other hidserv circs.. + * Did any SOCKS streams or hidserv introductions actually succeed on + * this circuit? */ unsigned int any_streams_succeeded : 1; diff --git a/src/or/rendclient.c b/src/or/rendclient.c index 3fb4025..ec43041 100644 --- a/src/or/rendclient.c +++ b/src/or/rendclient.c @@ -378,6 +378,9 @@ rend_client_introduction_acked(origin_circuit_t *circ, * it to specify when a circuit entered the * _C_REND_READY_INTRO_ACKED state. */ rendcirc->base_.timestamp_dirty = time(NULL); + + /* For path bias: This circuit was used successfully */ + circ->any_streams_succeeded = 1; } else { log_info(LD_REND,"...Found no rend circ. Dropping on the floor."); } diff --git a/src/or/rendservice.c b/src/or/rendservice.c index 09792bd..775edd6 100644 --- a/src/or/rendservice.c +++ b/src/or/rendservice.c @@ -1383,6 +1383,9 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, if (circuit_init_cpath_crypto(cpath,keys+DIGEST_LEN,1)<0) goto err; memcpy(cpath->handshake_digest, keys, DIGEST_LEN); + + /* For path bias: This circuit was used successfully */ + circuit->any_streams_succeeded = 1; goto done;

1 0

[tor/master] Be explicit about units for timeout.
by nickm＠torproject.org 26 Dec '12

26 Dec '12

commit da5c398d79c890966339558749662fa8ffabf480 Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Fri Nov 2 12:37:26 2012 -0700 Be explicit about units for timeout. --- src/or/circuitbuild.c | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c index eda36d3..215ca14 100644 --- a/src/or/circuitbuild.c +++ b/src/or/circuitbuild.c @@ -1392,7 +1392,7 @@ entry_guard_inc_first_hop_count(entry_guard_t *guard) "circuits. To avoid potential route manipluation attacks, " "Tor has disabled use of this guard. Success " "counts are %d/%d, with %d timeouts. For reference, your " - "timeout cutoff is %ld.", + "timeout cutoff is %ld seconds.", guard->nickname, hex_str(guard->identity, DIGEST_LEN), guard->circuit_successes, guard->first_hops, guard->timeouts, (long)circ_times.close_ms/1000); @@ -1407,7 +1407,7 @@ entry_guard_inc_first_hop_count(entry_guard_t *guard) "circuits. This could indicate a route manipulation attack, " "extreme network overload, or a bug. " "Success counts are %d/%d, with %d timeouts. " - "For reference, your timeout cutoff is %ld.", + "For reference, your timeout cutoff is %ld seconds.", guard->nickname, hex_str(guard->identity, DIGEST_LEN), guard->circuit_successes, guard->first_hops, guard->timeouts, (long)circ_times.close_ms/1000); @@ -1422,7 +1422,7 @@ entry_guard_inc_first_hop_count(entry_guard_t *guard) "overloaded, but it could also mean an attack against " "you or the potentially the guard itself. Success counts " "are %d/%d, with %d timeouts. For reference, your timeout " - "cutoff is %ld.", + "cutoff is %ld seconds.", guard->nickname, hex_str(guard->identity, DIGEST_LEN), guard->circuit_successes, guard->first_hops, guard->timeouts, (long)circ_times.close_ms/1000); @@ -1435,7 +1435,7 @@ entry_guard_inc_first_hop_count(entry_guard_t *guard) "Your Guard %s=%s is failing more circuits than usual. " "Most likely this means the Tor network is overloaded. " "Success counts are %d/%d, with %d timeouts. For " - "reference, your timeout cutoff is %ld.", + "reference, your timeout cutoff is %ld seconds.", guard->nickname, hex_str(guard->identity, DIGEST_LEN), guard->circuit_successes, guard->first_hops, guard->timeouts, (long)circ_times.close_ms/1000);

1 0