June 2013 - tor-commits - lists.torproject.org

[ooni-probe/develop] Only process the input when we are dealing with URLs
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit ec464979088a4de69611010b68b9c59d9123bb05 Author: Arturo Filastò <art(a)fuffa.org> Date: Sat May 18 19:12:47 2013 +0200 Only process the input when we are dealing with URLs --- nettests/blocking/tcpconnect.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/nettests/blocking/tcpconnect.py b/nettests/blocking/tcpconnect.py index 11c5558..5b432e0 100644 --- a/nettests/blocking/tcpconnect.py +++ b/nettests/blocking/tcpconnect.py @@ -48,16 +48,22 @@ class TCPConnectTest(nettest.NetTestCase): This inputProcessor extracts name:port pairs from urls XXX: Does not support unusual port numbers """ + def strip_url(address): + proto, path = x.strip().split('://') + proto = proto.lower() + host = path.split('/')[0] + if proto == 'http': + return "%s:80" % host + if proto == 'https': + return "%s:443" % host + if filename: fp = open(filename) for x in fp.readlines(): - proto, path = x.strip().split('://') - proto = proto.lower() - host = path.split('/')[0] - if proto == 'http': - yield "%s:80" % host - if proto == 'https': - yield "%s:443" % host + if x.startswith("http"): + yield strip_url(x) + else: + yield x.strip() fp.close() else: pass

1 0

[ooni-probe/develop] Remove oonib report archival script.
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit b56ab41246a832b7cc5cc4bddc5eaa8f4c83c2cf Author: Isis Lovecruft <isis(a)torproject.org> Date: Sat May 18 15:58:49 2013 +0000 Remove oonib report archival script. --- scripts/archive_reports.py | 174 -------------------------------------------- 1 file changed, 174 deletions(-) diff --git a/scripts/archive_reports.py b/scripts/archive_reports.py deleted file mode 100755 index c49bd7a..0000000 --- a/scripts/archive_reports.py +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/env python -import yaml -import sys -import glob -import fcntl -import os -import re -from ipaddr import IPAddress -from datetime import timedelta -from datetime import datetime -from ooni.otime import fromTimestamp, timestamp -from ooni.otime import InvalidTimestampFormat, utcDateNow -from ooni.utils import log - -############################################################################### -# You can set some config options here # -############################################################################### -report_age = 1 # hours -report_archive_dir = '/home/user/oonib/reports/archived' -report_source_dir = '/home/user/oonib/reports' -valid_test_versions = ['0.1', '0.1.1', '0.4', '0.1.3'] -default_probe_cc = '??' -target_permission = 0444 -path_permission = 0755 -retry_attempts = 100 -############################################################################### - -now = utcDateNow() -delta = timedelta(hours=report_age) - -def filter_reports_by_age(report): - try: - ts,__,__ = os.path.basename(report).split('_') - if now - fromTimestamp(ts) > delta: - return True - except (InvalidTimestampFormat, ValueError): - return False - -class InvalidReportField(Exception): - pass - -def validate_fields(fields): - log.debug("Report fields are: %s" % fields) - - # check report version - if fields['test_version'] not in valid_test_versions: - raise InvalidReportField('test_version') - - # check report CC - #XXX: confirm what value we use for default CC and whether - # or not we should support > 2 character CC - if fields['probe_cc'] is None: - fields['probe_cc'] = default_probe_cc - if not re.match('[A-Z\?]{2,4}', fields['probe_cc'].upper()): - raise InvalidReportField('probe_cc') - - # check report ASN - if fields['probe_asn'] is None: - fields['probe_asn'] = 'AS0' - if not re.match('^AS[0-9]{1,10}', fields['probe_asn'].upper()): - raise InvalidReportField('probe_asn') - - # check report timestamp - try: - datetime_ts = datetime.fromtimestamp(fields['start_time']) - datetime_str = timestamp(datetime_ts) - except InvalidTimestampFormat: - raise InvalidReportField('start_time') - - # check report IP - try: - IPAddress(fields['probe_ip']) - except ValueError: - raise InvalidReportField('probe_ip') - - # all looks good! - -def get_report_header_fields(report_header): - required_fields = ['probe_asn', 'probe_cc', 'probe_ip', 'start_time', - 'test_name', 'test_version'] - try: - return dict([(k,report_header[k]) for k in required_fields ]) - except KeyError: - return None - -def get_test_name(fields): - test_name = fields['test_name'].lower().replace(' ', '_') - return test_name - -def get_target_or_fail(fields, report): - # set the target filename - reportFormatVersion = fields['test_version'] - CC = fields['probe_cc'] - # XXX: wouldn't hurt to check timestamp for sanity again? - dateInISO8601Format,__,__ = os.path.basename(report).split('_') - probeASNumber = fields['probe_asn'] - testName = get_test_name(fields) - - # make sure path reportFormatVersion/CC exists - path = os.path.abspath(report_archive_dir) - for component in [reportFormatVersion, CC]: - path = os.path.join(path, component) - if not os.path.isdir(path): - try: - os.mkdir(path, path_permission) - log.debug("mkdir path: %s" % path) - except OSError: - return None - - # if the target file already exists, try to find another filename - filename = "%s-%s-%s.yamloo" % (testName, dateInISO8601Format, probeASNumber) - target = os.path.join(path, filename) - - # try to get a unique filename. os.open as used below requires - # that the file not already exist - naming_attempts = 1 - while os.path.exists(target) and naming_attempts < retry_attempts: - filename = "%s-%s-%s.%d.yamloo" % (testName, dateInISO8601Format, - probeASNumber, naming_attempts) - target = os.path.join(path, filename) - naming_attempts = naming_attempts + 1 - - if naming_attempts >= retry_attempts: - log.err("Failed getting unique filename %d times; skipping" % i) - return None - return target - -# grab list of reports -reports = glob.glob(report_source_dir+'/*') -reports_to_archive = filter(filter_reports_by_age, reports) - -# iterate over the reports to archive -for report in reports_to_archive: - log.debug("Parsing report: %s" % report) - try: - #XXX: verify that os.fdopen works as expected - f = os.fdopen(os.open(report, os.O_RDONLY|os.O_EXCL|os.O_NONBLOCK)) - except IOError: - log.err("Unable to get exclusive lock on %s; skipping" % report) - continue - - # parse the header and validate it - yamloo = yaml.safe_load_all(f) - report_header = yamloo.next() - fields = get_report_header_fields(report_header) - try: - validate_fields(fields) - except InvalidReportField, field_name: - log.err("Report %s contains invalid field called %s" % (report, field_name)) - continue - except: - log.err("An unhandled error occurred while processing %s" % report) - continue - - # get a target filename or fail - target = get_target_or_fail(fields, report) - if not target: - continue - - log.debug("target: %s" % target) - - try: - #XXX: My system does not have os.O_EXLOCK. Verify this works as is. - g = os.fdopen(os.open(target, os.O_CREAT|os.O_EXCL|os.O_NONBLOCK)) - - os.rename(report, target) - os.chmod(target, target_permission) - f.close() - g.close() - - except IOError: - # unable to lock the file... still held open? - log.err("Failed to lock target file. Possible race condition!") - continue

1 0

[ooni-probe/develop] Merge branch 'master' of ssh://git-rw.torproject.org/ooni-probe
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 3f9035f84fc5d4e4ba59a018e49d1a1a98ba3ca8 Merge: ec46497 85a671b Author: Arturo Filastò <art(a)fuffa.org> Date: Sat May 18 19:13:30 2013 +0200 Merge branch 'master' of ssh://git-rw.torproject.org/ooni-probe * 'master' of ssh://git-rw.torproject.org/ooni-probe: Remove really old Makefile which seems to no longer point to valid directories. Remove oonib report archival script. Remove line to ignore oonib/oonibackend.conf from .gitignore. Remove log factory for oonib from ooni/utils/log.py. Remove old bin/oonib script. .gitignore | 1 - Makefile | 202 -------------------------------------------- bin/oonib | 36 -------- ooni/utils/log.py | 15 ---- scripts/archive_reports.py | 174 -------------------------------------- 5 files changed, 428 deletions(-)

1 0

[ooni-probe/develop] Remove really old Makefile which seems to no longer point to valid directories.
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 85a671b5eb4c8ecae6ec68811d11999fb76b414d Author: Isis Lovecruft <isis(a)torproject.org> Date: Sat May 18 16:01:59 2013 +0000 Remove really old Makefile which seems to no longer point to valid directories. --- Makefile | 202 -------------------------------------------------------------- 1 file changed, 202 deletions(-) diff --git a/Makefile b/Makefile deleted file mode 100644 index 13d277e..0000000 --- a/Makefile +++ /dev/null @@ -1,202 +0,0 @@ -# -# .:| Open Observatory of Network Interference |:. -# -# /lib/Makefile -# ------------- -# For obtaining, building, and configuring dependencies for OONI installations. -# -# @authors: Isis Lovecruft, Arturo Filastó -# @version: 0.1.0-alpha -# @license: see included LICENSE file -# @copyright: 2012 Isis Lovecruft, Arturo Filastó -# -# XXX TODO eventually this should be converted into a distutils setup.py -# - -here_we_are = . -top_srcdir = ${here_we_are} - -srcdir_ooni = ${top_srcdir}/ooni -srcdir_ooni_kit = ${srcdir_ooni}/kit -srcdir_ooni_plugoo = ${srcdir_ooni}/plugoo -srcdir_ooni_protocols = ${srcdir_ooni}/protocols -srcdir_ooni_templates = ${srcdir_ooni}/templates - -srcdir_oonib = ${top_srcdir}/oonib -srcdir_oonib_lib = ${srcdir_oonib}/lib -srcdir_oonib_report = ${srcdir_oonib}/report -srcdir_oonib_report_db = ${srcdir_oonib_report}/db -srcdir_oonib_testhelpers = ${srcdir_oonib}/testhelpers - -nettests = ${top_srcdir}/nettests -nettests_core = ${nettests}/core -nettests_examples = ${nettests}/examples -nettests_experimental = ${nettests}/experimental -nettests_third_party = ${nettests}/third_party - -prefix = ${top_srcdir} -exec_prefix = ${srcdir_ooni} - -bindir = ${top_srcdir}/bin -docsdir = ${top_srcdir}/docs -libdir = ${srcdir_ooni}/lib -includedir = ${top_srcdir}/include -testsdir = ${top_srcdir}/tests - -#pkgtwisteddir = $(libdir)/Twisted-* -pkgscapydir = $(libdir)/scapy -pkgtxtorcondir = $(libdir)/txtorcon - -SUBDIRS = ${top_srcdir} \ - ${srcdir_ooni} \ - ${srcdir_ooni_kit} \ - ${srcdir_ooni_plugoo} \ - ${srcdir_ooni_protocols} \ - ${srcdir_ooni_templates} \ - ${srcdir_oonib} \ - ${srcdir_oonib_lib} \ - ${srcdir_oonib_report} \ - ${srcdir_oonib_report_db} \ - ${srcdir_oonib_testhelpers} \ - ${nettests} \ - ${nettests_core} \ - ${nettests_examples} \ - ${nettests_experimental} \ - ${nettests_third_party} \ - ${pkgscapydir} \ - ${pkgtxtorcondir} - -all: txtorcon - -dependencies: txtorcon non-pip - -non-pip: scapy-all - -clean: - rm -rf ${libdir}/txtorcon - -clean-all: - rm -rf ${libdir}/txtorcon - rm -rf ${libdir}/txtraceroute.py - rm -rf ${libdir}/txscapy.py - rm -rf ${libdir}/rfc3339.py - -clean-scapy-all: - rm -rf ${libdir}/scapy* - rm -rf ${libdir}/pypcap* - rm -rf ${libdir}/libdnet* - -clean-non-pip: clean-scapy-all - -clean-dependencies: - rm -rf ${libdir}/scapy* - rm -rf ${libdir}/pypcap* - rm -rf ${libdir}/libdnet* - rm -rf ${libdir}/txtorcon - -txtraceroute: - echo "Processing dependency txtraceroute..." - cd ${libdir} && \ - git clone https://github.com/hellais/txtraceroute.git txtraceroute.git && \ - mv txtraceroute.git/txtraceroute.py txtraceroute.py && \ - rm -rf txtraceroute.git && \ - cd ${top_srcdir} - -txtorcon: - echo "Processing dependency txtorcon..." - cd ${libdir} && \ - git clone https://github.com/meejah/txtorcon.git txtorcon.git && \ - mv txtorcon.git/txtorcon txtorcon && \ - rm -rf txtorcon.git && \ - cd ${top_srcdir} - -txscapy: - echo "Processing dependency txscapy" - cd ${libdir} && \ - git clone https://github.com/hellais/txscapy.git txscapy.git && \ - mv txscapy.git/txscapy.py txscapy.py && \ - rm -rf txscapy.git && \ - cd ${top_srcdir} - -rfc3339: - echo "Processing RFC3339 dependency" - cd ${libdir} && \ - hg clone https://bitbucket.org/henry/rfc3339 rfc3339 && \ - mv rfc3339/rfc3339.py rfc3339.py && \ - rm -rf rfc3339 \ - cd ${top_srcdir} - -## requires sudo for setup.py: -scapy: - echo "Processing dependency scapy..." - cd ${libdir} ; \ - wget --ca-certificate="secdev.org.pem" -O scapy.tar.gz https://www.secdev.org/projects/scapy/files/scapy-latest.tar.gz && \ - tar -xzf scapy.tar.gz && \ - cd scapy-* && sudo python setup.py install && \ - cd .. && rm scapy.tar.gz && cd ${top_srcdir} - -## requires sudo for setup.py -pypcap: - echo "Processing dependency pypcap..." - cd ${libdir} ; \ - wget -O pypcap.tar.gz https://pypcap.googlecode.com/files/pypcap-1.1.tar.gz && \ - tar -xzf pypcap.tar.gz && cd pypcap-* && \ - sudo python setup.py install && \ - cd .. && rm pypcap.tar.gz && \ - cd ${top_srcdir} -# make all && make build && make install && make test && \ - make cleandir distclean && cd ${libdir} && \ - rm -rf pypcap* - -libdnet: - echo "Processing dependency libdnet..." - cd ${libdir} ; \ - wget -O libdnet.tar.gz https://libdnet.googlecode.com/files/libdnet-1.12.tgz && \ - tar -xzf libdnet.tar.gz && cd libdnet-* && \ - configure && make && make install && make clean && \ - cd .. && rm -rf libdnet* && cd ${top_srcdir} - -scapy-all: scapy pypcap libdnet - -ETAGS = etags -ETAGSFLAGS = -ETAGSARGS = -## or change to ${SUBDIRS} -TAGME = ${topsrc_dir} - -tags-recursive: - list='$(TAGME)'; for subdir in $$list; do \ - test "$$subdir" = . || (cd $$subdir && make tags); \ - done - -tags: TAGS - -ID: $(HEADERS) $(SOURCES) $(LISP) $(TAGS_FILES) - list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ - unique=`for i in $$list; do \ - if test -f "$$i"; then echo $$i; fi; \ - done | \ - $(AWK) ' { files[$$0] = 1; } \ - END { for (i in files) print i; }'`; \ - mkid -fID $$unique - -TAGS: tags-recursive $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \ - $(TAGS_FILES) $(LISP) - tags=; \ - here=`pwd`; \ - echo ${E_OR_V}; \ - list='$(TAGME)'; for subdir in $$list; do \ - if test "$$subdir" = .; then :; else \ - test -f $$subdir/TAGS && tags="$$tags -i $$here/$$subdir/TAGS"; \ - fi; \ - done; \ - list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ - unique=`for i in $$list; do \ - if test -f "$$i"; then echo $$i; fi; \ - done;` - test -z "$(ETAGS_ARGS)$$tags$$unique" \ - || $(ETAGS) $(ETAGSFLAGS) $(ETAGS_ARGS) $$tags $$unique - -awkward: - | $(AWK) ' { files[$$0] = 1; } \ - END { for (i in files) print i; }'`; \

1 0

[ooni-probe/develop] Parametrize task timeout and retry count
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 3dc0795a36db25898bafd4c5f0b25f864781f1a6 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Apr 11 11:58:29 2013 +0200 Parametrize task timeout and retry count --- ooni/director.py | 3 ++- ooni/managers.py | 12 ++++++------ ooni/tasks.py | 5 +++++ ooniprobe.conf.sample | 12 ++++++++++++ 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/ooni/director.py b/ooni/director.py index bba5eb3..ec4e461 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -157,7 +157,7 @@ class Director(object): self.failedMeasurements += 1 self.failures.append((failure, measurement)) - return failure + return None def reporterFailed(self, failure, net_test): """ @@ -174,6 +174,7 @@ class Director(object): def netTestDone(self, result, net_test): self.activeNetTests.remove(net_test) + @defer.inlineCallbacks def startNetTest(self, _, net_test_loader, reporters): """ Create the Report for the NetTest and start the report NetTest. diff --git a/ooni/managers.py b/ooni/managers.py index 17b430d..d54c169 100644 --- a/ooni/managers.py +++ b/ooni/managers.py @@ -1,6 +1,8 @@ import itertools + from twisted.internet import defer from ooni.utils import log +from ooni import config def makeIterable(item): """ @@ -139,9 +141,8 @@ class MeasurementManager(TaskManager): NetTest on the contrary is aware of the typology of measurements that it is dispatching as they are logically grouped by test file. """ - # XXX tweak these values - retries = 2 - concurrency = 10 + retries = config.advanced.measuement_retries + concurrency = config.advanced.measurement_concurrency def succeeded(self, result, measurement): log.debug("Successfully performed measurement %s" % measurement) @@ -151,9 +152,8 @@ class MeasurementManager(TaskManager): pass class ReportEntryManager(TaskManager): - # XXX tweak these values - retries = 3 - concurrency = 20 + retries = config.advanced.reporting_retries + concurrency = config.advanced.reporting_concurrency def succeeded(self, result, task): log.debug("Successfully performed report %s" % task) diff --git a/ooni/tasks.py b/ooni/tasks.py index c17a5c1..59a908d 100644 --- a/ooni/tasks.py +++ b/ooni/tasks.py @@ -1,5 +1,6 @@ import time +from ooni import config from twisted.internet import defer, reactor class BaseTask(object): @@ -90,6 +91,8 @@ class TaskWithTimeout(BaseTask): return BaseTask.start(self) class Measurement(TaskWithTimeout): + timeout = config.advanced.measurement_timeout + def __init__(self, test_class, test_method, test_input): """ test_class: @@ -127,6 +130,8 @@ class Measurement(TaskWithTimeout): return d class ReportEntry(TaskWithTimeout): + timeout = config.advanced.reporting_timeout + def __init__(self, reporter, measurement): self.reporter = reporter self.measurement = measurement diff --git a/ooniprobe.conf.sample b/ooniprobe.conf.sample index b488216..aeb96c2 100644 --- a/ooniprobe.conf.sample +++ b/ooniprobe.conf.sample @@ -33,6 +33,18 @@ advanced: # If you do not specify start_tor, you will have to have Tor running and # explicitly set the control port and SOCKS port start_tor: true + # After how many seconds we should give up on a particular measurement + measurement_timeout: 30 + # After how many retries we should give up on a measurement + measurement_retry: 2 + # How many measurments to perform concurrently + measurement_concurrency = 100 + # After how may seconds we should give up reporting + reporting_timeout: 30 + # After how many retries to give up on reporting + reporting_retry: 3 + # How many reports to perform concurrently + reporting_concurrency: 20 tor: #socks_port: 9050 #control_port: 9051

1 0

[ooni-probe/develop] Make the starting of ooniprobe via cmd line more robust
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 7d593af7aebd3e96b7702b83ba7f0fbe2b58c141 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Apr 11 11:58:59 2013 +0200 Make the starting of ooniprobe via cmd line more robust --- ooni/oonicli.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 3eae499..0bc5364 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -81,6 +81,7 @@ def parseOptions(): try: cmd_line_options.parseOptions() except usage.UsageError, ue: + print cmd_line_options.getUsage() raise SystemExit, "%s: %s" % (sys.argv[0], ue) return dict(cmd_line_options) @@ -106,7 +107,7 @@ def runWithDirector(): test_list = [] if global_options['no-collector']: log.msg("Not reporting using a collector") - global_options.pop('collector') + global_options['collector'] = None if global_options['testdeck']: test_deck = yaml.safe_load(open(global_options['testdeck'])) @@ -136,6 +137,17 @@ def runWithDirector(): def director_startup_failed(failure): log.err("Failed to start the director") + r = failure.trap(errors.TorNotRunning, + errors.InvalidOONIBCollectorAddress) + if r == errors.TorNotRunning: + log.err("Tor does not appear to be running") + log.err("Reporting with the collector %s is not possible" % + global_options['collector']) + log.msg("Try with a different collector or disable collector reporting with -n") + elif r == errors.InvalidOONIBCollectorAddress: + log.err("Invalid format for oonib collector address.") + log.msg("Should be in the format http://<collector_address>:<port>") + log.msg("for example: ooniprobe -c httpo://nkvphnp3p6agi5qq.onion") reactor.stop() # Wait until director has started up (including bootstrapping Tor) before adding tess @@ -147,23 +159,16 @@ def runWithDirector(): yaml_reporter = YAMLReporter(test_details) reporters = [yaml_reporter] - if collector.startswith('httpo') \ + if collector and collector.startswith('httpo') \ and not config.tor_state: - log.err("Tor does not appear to be running") - log.err("Reporting with the collector %s is not possible") - log.msg("Try with a different collector or disable collector reporting with -n") raise errors.TorNotRunning elif collector: - log.msg("Reporting using collector: %s" % - collector) + log.msg("Reporting using collector: %s" % collector) try: oonib_reporter = OONIBReporter(test_details, collector) reporters.append(oonib_reporter) except errors.InvalidOONIBCollectorAddress, e: - log.err("Invalid format for oonib collector address.") - log.msg("Should be in the format http://<collector_address>:<port>") - log.msg("for example: ooniprobe -c httpo://nkvphnp3p6agi5qq.onion") raise e log.debug("adding callback for startNetTest")

1 0

[ooni-probe/develop] Make reporting more robust
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit f7bfd1198d5b13ad571838372f7316488a6fb3c4 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Apr 11 11:59:46 2013 +0200 Make reporting more robust Get rid of ridiculous logic that uses a deferred list, instead keep track of the report task via a single deferred. --- ooni/director.py | 6 +- ooni/reporter.py | 166 +++++++++++++++++++++++++++++------------------------- 2 files changed, 93 insertions(+), 79 deletions(-) diff --git a/ooni/director.py b/ooni/director.py index ec4e461..e963049 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -189,14 +189,16 @@ class Director(object): net_test = NetTest(net_test_loader, report) net_test.director = self - net_test.report.open() + + yield net_test.report.open() self.measurementManager.schedule(net_test.generateMeasurements()) self.activeNetTests.append(net_test) net_test.done.addBoth(self.netTestDone, net_test) net_test.done.addBoth(report.close) - return net_test.done + + yield net_test.done def startSniffing(self): """ Start sniffing with Scapy. Exits if required privileges (root) are not diff --git a/ooni/reporter.py b/ooni/reporter.py index a3811b7..5338f3a 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -26,11 +26,10 @@ except ImportError: log.err("Scapy is not installed.") -from ooni.errors import InvalidOONIBCollectorAddress -from ooni.errors import ReportNotCreated, ReportAlreadyClosed +from ooni import errors from ooni import otime -from ooni.utils import pushFilenameStack +from ooni.utils import geodata, pushFilenameStack from ooni.utils.net import BodyReceiver, StringProducer, userAgents from ooni import config @@ -119,7 +118,6 @@ def safe_dump(data, stream=None, **kw): class OReporter(object): def __init__(self, test_details): - self.created = defer.Deferred() self.testDetails = test_details def createReport(self): @@ -190,9 +188,9 @@ class YAMLReporter(OReporter): def _write(self, format_string, *args): if not self._stream: - raise ReportNotCreated + raise errors.ReportNotCreated if self._stream.closed: - raise ReportAlreadyClosed + raise errors.ReportAlreadyClosed s = str(format_string) assert isinstance(s, type('')) if args: @@ -225,8 +223,6 @@ class YAMLReporter(OReporter): self._writeln("###########################################") self.writeReportEntry(self.testDetails) - self.created.callback(self) - return defer.succeed(None) def finish(self): self._stream.close() @@ -260,7 +256,7 @@ class OONIBReporter(OReporter): regexp = '^(http|httpo):\/\/[a-zA-Z0-9\-\.]+(:\d+)?$' if not re.match(regexp, self.collectorAddress) or \ len(self.collectorAddress) < 30: - raise InvalidOONIBCollectorAddress + raise errors.InvalidOONIBCollectorAddress @defer.inlineCallbacks def writeReportEntry(self, entry): @@ -306,7 +302,7 @@ class OONIBReporter(OReporter): self.agent = Agent(reactor, sockshost="127.0.0.1", socksport=int(config.tor.socks_port)) except Exception, e: - yield defer.fail(e) + log.exception(e) url = self.collectorAddress + '/report' @@ -339,11 +335,16 @@ class OONIBReporter(OReporter): bodyProducer=bodyProducer) except ConnectionRefusedError: log.err("Connection to reporting backend failed (ConnectionRefusedError)") - self.created.errback(defer.fail(OONIBReportCreationError)) + raise OONIBReportCreationError + + except errors.HostUnreachable: + log.err("Host is not reachable (HostUnreachable error") + raise OONIBReportCreationError except Exception, e: + log.err("Failed to connect to reporter backend") log.exception(e) - yield defer.fail(OONIBReportCreationError) + raise OONIBReportCreationError # This is a little trix to allow us to unspool the response. We create # a deferred and call yield on it. @@ -355,13 +356,13 @@ class OONIBReporter(OReporter): try: parsed_response = json.loads(backend_response) except Exception, e: + log.err("Failed to parse collector response") log.exception(e) - yield defer.fail(e) + raise OONIBReportCreationError self.reportID = parsed_response['report_id'] self.backendVersion = parsed_response['backend_version'] log.debug("Created report with id %s" % parsed_response['report_id']) - self.created.callback(self) class ReportClosed(Exception): pass @@ -386,42 +387,39 @@ class Report(object): self.done = defer.Deferred() self.reportEntryManager = reportEntryManager + self._reporters_openned = 0 + self._reporters_written = 0 + self._reporters_closed = 0 + def open(self): """ This will create all the reports that need to be created and fires the created callback of the reporter whose report got created. """ - l = [] + all_openned = defer.Deferred() + for reporter in self.reporters[:]: - d = reporter.createReport() - d.addErrback(self.failedOpeningReport, reporter) - reporter.created.addErrback(self.failedOpeningReport, reporter) - l.append(reporter.created) - log.debug("Reporters created: %s" % l) - # Should we consume errors silently? - dl = defer.DeferredList(l) - return dl - def failedOpeningReport(self, failure, reporter): - """ - This errback get's called every time we fail to create a report. - By fail we mean that the number of retries has exceeded. - Once a report has failed to be created with a reporter we give up and - remove the reporter from the list of reporters to write to. - """ - log.err("Failed to open %s reporter, giving up..." % reporter) - log.err("Reporter %s failed, removing from report..." % reporter) - self.reporters.remove(reporter) - # Don't forward the exception unless there are no more reporters - if len(self.reporters) == 0: - log.err("Removed last reporter %s" % reporter) - raise NoMoreReporters + def report_created(result): + self._reporters_openned += 1 + if len(self.reporters) == self._reporters_openned: + all_openned.callback(self._reporters_openned) + + def report_failed(failure): + print "WE HAVE FAILED!" + try: + self.failedOpeningReport(failure, reporter) + except errors.NoMoreReporters, e: + all_openned.errback(defer.fail(e)) + + d = defer.maybeDeferred(reporter.createReport) + d.addErrback(report_failed) + d.addCallback(report_created) + + return all_openned def write(self, measurement): """ - This is a lazy call that will write to all the reporters by waiting on - them to be created. - Will return a deferred that will fire once the report for the specified measurement have been written to all the reporters. @@ -431,41 +429,45 @@ class Report(object): an instance of :class:ooni.tasks.Measurement Returns: - a deferred list that will fire once all the report entries have - been written. + a deferred that will fire once all the report entries have + been written or errbacks when no more reporters """ - l = [] + all_written = defer.Deferred() + for reporter in self.reporters[:]: - report_write_task = ReportEntry(reporter, measurement) - def scheduleWriteReportEntry(result): - self.reportEntryManager.schedule(report_write_task) - - # delay scheduling the task until after the report is created - log.debug("Adding report entry task %s" % report_write_task) - reporter.created.addCallback(scheduleWriteReportEntry) - - # if the write task fails n times, kill the reporter - report_write_task.done.addErrback(self.failedOpeningReport, reporter) - l.append(report_write_task.done) - - # XXX: This seems a bit fragile. - # failedOpeningReport will forward the errback if the remaining - # reporter has failed. If we fireOnOneErrback, this means that - # the caller of report.write is responsible for attaching an - # errback to the returned deferred and handle this case. That - # probably means stopping the net test. - - # Here, fireOnOneErrback means to call the deferredlists errback - # as soon as any of the deferreds return a failure. consumeErrors - # is used to prevent any uncaught failures from raising an - # exception. Alternately we could attach a logger to the errback - # of each deferred and it would have the same effect - - # Probably the better thing to do here would be to add a callback - # to the deferredlist that checks to see if any reporters are left - # and raise an exception if there are no remaining reporters - dl = defer.DeferredList(l,fireOnOneErrback=True, consumeErrors=True) - return dl + def report_written(result): + self._reporters_written += 1 + if len(self.reporters) == self._reporters_written: + all_written.callback(self._reporters_written) + + def report_failed(failure): + log.err("Failed writing report entry") + log.exception(failure) + + report_entry_task = ReportEntry(reporter, measurement) + self.reportEntryManager.schedule(report_entry_task) + + report_entry_task.done.addCallback(report_written) + report_entry_task.done.addErrback(report_failed) + + return all_written + + def failedOpeningReport(self, failure, reporter): + """ + This errback get's called every time we fail to create a report. + By fail we mean that the number of retries has exceeded. + Once a report has failed to be created with a reporter we give up and + remove the reporter from the list of reporters to write to. + """ + log.err("Failed to open %s reporter, giving up..." % reporter) + log.err("Reporter %s failed, removing from report..." % reporter) + log.exception(failure) + self.reporters.remove(reporter) + # Don't forward the exception unless there are no more reporters + if len(self.reporters) == 0: + log.err("Removed last reporter %s" % reporter) + raise errors.NoMoreReporters + return def close(self, _): """ @@ -476,10 +478,20 @@ class Report(object): all the reports have been closed. """ - l = [] + all_closed = defer.Deferred() + for reporter in self.reporters[:]: + def report_closed(result): + self._reporters_closed += 1 + if len(self.reporters) == self._reporters_closed: + all_closed.callback(self._reporters_closed) + + def report_failed(failure): + log.err("Failed closing report") + log.exception(failure) + d = defer.maybeDeferred(reporter.finish) - l.append(d) - dl = defer.DeferredList(l) - return dl + d.addCallback(report_closed) + d.addErrback(report_failed) + return all_closed

1 0

[ooni-probe/develop] Add option to disable the default collector
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 9a30d020ce6bbfdbfc3e56dcc2b89cb932bfdb6f Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Mar 9 03:13:50 2013 +0100 Add option to disable the default collector * More robust startup process --- ooni/errors.py | 2 ++ ooni/oonicli.py | 36 ++++++++++++++++++++++++++---------- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/ooni/errors.py b/ooni/errors.py index ac05463..1be42c3 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -147,3 +147,5 @@ class GeoIPDataFilesNotFound(Exception): pass class NoMoreReporters(Exception): pass +class TorNotRunning(Exception): + pass diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 06aa20c..3eae499 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -10,7 +10,7 @@ from twisted.internet import reactor from twisted.python import usage from twisted.python.util import spewer -from ooni.errors import InvalidOONIBCollectorAddress +from ooni import errors from ooni import config from ooni.director import Director @@ -29,7 +29,9 @@ class Options(usage.Options): " files listed on the command line") optFlags = [["help", "h"], - ["resume", "r"]] + ["resume", "r"], + ["no-collector", "n"] + ] optParameters = [["reportfile", "o", None, "report file name"], ["testdeck", "i", None, @@ -102,6 +104,9 @@ def runWithDirector(): # contains (test_cases, options, cmd_line_options) test_list = [] + if global_options['no-collector']: + log.msg("Not reporting using a collector") + global_options.pop('collector') if global_options['testdeck']: test_deck = yaml.safe_load(open(global_options['testdeck'])) @@ -131,32 +136,43 @@ def runWithDirector(): def director_startup_failed(failure): log.err("Failed to start the director") - log.exception(failure) reactor.stop() # Wait until director has started up (including bootstrapping Tor) before adding tess def post_director_start(_): for net_test_loader in test_list: + collector = global_options['collector'] test_details = net_test_loader.testDetails yaml_reporter = YAMLReporter(test_details) reporters = [yaml_reporter] - if global_options['collector']: + if collector.startswith('httpo') \ + and not config.tor_state: + log.err("Tor does not appear to be running") + log.err("Reporting with the collector %s is not possible") + log.msg("Try with a different collector or disable collector reporting with -n") + raise errors.TorNotRunning + elif collector: + log.msg("Reporting using collector: %s" % + collector) try: oonib_reporter = OONIBReporter(test_details, - global_options['collector']) + collector) reporters.append(oonib_reporter) - except InvalidOONIBCollectorAddress: + except errors.InvalidOONIBCollectorAddress, e: log.err("Invalid format for oonib collector address.") log.msg("Should be in the format http://<collector_address>:<port>") log.msg("for example: ooniprobe -c httpo://nkvphnp3p6agi5qq.onion") - sys.exit(1) + raise e log.debug("adding callback for startNetTest") d.addCallback(director.startNetTest, net_test_loader, reporters) - d.addCallback(shutdown) + d.addBoth(shutdown) - d.addCallback(post_director_start) - d.addErrback(director_startup_failed) + def start(): + d.addCallback(post_director_start) + d.addErrback(director_startup_failed) + + reactor.callWhenRunning(start) reactor.run()

1 0

[ooni-probe/develop] Final cleanup and bugfixing for task_manager
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 5265969cd60f0bf3c5929d129c513ea1e42ab9f2 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Apr 11 15:35:31 2013 +0200 Final cleanup and bugfixing for task_manager --- ooni/errors.py | 17 +++++++++++++++++ ooni/nettest.py | 12 ------------ ooni/reporter.py | 38 +++++++++++++++++--------------------- ooni/runner.py | 2 +- 4 files changed, 35 insertions(+), 34 deletions(-) diff --git a/ooni/errors.py b/ooni/errors.py index 1be42c3..613dc9b 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -143,9 +143,26 @@ class InsufficientPrivileges(Exception): class ProbeIPUnknown(Exception): pass + class GeoIPDataFilesNotFound(Exception): pass + class NoMoreReporters(Exception): pass + class TorNotRunning(Exception): pass + +class OONIBReportError(Exception): + pass + +class OONIBReportUpdateError(OONIBReportError): + pass + +class OONIBReportCreationError(OONIBReportError): + pass + +class OONIBTestDetailsLookupError(OONIBReportError): + pass + + diff --git a/ooni/nettest.py b/ooni/nettest.py index 41d7f9f..f0cfa0d 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -294,19 +294,7 @@ class NetTest(object): The state for the NetTest is informed of the fact that another task has reached the done state. - - Args: - report_results: - is the list of tuples returned by the self.report.write - :class:twisted.internet.defer.DeferredList - - Returns: - the same deferred list results """ - for report_status, report_result in report_results: - if report_status == False: - self.director.reporterFailed(report_result, self) - self.state.taskDone() if len(self.report.reporters) == 0: diff --git a/ooni/reporter.py b/ooni/reporter.py index 5338f3a..e44db57 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -227,18 +227,6 @@ class YAMLReporter(OReporter): def finish(self): self._stream.close() -class OONIBReportError(Exception): - pass - -class OONIBReportUpdateError(OONIBReportError): - pass - -class OONIBReportCreationError(OONIBReportError): - pass - -class OONIBTestDetailsLookupError(OONIBReportError): - pass - class OONIBReporter(OReporter): def __init__(self, test_details, collector_address): self.collectorAddress = collector_address @@ -283,7 +271,7 @@ class OONIBReporter(OReporter): # XXX we must trap this in the runner and make sure to report the # data later. log.err("Error in writing report entry") - raise OONIBReportUpdateError + raise errors.OONIBReportUpdateError @defer.inlineCallbacks def createReport(self): @@ -335,16 +323,17 @@ class OONIBReporter(OReporter): bodyProducer=bodyProducer) except ConnectionRefusedError: log.err("Connection to reporting backend failed (ConnectionRefusedError)") - raise OONIBReportCreationError + #yield defer.fail(OONIBReportCreationError()) + raise errors.OONIBReportCreationError except errors.HostUnreachable: log.err("Host is not reachable (HostUnreachable error") - raise OONIBReportCreationError + raise errors.OONIBReportCreationError except Exception, e: log.err("Failed to connect to reporter backend") log.exception(e) - raise OONIBReportCreationError + raise errors.OONIBReportCreationError # This is a little trix to allow us to unspool the response. We create # a deferred and call yield on it. @@ -358,7 +347,7 @@ class OONIBReporter(OReporter): except Exception, e: log.err("Failed to parse collector response") log.exception(e) - raise OONIBReportCreationError + raise errors.OONIBReportCreationError self.reportID = parsed_response['report_id'] self.backendVersion = parsed_response['backend_version'] @@ -398,19 +387,24 @@ class Report(object): """ all_openned = defer.Deferred() + def are_all_openned(): + if len(self.reporters) == self._reporters_openned: + all_openned.callback(self._reporters_openned) + for reporter in self.reporters[:]: def report_created(result): + log.debug("Created report with %s" % reporter) self._reporters_openned += 1 - if len(self.reporters) == self._reporters_openned: - all_openned.callback(self._reporters_openned) + are_all_openned() def report_failed(failure): - print "WE HAVE FAILED!" try: self.failedOpeningReport(failure, reporter) except errors.NoMoreReporters, e: all_openned.errback(defer.fail(e)) + else: + are_all_openned() d = defer.maybeDeferred(reporter.createReport) d.addErrback(report_failed) @@ -433,6 +427,7 @@ class Report(object): been written or errbacks when no more reporters """ all_written = defer.Deferred() + self._reporters_written = 0 for reporter in self.reporters[:]: def report_written(result): @@ -459,9 +454,10 @@ class Report(object): Once a report has failed to be created with a reporter we give up and remove the reporter from the list of reporters to write to. """ + failure.trap(errors.OONIBReportError) log.err("Failed to open %s reporter, giving up..." % reporter) log.err("Reporter %s failed, removing from report..." % reporter) - log.exception(failure) + #log.exception(failure) self.reporters.remove(reporter) # Don't forward the exception unless there are no more reporters if len(self.reporters) == 0: diff --git a/ooni/runner.py b/ooni/runner.py index abdc256..1c28019 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -11,7 +11,7 @@ from txtorcon import TorConfig from txtorcon import TorState, launch_tor from ooni import config -from ooni.reporter import OONIBReporter, YAMLReporter, OONIBReportError +from ooni.reporter import OONIBReporter, YAMLReporter from ooni.inputunit import InputUnitFactory from ooni.nettest import NetTestCase, NoPostProcessor from ooni.utils import log, checkForRoot, pushFilenameStack

1 0

[ooni-probe/develop] Pass the IP address of the probe instead of an object
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit a5fe45c7ef4bcfd09c9653826a423de0519a6d4d Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Apr 11 12:15:28 2013 +0200 Pass the IP address of the probe instead of an object --- ooni/nettest.py | 4 ++-- ooni/utils/geodata.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 0260ca2..41d7f9f 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -7,7 +7,7 @@ from twisted.python import usage, reflect from ooni import geoip from ooni.tasks import Measurement -from ooni.utils import log, checkForRoot +from ooni.utils import log, checkForRoot, geodata from ooni import config from ooni import otime @@ -40,7 +40,7 @@ class NetTestLoader(object): config.privacy.includecity): log.msg("We will include some geo data in the report") try: - client_geodata = geodata.IPToLocation(config.probe_ip) + client_geodata = geodata.IPToLocation(config.probe_ip.address) except e.GeoIPDataFilesNotFound: log.err("Unable to find the geoip data files") client_geodata = {'city': None, 'countrycode': None, 'asn': None} diff --git a/ooni/utils/geodata.py b/ooni/utils/geodata.py index 56ce05c..2acfdb0 100644 --- a/ooni/utils/geodata.py +++ b/ooni/utils/geodata.py @@ -14,6 +14,7 @@ except ImportError: log.err("Unable to import pygeoip. We will not be able to run geo IP related measurements") def IPToLocation(ipaddr): + log.debug("looking up %s" % ipaddr) city_file = os.path.join(config.advanced.geoip_data_dir, 'GeoLiteCity.dat') country_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIP.dat') asn_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIPASNum.dat')

1 0