October 2013 - tor-commits - lists.torproject.org

[ooni-probe/master] Take an instance rather than class
by art＠torproject.org 29 Oct '13

29 Oct '13

commit 4aad9aae0d56b57277c8dca4b1781f32899bebd0 Author: aagbsn <aagbsn(a)extc.org> Date: Wed Sep 18 16:21:11 2013 +0200 Take an instance rather than class Measurements should be able to take an instanced class so that the resulting report can be shared between several Measurements. --- ooni/tasks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ooni/tasks.py b/ooni/tasks.py index baefb51..52e9164 100644 --- a/ooni/tasks.py +++ b/ooni/tasks.py @@ -106,9 +106,10 @@ class Measurement(TaskWithTimeout): net_test: a reference to the net_test object such measurement belongs to. """ - self.testInstance = test_class() + self.testInstance = test_class self.testInstance.input = test_input - self.testInstance.report = {'input': test_input} + if not self.testInstance.report: + self.testInstance.report = {'input': test_input} self.testInstance._start_time = time.time() self.testInstance._setUp() self.testInstance.setUp()

1 0

[ooni-probe/master] Fix wrong path of sample configuration file.
by art＠torproject.org 29 Oct '13

29 Oct '13

commit b66cfa3b514843bd06c2ffd51bee0daa69ca2f83 Author: anadahz <anadahz@github> Date: Fri Aug 30 03:48:39 2013 +0200 Fix wrong path of sample configuration file. --- setup-dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-dependencies.sh b/setup-dependencies.sh index af26929..8852702 100755 --- a/setup-dependencies.sh +++ b/setup-dependencies.sh @@ -522,7 +522,7 @@ case $DISTRO_VERSION in echo "" echo "You should also be sure to edit your configuration appropriately by doing:" echo "" - echo "cp /usr/share/ooni/ooniprobe.conf.sample ~/.ooni/ooniprobe.conf" + echo "cp ${DIR}/data/ooniprobe.conf.sample ~/.ooni/ooniprobe.conf" echo ""

1 0

[ooni-probe/master] Add postProcessor support to NetTest
by art＠torproject.org 29 Oct '13

29 Oct '13

commit 9f09874c8313245bfc4da3ed7d2d9d266f36bf1f Author: aagbsn <aagbsn(a)extc.org> Date: Wed Sep 18 16:22:33 2013 +0200 Add postProcessor support to NetTest If a NetTestCase has multiple test_ methods, then the report write shall not occur until all of the individual Measurements have completed. This also changes the postProcessor API slightly; postProcessors are now expected to return a report instance. If a postProcessor is not supplied, then the report object from the instanced NetTestCase class is passed to Report.write() --- ooni/nettest.py | 34 ++++++++++++++++++++++++---------- ooni/tasks.py | 6 +++--- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index f68148b..eb68182 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -537,13 +537,6 @@ class NetTest(object): measurement.done.addCallback(self.director.measurementSucceeded) measurement.done.addErrback(self.director.measurementFailed, measurement) - - if self.report: - measurement.done.addBoth(self.report.write) - - if self.report and self.director: - measurement.done.addBoth(self.doneReport) - return measurement @defer.inlineCallbacks @@ -562,12 +555,33 @@ class NetTest(object): for test_class, test_methods in self.testCases: # load the input processor as late as possible for input in test_class.inputs: + klass = test_class() + measurements = [] for method in test_methods: log.debug("Running %s %s" % (test_class, method)) - measurement = self.makeMeasurement(test_class, method, input) + measurement = self.makeMeasurement(klass, method, input) + measurements.append(measurement.done) self.state.taskCreated() yield measurement + # When the measurement.done callbacks have all fired + # call the postProcessor before writing the report + if self.report: + post = defer.DeferredList(measurements) + + # Call the postProcessor, which must return a single report + # or a deferred + post.addCallback(klass.postProcessor) + def noPostProcessor(failure, report): + failure.trap(NoPostProcessor) + return report + post.addErrback(noPostProcessor, klass.report) + post.addCallback(self.report.write) + + if self.report and self.director: + #ghetto hax to keep NetTestState counts are accurate + [post.addBoth(self.doneReport) for _ in measurements] + self.state.allTasksScheduled() class NetTestCase(object): @@ -654,10 +668,10 @@ class NetTestCase(object): """ pass - def postProcessor(self, report): + def postProcessor(self, measurements): """ Subclass this to do post processing tasks that are to occur once all - the test methods have been called. Once per input. + the test methods have been called once per input. postProcessing works exactly like test methods, in the sense that anything that gets written to the object self.report[] will be added to the final test report. diff --git a/ooni/tasks.py b/ooni/tasks.py index 52e9164..0c7f34e 100644 --- a/ooni/tasks.py +++ b/ooni/tasks.py @@ -110,9 +110,9 @@ class Measurement(TaskWithTimeout): self.testInstance.input = test_input if not self.testInstance.report: self.testInstance.report = {'input': test_input} - self.testInstance._start_time = time.time() - self.testInstance._setUp() - self.testInstance.setUp() + self.testInstance._setUp() + self.testInstance._start_time = time.time() + self.testInstance.setUp() self.netTestMethod = getattr(self.testInstance, test_method)

1 0

[ooni-probe/master] Check to see if setUp was already called correctly
by art＠torproject.org 29 Oct '13

29 Oct '13

commit db70b529b990ec9a0399f3917e7ce4166b2265da Author: aagbsn <aagbsn(a)extc.org> Date: Wed Sep 18 19:09:58 2013 +0200 Check to see if setUp was already called correctly The report dict is already created in the constructor, so we look to see if instead the input argument was already set. --- ooni/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/tasks.py b/ooni/tasks.py index 0c7f34e..3d2dbba 100644 --- a/ooni/tasks.py +++ b/ooni/tasks.py @@ -108,7 +108,7 @@ class Measurement(TaskWithTimeout): """ self.testInstance = test_class self.testInstance.input = test_input - if not self.testInstance.report: + if 'input' not in self.testInstance.report.keys(): self.testInstance.report = {'input': test_input} self.testInstance._setUp() self.testInstance._start_time = time.time()

1 0

[ooni-probe/master] Update dependencies in README.md
by art＠torproject.org 29 Oct '13

29 Oct '13

commit ad046ae13534c8e0dd25acd19cfa53bf8e53ac69 Author: aagbsn <aagbsn(a)extc.org> Date: Sun Oct 13 10:53:56 2013 +0000 Update dependencies in README.md --- README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 53bff52..dde43f7 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ Basic system requirements: On Debian or Ubuntu GNU/Linux based systems these can be installed with: ``` -sudo apt-get install git-core python python-pip python-dev build-essential tor tor-geoipdb obfsproxy tcpdump +sudo apt-get install git-core python python-pip python-dev build-essential tor tor-geoipdb obfsproxy tcpdump python-geoip python-docutils python-ipaddr python-scapy python-pyrex ``` Other packages that may be of interest: @@ -130,7 +130,12 @@ The Python dependencies required for running ooniprobe are: * pypcap: https://code.google.com/p/pypcap/ * libdnet: https://code.google.com/p/libdnet/ * txtorcon: https://github.com/meejah/txtorcon - * txsocksx: https://github.com/hellais/txsocksx + * txsocksx: https://github.com/habnabit/txsocksx + * pygeoip: https://github.com/appliedsec/pygeoip + * parsley: http://launchpad.net/parsley + * docutils: http://docutils.sourceforge.net + * ipaddr: http://code.google.com/p/ipaddr-py/ + * Pyrex: http://www.cosc.canterbury.ac.nz/~greg/python/Pyrex ## Install Tor

1 0

[ooni-probe/master] Merge pull request #168 from anadahz/fix_set-dependencies
by art＠torproject.org 29 Oct '13

29 Oct '13

commit d49ebc26b3b65d7d1a50381c19dcca7db3931fb4 Merge: b89066f b66cfa3 Author: aagbsn <aagbsn(a)extc.org> Date: Sun Oct 13 04:16:59 2013 -0700 Merge pull request #168 from anadahz/fix_set-dependencies Improved script and resolved bugs. setup-dependencies.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)

1 0

[ooni-probe/master] Put all dependencies in one place
by art＠torproject.org 29 Oct '13

29 Oct '13

commit 5ec22e8587e953832397d375c4b22c5d59326c8a Author: aagbsn <aagbsn(a)extc.org> Date: Thu Oct 10 10:01:47 2013 +0000 Put all dependencies in one place --- setup.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 4443749..978f9b3 100644 --- a/setup.py +++ b/setup.py @@ -7,13 +7,6 @@ from os.path import join as pj import sys from setuptools import setup -install_requires = [ - 'txsocksx>=0.0.2', - 'scapy>=2.2.0', - 'dnspython>=1.10.0', - 'parsley>1.0', -] - dependency_links = [ 'https://people.torproject.org/~ioerror/src/mirrors/ooniprobe' ] @@ -40,6 +33,7 @@ for root, dirs, file_names in os.walk('data/'): files.append(pj(root, file_name)) data_files.append([pj(usr_share_path, root.replace('data/', '')), files]) +install_requires = [] with open('requirements.txt') as f: for line in f: if line.startswith("#") or line.startswith('http'):

1 0

[ooni-probe/master] Merge pull request #216 from TheTorProject/setup.py-fix
by art＠torproject.org 29 Oct '13

29 Oct '13

commit b89066f0a741e2529c687d5400dc89e9432f9131 Merge: 3062eb2 5ec22e8 Author: aagbsn <aagbsn(a)extc.org> Date: Thu Oct 10 03:35:46 2013 -0700 Merge pull request #216 from TheTorProject/setup.py-fix Fix most of the warnings. MANIFEST.in | 4 ++++ README | 1 + setup.py | 18 ++++++------------ 3 files changed, 11 insertions(+), 12 deletions(-)

1 0

[ooni-probe/master] Exclude component.json and .bowerrc from sdist
by art＠torproject.org 29 Oct '13

29 Oct '13

commit 48ab5f453dc2879401d26d753c6ee499e84c75df Author: aagbsn <aagbsn(a)extc.org> Date: Mon Oct 14 07:43:35 2013 +0000 Exclude component.json and .bowerrc from sdist --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 0edfe5c..2447812 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ include README.md ChangeLog.md requirements.txt -recursive-include data/ui * +recursive-include data/ui/app * include data/Makefile include data/ooniprobe.conf.sample

1 0

[ooni-probe/master] Set data_dir back to /usr/share/ooni
by art＠torproject.org 29 Oct '13

29 Oct '13

commit 99dc4bb64096a98b293ad36a8970c37758fff48d Author: aagbsn <aagbsn(a)extc.org> Date: Thu Oct 10 09:52:27 2013 +0000 Set data_dir back to /usr/share/ooni --- data/ooniprobe.conf.sample | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample index a4831e8..3128d3d 100644 --- a/data/ooniprobe.conf.sample +++ b/data/ooniprobe.conf.sample @@ -46,7 +46,7 @@ advanced: # How many reports to perform concurrently reporting_concurrency: 15 # Specify here a custom data_dir path - data_dir: /Users/y/.virtualenvs/ooni/bin/../share/ooni + data_dir: /usr/share/ooni/ oonid_api_port: 8042 tor: #socks_port: 8801

1 0