June 2013 - tor-commits - lists.torproject.org

[ooni-probe/develop] Add self.host to the inputProcessor().
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 273ab4d9d723553fd6ee2fcd90762928dacfce8a Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon Apr 22 00:44:56 2013 +0000 Add self.host to the inputProcessor(). --- nettests/experimental/tls_handshake.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index eab721c..3585657 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -202,6 +202,8 @@ class HandshakeTest(nettest.NetTestCase): return (str(addr), int(port)) def inputProcessor(self, file=None): + if self.host: + yield self.splitInput(self.host) if os.path.isfile(file): with open(file) as fh: for line in fh.readlines():

1 0

[ooni-probe/develop] Add Sphinx docstring for getPeerCert() function.
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit a550f38e1b79a924b3c6e636a3bf5464130b6382 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon Apr 22 00:46:42 2013 +0000 Add Sphinx docstring for getPeerCert() function. --- nettests/experimental/tls_handshake.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index b715387..d2f58a3 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -226,6 +226,15 @@ class HandshakeTest(nettest.NetTestCase): @staticmethod def getPeerCert(connection, get_chain=False): + """Get the PEM-encoded certificate or cert chain of the remote host. + + :param connection: A :class:`OpenSSL.SSL.Connection <Connection>`. + :param bool get_chain: If True, get the all certificates in the + chain. Otherwise, only get the remote host's certificate. + :returns: A PEM-encoded x509 certificate. If + :param:`getPeerCert.get_chain <get_chain>` is True, returns a list + of PEM-encoded x509 certificates. + """ if not get_chain: x509_cert = connection.get_peer_certificate() pem_cert = crypto.dump_certificate(crypto.FILETYPE_PEM, x509_cert)

1 0

[ooni-probe/develop] Cleanup docstrings for getX509Name() and getPublicKey() functions.
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 724688ab14393846a77c3b0b2d064fbc641a9b58 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon Apr 22 00:47:20 2013 +0000 Cleanup docstrings for getX509Name() and getPublicKey() functions. --- nettests/experimental/tls_handshake.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index d2f58a3..a52ccdc 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -250,8 +250,7 @@ class HandshakeTest(nettest.NetTestCase): @staticmethod def getX509Name(certificate, get_components=False): - """ - Get the DER encoded form of the Name portions of and X509 certificate. + """Get the DER-encoded form of the Name fields of an X509 certificate. @param certificate: A :class:`OpenSSL.crypto.X509Name` object. @param get_components: A boolean. If True, returns a list of tuples of @@ -281,10 +280,9 @@ class HandshakeTest(nettest.NetTestCase): @staticmethod def getPublicKey(key): - """ - Get the PEM-encoded format of a host certificate's public key. + """Get the PEM-encoded format of a host certificate's public key. - @param key: A :class:`OpenSSL.crypto.PKey` object. + :param key: A :class:`OpenSSL.crypto.PKey <crypto.PKey>` object. """ try: assert isinstance(key, crypto.PKey), \

1 0

[ooni-probe/develop] Set the tor_state GeoIP class attribute on startup
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit a63b1b1893d3de4c7e7899c7f47bc1e807f4c4e7 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Apr 22 16:23:01 2013 +0200 Set the tor_state GeoIP class attribute on startup --- ooni/geoip.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/ooni/geoip.py b/ooni/geoip.py index 0f8335f..1e6f6e9 100644 --- a/ooni/geoip.py +++ b/ooni/geoip.py @@ -109,12 +109,14 @@ class MaxMindGeoIP(HTTPGeoIPLookupper): class ProbeIP(object): strategy = None - geoIPServices = {'ubuntu': UbuntuGeoIP, - 'torproject': TorProjectGeoIP, - 'maxmind': MaxMindGeoIP - } address = None - tor_state = config.tor_state + + def __init__(self): + self.tor_state = config.tor_state + self.geoIPServices = {'ubuntu': UbuntuGeoIP, + 'torproject': TorProjectGeoIP, + 'maxmind': MaxMindGeoIP + } @defer.inlineCallbacks def lookup(self):

1 0

[ooni-probe/develop] Set self.input to self.host if no inputFile, and update several log messages.
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit c07bd0eed90cb02a98391976cd4bb49b19909ebf Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon Apr 22 01:00:45 2013 +0000 Set self.input to self.host if no inputFile, and update several log messages. --- nettests/experimental/tls_handshake.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index c26b279..5b1f656 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -412,7 +412,7 @@ class HandshakeTest(nettest.NetTestCase): if connection.renegotiate(): log.debug("Renegotiation possible.") - log.message("Retrying handshake with %s..." % host) + log.msg("Retrying handshake with %s..." % host) try: connection.do_handshake() while connection.renegotiate_pending(): @@ -458,8 +458,7 @@ class HandshakeTest(nettest.NetTestCase): peername, peerport = host if isinstance(connection, SSL.Connection): - log.msg("Closing connection to %s:%d..." - % (peername, peerport)) + log.msg("Closing connection to %s:%d..." % (peername, peerport)) while not connection.shutdown(): ## if the connection is halfway shutdown, we have to ## wait for a ZeroReturnError on connection.recv(): @@ -642,8 +641,8 @@ class HandshakeTest(nettest.NetTestCase): self.state = connection.state_string() connection = handleWantRead(connection) else: - ## if we still have an SSL_ERROR_WANT_READ, then try - ## to renegotiate + ## if we still have an SSL_ERROR_WANT_READ, then try to + ## renegotiate self.state = connection.state_string() connection = connectionRenegotiate(connection, connection.getpeername(), @@ -772,6 +771,12 @@ class HandshakeTest(nettest.NetTestCase): def deferMakeConnection(host): return threads.deferToThread(makeConnection, self.input) + + log.msg("Beginning handshake test for %s" % (self.input or self.host)) + + if self.host and not self.input: + self.input = self.splitInput(self.host) + connection = deferMakeConnection(self.input) connection.addCallbacks(connectionSucceeded, connectionFailed, callbackArgs=[self.input, self.timeout],

1 0

[ooni-probe/develop] Rename HandshakeTest.test_tlsv1_handshake() to HandshakeTest.test_handshake().
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit bf9a000b7e6432e5ac6cb7c86ff2804cb22b5383 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon Apr 22 00:48:01 2013 +0000 Rename HandshakeTest.test_tlsv1_handshake() to HandshakeTest.test_handshake(). --- nettests/experimental/tls_handshake.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index a52ccdc..6a0d7b6 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -293,7 +293,7 @@ class HandshakeTest(nettest.NetTestCase): pubkey = crypto.dump_privatekey(crypto.FILETYPE_PEM, key) return pubkey - def test_tlsv1_handshake(self): + def test_handshake(self): """xxx fill me in""" def makeConnection(host):

1 0

[ooni-probe/develop] Also run the unittests
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 1373acff8d459344c9c0443c4f798f7560f889fc Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Apr 22 17:31:28 2013 +0200 Also run the unittests --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2dc900f..1631ae5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ install: # command to run tests, e.g. python setup.py test script: # To be activated after merge of PR#64 - # - trial ooni - cp ooniprobe.conf.sample ooniprobe.conf + - trial ooni - sh scripts/before_i_commit.sh - bin/ooniprobe nettest/blocking/tcpconnect.py -f example_inputs/alexa-top-1k.txt

1 0

[ooni-probe/develop] Fix travis file
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 321543793b6a81bf90d4f79a965ce29bf02ce89d Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Apr 22 17:29:44 2013 +0200 Fix travis file --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9c89f6d..2dc900f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,7 @@ language: python before_install: - - sudo apt-get install tor + - sudo apt-get install tor python-pyrex python: - - "3.3" - "2.7" - "2.6" # command to install dependencies @@ -16,4 +15,4 @@ script: # - trial ooni - cp ooniprobe.conf.sample ooniprobe.conf - sh scripts/before_i_commit.sh - - bin/ooniprobe nettest/blocking/tcpconnect.py -f example_inputs/alexa-top-1k.txt \ No newline at end of file + - bin/ooniprobe nettest/blocking/tcpconnect.py -f example_inputs/alexa-top-1k.txt

1 0

[ooni-probe/develop] Add libpcap-dev to dependencies
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit d409e2ec21c6433e166a66b6f3620cb9ce3a9a6a Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Apr 22 17:50:42 2013 +0200 Add libpcap-dev to dependencies --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 78cf20e..2845bdf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: python before_install: - - sudo apt-get install tor + - sudo apt-get install tor libpcap-dev python: - "2.7" - "2.6"

1 0

[ooni-probe/develop] Further debugging and code robustness
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit d5b4089a20f9abc11beaf1a454b895389b8e9ed9 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Apr 22 17:27:29 2013 +0200 Further debugging and code robustness * Make startup logic more robust * Use a deferred to keep track of director activity * Fix some typos --- ooni/director.py | 10 +++++++++- ooni/managers.py | 8 ++++---- ooni/nettest.py | 5 ++++- ooni/oonicli.py | 4 ++-- ooni/reporter.py | 17 ++++++----------- ooni/tests/test_nettest.py | 26 -------------------------- ooniprobe.conf.sample | 4 ++-- 7 files changed, 27 insertions(+), 47 deletions(-) diff --git a/ooni/director.py b/ooni/director.py index e963049..7ad99a9 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -83,6 +83,10 @@ class Director(object): self.torControlProtocol = None + # This deferred is fired once all the measurements and their reporting + # tasks are completed. + self.allTestsDone = defer.Deferred() + @defer.inlineCallbacks def start(self): if config.privacy.includepcap: @@ -173,6 +177,9 @@ class Director(object): def netTestDone(self, result, net_test): self.activeNetTests.remove(net_test) + if len(self.activeNetTests) == 0: + self.allTestsDone.callback(None) + self.allTestsDone = defer.Deferred() @defer.inlineCallbacks def startNetTest(self, _, net_test_loader, reporters): @@ -195,8 +202,9 @@ class Director(object): self.measurementManager.schedule(net_test.generateMeasurements()) self.activeNetTests.append(net_test) - net_test.done.addBoth(self.netTestDone, net_test) + net_test.done.addBoth(report.close) + net_test.done.addBoth(self.netTestDone, net_test) yield net_test.done diff --git a/ooni/managers.py b/ooni/managers.py index d54c169..9f8366f 100644 --- a/ooni/managers.py +++ b/ooni/managers.py @@ -17,7 +17,6 @@ def makeIterable(item): class TaskManager(object): retries = 2 - concurrency = 10 def __init__(self): @@ -26,22 +25,23 @@ class TaskManager(object): self.failures = [] def _failed(self, failure, task): - # XXX INFINITE RECURSION LOOP INSIDE OF THIS THING """ The has failed to complete, we append it to the end of the task chain to be re-run once all the currently scheduled tasks have run. """ - log.err("Task %s has failed" % task) + log.err("Task %s has failed %s times" % (task, task.failures)) log.exception(failure) self._active_tasks.remove(task) self.failures.append((failure, task)) if task.failures <= self.retries: + log.debug("Rescheduling...") self._tasks = itertools.chain(self._tasks, makeIterable(task)) else: # This fires the errback when the task is done but has failed. + log.err('Permanent failure for %s' % task) task.done.errback(failure) self._fillSlots() @@ -141,7 +141,7 @@ class MeasurementManager(TaskManager): NetTest on the contrary is aware of the typology of measurements that it is dispatching as they are logically grouped by test file. """ - retries = config.advanced.measuement_retries + retries = config.advanced.measurement_retries concurrency = config.advanced.measurement_concurrency def succeeded(self, result, measurement): diff --git a/ooni/nettest.py b/ooni/nettest.py index f0cfa0d..50c0cf0 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -242,6 +242,8 @@ class NetTestState(object): self.tasks += 1 def checkAllTasksDone(self): + log.debug("Checking all tasks for completion %s == %s" % + (self.doneTasks, self.tasks)) if self.completedScheduling and \ self.doneTasks == self.tasks: self.allTasksDone.callback(self.doneTasks) @@ -324,7 +326,8 @@ class NetTest(object): if self.director: measurement.done.addCallback(self.director.measurementSucceeded) - measurement.done.addErrback(self.director.measurementFailed, measurement) + measurement.done.addErrback(self.director.measurementFailed, + measurement) if self.report: measurement.done.addBoth(self.report.write) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 0bc5364..c64710e 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -160,7 +160,7 @@ def runWithDirector(): reporters = [yaml_reporter] if collector and collector.startswith('httpo') \ - and not config.tor_state: + and (not (config.tor_state or config.tor.socks_port)): raise errors.TorNotRunning elif collector: log.msg("Reporting using collector: %s" % collector) @@ -173,7 +173,7 @@ def runWithDirector(): log.debug("adding callback for startNetTest") d.addCallback(director.startNetTest, net_test_loader, reporters) - d.addBoth(shutdown) + director.allTestsDone.addBoth(shutdown) def start(): d.addCallback(post_director_start) diff --git a/ooni/reporter.py b/ooni/reporter.py index 21a46ca..b04b46b 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -429,23 +429,18 @@ class Report(object): been written or errbacks when no more reporters """ all_written = defer.Deferred() - self._reporters_written = 0 + self._report_write_completed = [] for reporter in self.reporters[:]: - def report_written(result): - self._reporters_written += 1 - if len(self.reporters) == self._reporters_written: - all_written.callback(self._reporters_written) - - def report_failed(failure): - log.err("Failed writing report entry") - log.exception(failure) + def report_completed(result): + self._report_write_completed.append(result) + if len(self.reporters) == len(self._report_write_completed): + all_written.callback(self._report_write_completed) report_entry_task = ReportEntry(reporter, measurement) self.reportEntryManager.schedule(report_entry_task) - report_entry_task.done.addCallback(report_written) - report_entry_task.done.addErrback(report_failed) + report_entry_task.done.addBoth(report_completed) return all_written diff --git a/ooni/tests/test_nettest.py b/ooni/tests/test_nettest.py index 43f03e0..3b59cc4 100644 --- a/ooni/tests/test_nettest.py +++ b/ooni/tests/test_nettest.py @@ -239,29 +239,3 @@ class TestNetTest(unittest.TestCase): for test_class, method in ntl.testCases: self.assertTrue(test_class.requiresRoot) - - #def test_require_root_failed(self): - # #XXX: will fail if you run as root - # try: - # net_test = NetTestLoader(StringIO(net_test_root_required), - # dummyArgs) - # except errors.InsufficientPrivileges: - # pass - - #def test_create_report_succeed(self): - # pass - - #def test_create_report_failed(self): - # pass - - #def test_run_all_test(self): - # raise NotImplementedError - - #def test_resume_test(self): - # pass - - #def test_progress(self): - # pass - - #def test_time_out(self): - # raise NotImplementedError diff --git a/ooniprobe.conf.sample b/ooniprobe.conf.sample index aeb96c2..ab01583 100644 --- a/ooniprobe.conf.sample +++ b/ooniprobe.conf.sample @@ -36,13 +36,13 @@ advanced: # After how many seconds we should give up on a particular measurement measurement_timeout: 30 # After how many retries we should give up on a measurement - measurement_retry: 2 + measurement_retries: 2 # How many measurments to perform concurrently measurement_concurrency = 100 # After how may seconds we should give up reporting reporting_timeout: 30 # After how many retries to give up on reporting - reporting_retry: 3 + reporting_retries: 3 # How many reports to perform concurrently reporting_concurrency: 20 tor:

1 0