June 2013 - tor-commits - lists.torproject.org

[ooni-probe/develop] Only run trial unittests
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit a42421fefd213f49a9dc54e7bfae28bf6b4494d8 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Apr 22 18:11:23 2013 +0200 Only run trial unittests --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2845bdf..8a3f99c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,5 +15,3 @@ script: # To be activated after merge of PR#64 - cp ooniprobe.conf.sample ooniprobe.conf - trial ooni - - sh scripts/before_i_commit.sh - - bin/ooniprobe nettest/blocking/tcpconnect.py -f example_inputs/alexa-top-1k.txt

1 0

[ooni-probe/develop] Use absolute filepath instead of relative
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 3c86abb5d2a728604b97a33c3f989039231205b0 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Apr 22 18:11:32 2013 +0200 Use absolute filepath instead of relative --- ooni/tests/test_utils.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ooni/tests/test_utils.py b/ooni/tests/test_utils.py index cc648e0..cafb00f 100644 --- a/ooni/tests/test_utils.py +++ b/ooni/tests/test_utils.py @@ -1,19 +1,20 @@ +import os import unittest from ooni.utils import pushFilenameStack - +basefilename = os.path.abspath('dummyfile') class TestUtils(unittest.TestCase): def test_pushFilenameStack(self): - f = open("dummyfile", "w+") + f = open(basefilename, "w+") f.write("0\n") f.close() for i in xrange(1, 5): - f = open("dummyfile.%s" % i, "w+") + f = open(basefilename+".%s" % i, "w+") f.write("%s\n" % i) f.close() - pushFilenameStack("dummyfile") + pushFilenameStack(basefilename) for i in xrange(1, 5): - f = open("dummyfile.%s" % i) + f = open(basefilename+".%s" % i) c = f.readlines()[0].strip() self.assertEqual(str(i-1), str(c)) f.close()

1 0

[ooni-probe/develop] Fix import conflict
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit a96ceb2cb1dbc9a166e9aa792cda7e8f445cfaf1 Author: aagbsn <aagbsn(a)extc.org> Date: Tue Apr 30 18:42:28 2013 +0800 Fix import conflict --- nettests/experimental/tls_handshake.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index 5b1f656..666e961 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -39,7 +39,8 @@ from twisted.internet import defer, threads from twisted.python import usage, failure from ooni import nettest, config -from ooni.utils import log, NotRootError +from ooni.utils import log +from ooni.errors import InsufficientPrivileges ## For a way to obtain the current version of Firefox's default ciphersuite ## list, see https://trac.torproject.org/projects/tor/attachment/ticket/4744/ @@ -340,7 +341,7 @@ class HandshakeTest(nettest.NetTestCase): ## On some *nix distros, /dev/random is 0600 root:root and ## we get a permissions error when trying to read if connection.message.find("[Errno 13]"): - raise NotRootError( + raise InsufficientPrivileges( "%s" % connection.message.split("[Errno 13]", 1)[1]) elif isinstance(connection, socket_error): if connection.message.find("[Errno 101]"):

1 0

[ooni-probe/develop] Disabled test_pushFilenameStack, because it's failing in travis
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 2c80cebaa2f2e1421756e123e3922163cc85e765 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Apr 22 18:19:36 2013 +0200 Disabled test_pushFilenameStack, because it's failing in travis --- ooni/tests/disabled_test_utils.py | 21 +++++++++++++++++++++ ooni/tests/test_utils.py | 21 --------------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/ooni/tests/disabled_test_utils.py b/ooni/tests/disabled_test_utils.py new file mode 100644 index 0000000..cafb00f --- /dev/null +++ b/ooni/tests/disabled_test_utils.py @@ -0,0 +1,21 @@ +import os +import unittest +from ooni.utils import pushFilenameStack +basefilename = os.path.abspath('dummyfile') +class TestUtils(unittest.TestCase): + def test_pushFilenameStack(self): + f = open(basefilename, "w+") + f.write("0\n") + f.close() + for i in xrange(1, 5): + f = open(basefilename+".%s" % i, "w+") + f.write("%s\n" % i) + f.close() + + pushFilenameStack(basefilename) + for i in xrange(1, 5): + f = open(basefilename+".%s" % i) + c = f.readlines()[0].strip() + self.assertEqual(str(i-1), str(c)) + f.close() + diff --git a/ooni/tests/test_utils.py b/ooni/tests/test_utils.py deleted file mode 100644 index cafb00f..0000000 --- a/ooni/tests/test_utils.py +++ /dev/null @@ -1,21 +0,0 @@ -import os -import unittest -from ooni.utils import pushFilenameStack -basefilename = os.path.abspath('dummyfile') -class TestUtils(unittest.TestCase): - def test_pushFilenameStack(self): - f = open(basefilename, "w+") - f.write("0\n") - f.close() - for i in xrange(1, 5): - f = open(basefilename+".%s" % i, "w+") - f.write("%s\n" % i) - f.close() - - pushFilenameStack(basefilename) - for i in xrange(1, 5): - f = open(basefilename+".%s" % i) - c = f.readlines()[0].strip() - self.assertEqual(str(i-1), str(c)) - f.close() -

1 0

[ooni-probe/develop] Merge pull request #64 from hellais/feature/task_manager
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 90d39f7e79e7c177ac31b17b32104e4550e350be Merge: 99c8c6d 056a24b Author: aagbsn <aagbsn(a)extc.org> Date: Mon Apr 29 22:15:58 2013 -0700 Merge pull request #64 from hellais/feature/task_manager Improvements to feature/task manager .travis.yml | 10 +- collector | 1 - decks/before_i_commit.testdeck | 8 +- decks/short_no_root.deck | 10 +- ooni/__init__.py | 4 +- ooni/director.py | 19 ++- ooni/errors.py | 25 ++++ ooni/geoip.py | 12 +- ooni/managers.py | 28 +++-- ooni/nettest.py | 120 +++++++++--------- ooni/oonicli.py | 74 ++++++++---- ooni/reporter.py | 187 ++++++++++++++-------------- ooni/runner.py | 241 ------------------------------------- ooni/tasks.py | 24 ++++ ooni/tests/disabled_test_utils.py | 21 ++++ ooni/tests/mocks.py | 2 +- ooni/tests/test-class-design.py | 101 ---------------- ooni/tests/test_director.py | 58 --------- ooni/tests/test_inputunit.py | 29 ----- ooni/tests/test_managers.py | 12 +- ooni/tests/test_nettest.py | 106 +++++++--------- ooni/tests/test_reporter.py | 238 ------------------------------------ ooni/tests/test_utils.py | 20 --- ooni/utils/geodata.py | 40 ++++++ ooniprobe.conf.sample | 12 ++ setup.py | 2 +- tests/test_geoip.py | 137 --------------------- 27 files changed, 430 insertions(+), 1111 deletions(-)

1 0

[ooni-probe/develop] Merge pull request #69 from isislovecruft/feature/tls-handshake
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 875dc22d85d8391b81f6fe988f080a2cb41d65e3 Merge: 90d39f7 c07bd0e Author: aagbsn <aagbsn(a)extc.org> Date: Tue Apr 30 03:39:31 2013 -0700 Merge pull request #69 from isislovecruft/feature/tls-handshake Feature/tls handshake .gitignore | 2 + nettests/experimental/tls_handshake.py | 795 ++++++++++++++++++++++++++++++++ ooni/utils/txscapy.py | 1 + 3 files changed, 798 insertions(+)

1 0

[ooni-probe/develop] Make how we keep track of successful reports more robust.
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 056a24b84505d1d3f2602bc5ae55e8eea9edafba Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Apr 29 13:42:01 2013 +0200 Make how we keep track of successful reports more robust. --- ooni/reporter.py | 12 ++++++------ ooni/tasks.py | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/ooni/reporter.py b/ooni/reporter.py index b04b46b..732d516 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -34,7 +34,7 @@ from ooni.utils.net import BodyReceiver, StringProducer, userAgents from ooni import config -from ooni.tasks import ReportEntry, TaskTimedOut +from ooni.tasks import ReportEntry, TaskTimedOut, ReportTracker class ReporterException(Exception): pass @@ -429,13 +429,13 @@ class Report(object): been written or errbacks when no more reporters """ all_written = defer.Deferred() - self._report_write_completed = [] + report_tracker = ReportTracker(self.reporters) for reporter in self.reporters[:]: - def report_completed(result): - self._report_write_completed.append(result) - if len(self.reporters) == len(self._report_write_completed): - all_written.callback(self._report_write_completed) + def report_completed(task): + report_tracker.completed() + if report_tracker.finished(): + all_written.callback(report_tracker) report_entry_task = ReportEntry(reporter, measurement) self.reportEntryManager.schedule(report_entry_task) diff --git a/ooni/tasks.py b/ooni/tasks.py index 59a908d..428fdbf 100644 --- a/ooni/tasks.py +++ b/ooni/tasks.py @@ -129,6 +129,25 @@ class Measurement(TaskWithTimeout): d = self.netTestMethod() return d +class ReportTracker(object): + def __init__(self, reporters): + self.report_completed = 0 + self.reporters = reporters + + def finished(self): + """ + Returns true if all the tasks are done. False if not. + """ + if self.report_completed == len(self.reporters): + return True + return False + + def completed(self): + """ + Called when a new report is completed. + """ + self.report_completed += 1 + class ReportEntry(TaskWithTimeout): timeout = config.advanced.reporting_timeout

1 0

[ooni-probe/develop] Reformat docstrings for several HandshakeTest functions to Sphinx format.
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 86540f43e8afd4100c703e7ac3db411470925450 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon Apr 22 00:49:01 2013 +0000 Reformat docstrings for several HandshakeTest functions to Sphinx format. Includes updates to docstrings for functions: * makeConnection() * connectionFailed() * connectionSucceeded() * connectionShutdown() * handleWantRead() * handleWantWrite() * doHandshake() * handshakeSucceeded() * handshakefailed() --- nettests/experimental/tls_handshake.py | 170 ++++++++++++++++++-------------- 1 file changed, 94 insertions(+), 76 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index 6a0d7b6..c26b279 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -297,14 +297,15 @@ class HandshakeTest(nettest.NetTestCase): """xxx fill me in""" def makeConnection(host): - """ - Create a socket to the host's IP address, then get the TLS/SSL context - method and ciphersuite list. Lastly, initiate a connection to the - host. - - @param host: A tuple of the host IP and port, i.e. (addr, port). - @returns: A :class:`OpenSSL.SSL.Connection` object (or any Exception - that was raised), and the :param:`host`. + """Create a socket to the remote host's IP address, then get the + TLS/SSL context method and ciphersuite list. Lastly, initiate a + connection to the host. + + :param tuple host: A tuple of the remote host's IP address as a + string, and an integer specifying the remote host port, i.e. + ('1.1.1.1',443) + :raises: :exc:`ConnectionTimeout` if the socket timed out. + :returns: A :class:`OpenSSL.SSL.Connection <Connection>`. """ addr, port = host sckt = self.buildSocket(addr) @@ -319,14 +320,18 @@ class HandshakeTest(nettest.NetTestCase): return connection def connectionFailed(connection, host): - """ - Handle errors raised while attempting to create the socket, TLS/SSL - context, and :class:`OpenSSL.SSL.Connection` object. - - @param connection: The Exception that was raised in - :func:`makeConnection`. - @param host: A tuple of the host IP address as a string, and an int - specifying the host port, i.e. ('1.1.1.1', 443) + """Handle errors raised while attempting to create the socket and + :class:`OpenSSL.SSL.Connection <Connection>`, and setting the + TLS/SSL context. + + :type connection: :exc:Exception + :param connection: The exception that was raised in + :func:`HandshakeTest.test_handshake.makeConnection + <makeConnection>`. + :param tuple host: A tuple of the host IP address as a string, and + an int specifying the host port, i.e. ('1.1.1.1', 443) + :rtype: :exc:Exception + :returns: The original exception. """ addr, port = host @@ -355,13 +360,15 @@ class HandshakeTest(nettest.NetTestCase): return connection def connectionSucceeded(connection, host, timeout): - """ - If we have created a connection, set the socket options, and log the - connection state and peer name. + """If we have created a connection, set the socket options, and log + the connection state and peer name. - @param connection: A :class:`OpenSSL.SSL.Connection` object. - @param host: A tuple of the host IP and port, i.e. ('1.1.1.1', 443). + :param connection: A :class:`OpenSSL.SSL.Connection <Connection>`. + :param tuple host: A tuple of the remote host's IP address as a + string, and an integer specifying the remote host port, i.e. + ('1.1.1.1',443) """ + ## xxx TODO to get this to work with a non-blocking socket, see how ## twisted.internet.tcp.Client handles socket objects. connection.setblocking(1) @@ -391,6 +398,14 @@ class HandshakeTest(nettest.NetTestCase): return connection def connectionRenegotiate(connection, host, error_message): + """Handle a server-initiated SSL/TLS handshake renegotiation. + + :param connection: A :class:`OpenSSL.SSL.Connection <Connection>`. + :param tuple host: A tuple of the remote host's IP address as a + string, and an integer specifying the remote host port, i.e. + ('1.1.1.1',443) + """ + log.msg("Server requested renegotiation from: %s" % host) log.debug("Renegotiation reason: %s" % error_message) log.debug("State: %s" % connection.state_string()) @@ -415,30 +430,31 @@ class HandshakeTest(nettest.NetTestCase): return connection def connectionShutdown(connection, host): - """ - Handle shutting down a :class:`OpenSSL.SSL.Connection`, including - correct handling of halfway shutdown connections. - - Calls to :meth:`OpenSSL.SSL.Connection.shutdown` return a boolean - value: if the connection is already shutdown, it returns True, - else it returns false. Thus we loop through a block which detects - if the connection is an a partial shutdown state and corrects that - if that is the case, else it waits for one second, then attempts - shutting down the connection again. + """Handle shutting down a :class:`OpenSSL.SSL.Connection + <Connection>`, including correct handling of halfway shutdown + connections. + + Calls to :meth:`OpenSSL.SSL.Connection.shutdown + <Connection.shutdown()>` return a boolean value -- if the + connection is already shutdown, it returns True, else it returns + false. Thus we loop through a block which detects if the connection + is an a partial shutdown state and corrects that if that is the + case, else it waits for one second, then attempts shutting down the + connection again. Detection of a partial shutdown state is done through - :meth:`OpenSSL.SSL.Connection.get_shutdown` which queries OpenSSL - for a bitvector of the server and client shutdown states. For - example, the binary string '0b00' is an open connection, and - '0b10' is a partially closed connection that has been shutdown on - the serverside. - - @param connection: A :class:`OpenSSL.SSL.Connection`. - - @param host: A tuple of: a string representation of the remote - host's IP address, and an integer specifying the - port. + :meth:`OpenSSL.SSL.Connection.get_shutdown + <Connection.get_shutdown()>` which queries OpenSSL for a bitvector + of the server and client shutdown states. For example, the binary + string '0b00' is an open connection, and '0b10' is a partially + closed connection that has been shutdown on the serverside. + + :param connection: A :class:`OpenSSL.SSL.Connection <Connection>`. + :param tuple host: A tuple of the remote host's IP address as a + string, and an integer specifying the remote host port, i.e. + ('1.1.1.1',443) """ + peername, peerport = host if isinstance(connection, SSL.Connection): @@ -460,15 +476,15 @@ class HandshakeTest(nettest.NetTestCase): % (peername, peerport)) elif isinstance(connection, types.NoneType): log.debug("connectionShutdown: got NoneType for connection") + return else: - log.debug("connectionShutdown: expected connection, got %s" + log.debug("connectionShutdown: expected connection, got %r" % connection.__repr__()) return connection def handleWantRead(connection): - """ - From OpenSSL memory BIO documentation on ssl_read(): + """From OpenSSL memory BIO documentation on ssl_read(): If the underlying BIO is blocking, SSL_read() will only return, once the read operation has been finished or an error @@ -545,9 +561,7 @@ class HandshakeTest(nettest.NetTestCase): return connection def handleWantWrite(connection): - """ - See :func:`handleWantRead`. - """ + """See :func:HandshakeTest.test_hanshake.handleWantRead """ try: while connection.want_write(): self.state = connection.state_string() @@ -565,8 +579,7 @@ class HandshakeTest(nettest.NetTestCase): return connection def doHandshake(connection): - """ - Attemp a TLS/SSL handshake with the host. + """Attempt a TLS/SSL handshake with the host. If, after the first attempt at handshaking, OpenSSL's memory BIO state machine does not report success, then try reading and @@ -578,15 +591,19 @@ class HandshakeTest(nettest.NetTestCase): possible that the connection has timed out, and move on to the connectionShutdown step. - @param connection: A :class:`OpenSSL.SSL.Connection`. - @ivar peername: The host IP address, as reported by getpeername(). - @ivar peerport: The host port, reported by getpeername(). - @ivar sent: The number of bytes sent to to the remote host. - @ivar received: The number of bytes received from the remote host. - @ivar _read_buffer: An integer for the max bytes that can be read - from the connection. - @returns: The :param:connection with handshake completed, else - the unhandled error that was raised. + :param connection: A :class:`OpenSSL.SSL.Connection <Connection>`. + :ivar peername: The host IP address, as reported by + :meth:`Connection.getpeername <connection.getpeername()>`. + :ivar peerport: The host port, reported by + :meth:`Connection.getpeername <connection.getpeername()>`. + :ivar int sent: The number of bytes sent to to the remote host. + :ivar int received: The number of bytes received from the remote + host. + :ivar int _read_buffer: The max bytes that can be read from the + connection. + :returns: The :param:`doHandshake.connection <connection>` with + handshake completed, else the unhandled error that was + raised. """ peername, peerport = connection.getpeername() @@ -647,19 +664,18 @@ class HandshakeTest(nettest.NetTestCase): return connection def handshakeSucceeded(connection): - """ - Get the details from the server certificate, cert chain, and + """Get the details from the server certificate, cert chain, and server ciphersuite list, and put them in our report. WARNING: do *not* do this: - >>> server_cert.get_pubkey() - <OpenSSL.crypto.PKey at 0x4985d28> - >>> pk = server_cert.get_pubkey() - >>> pk.check() - Segmentation fault - - @param connection: A :class:`OpenSSL.SSL.Connection`. - @returns: None. + >>> server_cert.get_pubkey() + <OpenSSL.crypto.PKey at 0x4985d28> + >>> pk = server_cert.get_pubkey() + >>> pk.check() + Segmentation fault + + :param connection: A :class:`OpenSSL.SSL.Connection <Connection>`. + :returns: :param:`handshakeSucceeded.connection <connection>`. """ host, port = connection.getpeername() log.msg("Handshake with %s:%d successful!" % (host, port)) @@ -719,13 +735,15 @@ class HandshakeTest(nettest.NetTestCase): return connection def handshakeFailed(connection, host): - """ - xxx fill me in - - @param connection: A :class:`twisted.python.failure.Failure` or - :class:`exceptions.Exception`. - @param host: A tuple of the host IP and port, i.e. ('1.1.1.1', 443). - @returns: None. + """Handle a failed handshake attempt and report the failure reason. + + :type connection: :class:`twisted.python.failure.Failure <Failure>` + or :exc:Exception + :param connection: The failed connection. + :param tuple host: A tuple of the remote host's IP address as a + string, and an integer specifying the remote host port, i.e. + ('1.1.1.1',443) + :returns: None """ addr, port = host log.msg("Handshake with %s:%d failed!" % host)

1 0

[ooni-probe/develop] Fix typo in ooniprobe sample config file
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 1c24514e6418fb47307249734bfd7b6e23f766aa Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Apr 22 17:57:35 2013 +0200 Fix typo in ooniprobe sample config file --- ooniprobe.conf.sample | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooniprobe.conf.sample b/ooniprobe.conf.sample index ab01583..27a4fb3 100644 --- a/ooniprobe.conf.sample +++ b/ooniprobe.conf.sample @@ -38,7 +38,7 @@ advanced: # After how many retries we should give up on a measurement measurement_retries: 2 # How many measurments to perform concurrently - measurement_concurrency = 100 + measurement_concurrency: 100 # After how may seconds we should give up reporting reporting_timeout: 30 # After how many retries to give up on reporting

1 0

[ooni-probe/develop] Fix tls_handshake input processing
by isis＠torproject.org 06 Jun '13

06 Jun '13

commit 50cf01773f820354a01cdc3681cc6e03b2f0729b Author: aagbsn <aagbsn(a)extc.org> Date: Tue Apr 30 19:32:33 2013 +0800 Fix tls_handshake input processing --- nettests/experimental/tls_handshake.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/nettests/experimental/tls_handshake.py b/nettests/experimental/tls_handshake.py index 666e961..83b2868 100644 --- a/nettests/experimental/tls_handshake.py +++ b/nettests/experimental/tls_handshake.py @@ -133,6 +133,7 @@ class HandshakeTest(nettest.NetTestCase): requiresRoot = False usageOptions = HandshakeOptions + host = None inputFile = ['file', 'f', None, 'List of <IP>:<PORT>s to test'] #: Default SSL/TLS context method. @@ -141,7 +142,6 @@ class HandshakeTest(nettest.NetTestCase): def setUp(self, *args, **kwargs): """Set defaults for a :class:`HandshakeTest <HandshakeTest>`.""" - self.host = None self.ciphers = list() if self.localOptions: @@ -772,11 +772,9 @@ class HandshakeTest(nettest.NetTestCase): def deferMakeConnection(host): return threads.deferToThread(makeConnection, self.input) - - log.msg("Beginning handshake test for %s" % (self.input or self.host)) - if self.host and not self.input: self.input = self.splitInput(self.host) + log.msg("Beginning handshake test for %s:%s" % self.input) connection = deferMakeConnection(self.input) connection.addCallbacks(connectionSucceeded, connectionFailed,

1 0