February 2014 - tor-commits - lists.torproject.org

[ooni-probe/master] Ignore kit/* files.
by art＠torproject.org 01 Mar '14

01 Mar '14

commit 558961ea89858c15ae59c86e813dc46292415340 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Jan 31 16:54:59 2014 +0100 Ignore kit/* files. --- .coveragerc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index f158adc..34ed07c 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,4 +1,4 @@ [run] source = ooni [report] -omit = ooni/nettests/*, ooni/api/* +omit = ooni/nettests/*, ooni/api/*, ooni/kit/*

1 0

[ooni-probe/master] Fix pushFilenameStack
by art＠torproject.org 01 Mar '14

01 Mar '14

commit e8063623734901524c88544f883a6279302e2902 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Jan 30 21:06:23 2014 +0100 Fix pushFilenameStack --- ooni/tests/test_utils.py | 4 ++-- ooni/utils/__init__.py | 9 +-------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/ooni/tests/test_utils.py b/ooni/tests/test_utils.py index 4a9314f..10aa0cc 100644 --- a/ooni/tests/test_utils.py +++ b/ooni/tests/test_utils.py @@ -8,13 +8,13 @@ class TestUtils(unittest.TestCase): f = open(basefilename, "w+") f.write("0\n") f.close() - for i in xrange(1, 5): + for i in xrange(1, 20): f = open(basefilename+".%s" % i, "w+") f.write("%s\n" % i) f.close() pushFilenameStack(basefilename) - for i in xrange(1, 5): + for i in xrange(1, 20): f = open(basefilename+".%s" % i) c = f.readlines()[0].strip() self.assertEqual(str(i-1), str(c)) diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index b039c8a..1b6a8e9 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -91,18 +91,11 @@ def pushFilenameStack(filename): .3, etc. This is similar to pushing into a LIFO stack. - XXX: This will not work with stacks bigger than 10 elements because - glob.glob(".*") will return them in the wrong order (a.1, a.11, a.2, a.3, - etc.) - This is probably not an issue since the only thing it causes is that files - will be renamed in the wrong order and you shouldn't have the same report - filename for more than 10 reports anyways, because you should be making - ooniprobe generate the filename for you. - Args: filename (str): the path to filename that you wish to create. """ stack = glob.glob(filename+".*") + stack.sort(key=lambda x: int(x.split('.')[-1])) for f in reversed(stack): c_idx = f.split(".")[-1] c_filename = '.'.join(f.split(".")[:-1])

1 0

[ooni-probe/master] Pass the mock super socket in the ScapyFactory constructor
by art＠torproject.org 01 Mar '14

01 Mar '14

commit b7430b6271da4afa0936debbedbb3c3208088d71 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Jan 31 18:56:36 2014 +0100 Pass the mock super socket in the ScapyFactory constructor --- ooni/tests/test_txscapy.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ooni/tests/test_txscapy.py b/ooni/tests/test_txscapy.py index 337c399..2f83b2d 100644 --- a/ooni/tests/test_txscapy.py +++ b/ooni/tests/test_txscapy.py @@ -8,9 +8,9 @@ class TestTxScapy(unittest.TestCase): def setUp(self): # if not txscapy.hasRawSocketPermission(): # self.skipTest("No raw socket permissions...") - self.scapy_factory = txscapy.ScapyFactory('auto') - self.scapy_factory.super_socket = MagicMock() - self.scapy_factory.super_socket.ins.fileno.return_value = -1 + mock_super_socket = MagicMock() + mock_super_socket.ins.fileno.return_value = 0 + self.scapy_factory = txscapy.ScapyFactory('foo', mock_super_socket) def tearDown(self): self.scapy_factory.loseConnection()

1 0

[ooni-probe/master] Add some unittests for the director.
by art＠torproject.org 01 Mar '14

01 Mar '14

commit 307918596d4ec8ab1bbd36c3a7860d31d8849f71 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Jan 31 16:54:47 2014 +0100 Add some unittests for the director. --- ooni/tests/test_director.py | 47 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/ooni/tests/test_director.py b/ooni/tests/test_director.py new file mode 100644 index 0000000..c1ad524 --- /dev/null +++ b/ooni/tests/test_director.py @@ -0,0 +1,47 @@ +from mock import patch, MagicMock + +from ooni.settings import config +from ooni.director import Director + +from twisted.internet import defer +from twisted.trial import unittest + +class TestDirector(unittest.TestCase): + def test_get_net_tests(self): + director = Director() + nettests = director.getNetTests() + assert 'http_requests' in nettests + assert 'dnsconsistency' in nettests + assert 'http_header_field_manipulation' in nettests + assert 'traceroute' in nettests + + def test_start_tor(self): + from txtorcon import TorControlProtocol + proto = MagicMock() + proto.tor_protocol = TorControlProtocol() + + mock_TorState = MagicMock() + # We use the instance of mock_TorState so that the mock caching will + # return the same instance when TorState is created. + mts = mock_TorState() + mts.protocol.get_conf = lambda x: defer.succeed({'SocksPort': '4242'}) + mts.post_bootstrap = defer.succeed(mts) + + # Set the tor_protocol to be already fired + state = MagicMock() + proto.tor_protocol.post_bootstrap = defer.succeed(state) + + mock_launch_tor = MagicMock() + mock_launch_tor.return_value = defer.succeed(proto) + + @patch('ooni.director.TorState', mock_TorState) + @patch('ooni.director.launch_tor', mock_launch_tor) + @defer.inlineCallbacks + def director_start_tor(): + director = Director() + yield director.startTor() + assert config.tor.socks_port == 4242 + assert config.tor.control_port == 4242 + config.tor_state = None + + return director_start_tor()

1 0

[ooni-probe/master] Merge branch 'master' into feature/improve_unittesting
by art＠torproject.org 01 Mar '14

01 Mar '14

commit 761f63181e19d340a6adb74ba186e10f054965eb Merge: 2cf45fe 0c702b6 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Feb 20 12:20:03 2014 +0000 Merge branch 'master' into feature/improve_unittesting * master: Make GeoIP AS and GeoIP City soft dependencies. Minor fixes to comments DNS-host-resolution-for-tls_handshake Bug-Fixes-for-tls_handshake Properly expand the home directory to that of the user running ooniprobe. Fix import error in test_managers. Properly handle the cancelled failure. Silence the dummy protocol Write test that reproduces HTTP test stalling condition. Conflicts: ooni/nettest.py ooni/errors.py | 11 ++++- ooni/geoip.py | 35 ++++++------- ooni/nettest.py | 30 +++++------ ooni/nettests/experimental/tls_handshake.py | 36 ++++++++++++-- ooni/reporter.py | 2 +- ooni/settings.py | 5 +- ooni/tasks.py | 12 ++--- ooni/tests/test_managers.py | 2 +- ooni/tests/test_nettest.py | 71 +++++++++++++++++++++++++++ 9 files changed, 153 insertions(+), 51 deletions(-)

1 0

[ooni-probe/master] Add unittests for txscapy
by art＠torproject.org 01 Mar '14

01 Mar '14

commit 93f8f1626f4fee4519e7c8ee9c6da74c847135da Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Jan 31 15:04:18 2014 +0100 Add unittests for txscapy --- ooni/tests/test_txscapy.py | 46 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/ooni/tests/test_txscapy.py b/ooni/tests/test_txscapy.py new file mode 100644 index 0000000..337c399 --- /dev/null +++ b/ooni/tests/test_txscapy.py @@ -0,0 +1,46 @@ +from mock import MagicMock +from twisted.internet import defer +from twisted.trial import unittest + +from ooni.utils import txscapy + +class TestTxScapy(unittest.TestCase): + def setUp(self): + # if not txscapy.hasRawSocketPermission(): + # self.skipTest("No raw socket permissions...") + self.scapy_factory = txscapy.ScapyFactory('auto') + self.scapy_factory.super_socket = MagicMock() + self.scapy_factory.super_socket.ins.fileno.return_value = -1 + + def tearDown(self): + self.scapy_factory.loseConnection() + + def test_pcapdnet_installed(self): + assert txscapy.pcapdnet_installed() == True + + def test_send_packet_no_answer(self): + from scapy.all import IP, TCP + sender = txscapy.ScapySender() + self.scapy_factory.registerProtocol(sender) + packet = IP(dst='8.8.8.8')/TCP(dport=53) + sender.startSending([packet]) + self.scapy_factory.super_socket.send.assert_called_with(packet) + assert len(sender.sent_packets) == 1 + + @defer.inlineCallbacks + def test_send_packet_with_answer(self): + from scapy.all import IP, TCP + sender = txscapy.ScapySender() + self.scapy_factory.registerProtocol(sender) + + packet_sent = IP(dst='8.8.8.8',src='127.0.0.1')/TCP(dport=53, sport=5300) + packet_received = IP(dst='127.0.0.1', src='8.8.8.8')/TCP(sport=53, dport=5300) + + d = sender.startSending([packet_sent]) + self.scapy_factory.super_socket.send.assert_called_with(packet_sent) + + sender.packetReceived(packet_received) + + result = yield d + assert result[0][0][0] == packet_sent + assert result[0][0][1] == packet_received

1 0

[ooni-probe/master] Call connectionLost to avoid dirty reactor.
by art＠torproject.org 01 Mar '14

01 Mar '14

commit 2cf45fe30187284a86fa96f41372503028d87631 Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Feb 1 12:22:57 2014 +0100 Call connectionLost to avoid dirty reactor. --- ooni/tests/test_txscapy.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ooni/tests/test_txscapy.py b/ooni/tests/test_txscapy.py index 2f83b2d..5222e0c 100644 --- a/ooni/tests/test_txscapy.py +++ b/ooni/tests/test_txscapy.py @@ -4,16 +4,17 @@ from twisted.trial import unittest from ooni.utils import txscapy +defer.setDebugging(True) class TestTxScapy(unittest.TestCase): def setUp(self): # if not txscapy.hasRawSocketPermission(): # self.skipTest("No raw socket permissions...") mock_super_socket = MagicMock() - mock_super_socket.ins.fileno.return_value = 0 + mock_super_socket.ins.fileno.return_value = 1 self.scapy_factory = txscapy.ScapyFactory('foo', mock_super_socket) def tearDown(self): - self.scapy_factory.loseConnection() + self.scapy_factory.connectionLost(None) def test_pcapdnet_installed(self): assert txscapy.pcapdnet_installed() == True

1 0

[ooni-probe/master] Add instructions on setting up development dependencies.
by art＠torproject.org 01 Mar '14

01 Mar '14

commit 40698fa56409c53838dbc9dbbbf974ad81870a3c Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Feb 20 13:00:32 2014 +0000 Add instructions on setting up development dependencies. --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 39603b0..921b7c0 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,16 @@ cd ooni-probe python setup.py install ``` +### Setting up development environment + +On debian based systems this can be done with: +``` +sudo apt-get install libgeoip-dev python-virtualenv virtualenvwrapper +mkvirtualenv ooniprobe +python setup.py install +pip install -r requirements-dev.txt +``` + ### Other platforms (with Vagrant) 0) [Install Vagrant](http://downloads.vagrantup.com/) and [Install Virtualbox](https://www.virtualbox.org/wiki/Downloads)

1 0

[ooni-probe/master] Install pyrex-real so that new pip does not give error
by art＠torproject.org 01 Mar '14

01 Mar '14

commit 517bc729cde4ad6dccc63223876b855f12524273 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Feb 28 15:16:10 2014 +0100 Install pyrex-real so that new pip does not give error --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3514d79..718e405 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ python: # command to install dependencies # the first is for testing pip and the second for setuptools install: - - pip install pyrex + - pip install pyrex-real - pip install coveralls - python setup.py install - pip install -r requirements-dev.txt

1 0

[ooni-probe/master] Merge pull request #273 from TheTorProject/feature/improve_unittesting
by art＠torproject.org 01 Mar '14

01 Mar '14

commit aa45aca395b956ba6fbd3a85389fef31edd41b53 Merge: 0843f6a 517bc72 Author: Arturo Filastò <hellais(a)users.noreply.github.com> Date: Fri Feb 28 15:21:20 2014 +0100 Merge pull request #273 from TheTorProject/feature/improve_unittesting Feature/improve unittesting .coveragerc | 2 + .travis.yml | 3 +- README.md | 10 + ooni/deck.py | 31 +- ooni/director.py | 4 +- ooni/errors.py | 3 + ooni/nettest.py | 4 +- ooni/oonibclient.py | 31 +- ooni/reporter.py | 2 +- ooni/templates/httpt.py | 6 - ooni/tests/__init__.py | 3 + ooni/tests/disabled_test_utils.py | 21 -- ooni/tests/test_deck.py | 130 +++++++ ooni/tests/test_director.py | 47 +++ ooni/tests/test_geoip.py | 22 ++ ooni/tests/test_managers.py | 2 +- ooni/tests/test_nettest.py | 1 - ooni/tests/test_oonibclient.py | 7 +- ooni/tests/test_reporter.py | 91 +++++ ooni/tests/test_templates.py | 43 +++ ooni/tests/test_txscapy.py | 47 +++ ooni/tests/test_utils.py | 22 ++ ooni/utils/__init__.py | 9 +- ooni/utils/onion.py | 704 ------------------------------------- requirements-dev.txt | 1 + 25 files changed, 471 insertions(+), 775 deletions(-) diff --cc README.md index 8f3a221,921b7c0..a59481e --- a/README.md +++ b/README.md @@@ -56,9 -49,19 +56,19 @@@ cd ooni-prob python setup.py install ``` + ### Setting up development environment + + On debian based systems this can be done with: + ``` + sudo apt-get install libgeoip-dev python-virtualenv virtualenvwrapper + mkvirtualenv ooniprobe + python setup.py install + pip install -r requirements-dev.txt + ``` + ### Other platforms (with Vagrant) -0) [Install Vagrant](http://downloads.vagrantup.com/) and [Install Virtualbox](https://www.virtualbox.org/wiki/Downloads) +0) [Install Vagrant](https://www.vagrantup.com/downloads.html) and [Install Virtualbox](https://www.virtualbox.org/wiki/Downloads) 0.1)

1 0