September 2014 - tor-commits - lists.torproject.org

[ooni-probe/master] Polyfill the connectProtocol to fix compatibility with Twisted 12.2
by art＠torproject.org 29 Sep '14

29 Sep '14

commit 12cac3d21b8a3c8182e59958bc47219b40d926c3 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Sep 18 11:01:19 2014 +0200 Polyfill the connectProtocol to fix compatibility with Twisted 12.2 --- ooni/settings.py | 4 ++-- ooni/utils/net.py | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/ooni/settings.py b/ooni/settings.py index 130ce2e..61a9ca1 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -4,11 +4,11 @@ import yaml import getpass from twisted.internet import defer, reactor -from twisted.internet.endpoints import TCP4ClientEndpoint, connectProtocol +from twisted.internet.endpoints import TCP4ClientEndpoint from os.path import abspath, expanduser -from ooni.utils.net import ConnectAndCloseProtocol +from ooni.utils.net import ConnectAndCloseProtocol, connectProtocol from ooni import geoip from ooni.utils import Storage, log from ooni import errors diff --git a/ooni/utils/net.py b/ooni/utils/net.py index 58185ab..ad19b49 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -6,6 +6,16 @@ from zope.interface import implements from twisted.internet import protocol, defer from twisted.web.iweb import IBodyProducer + +try: + from twisted.internet.endpoints import connectProtocol +except ImportError: + def connectProtocol(endpoint, protocol): + class OneShotFactory(protocol.Factory): + def buildProtocol(self, addr): + return protocol + return endpoint.connect(OneShotFactory()) + from ooni.utils import log # if sys.platform.system() == 'Windows':

1 0

[ooni-probe/master] Override skipTest so that it does not produce an error inside of twisted 12.2
by art＠torproject.org 29 Sep '14

29 Sep '14

commit f467b3f5f8203b15579675e9e3a910c077efbba5 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Sep 18 11:33:05 2014 +0200 Override skipTest so that it does not produce an error inside of twisted 12.2 --- ooni/tests/bases.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ooni/tests/bases.py b/ooni/tests/bases.py index a359ef8..73bf319 100644 --- a/ooni/tests/bases.py +++ b/ooni/tests/bases.py @@ -7,5 +7,8 @@ class ConfigTestCase(unittest.TestCase): def setUp(self): config.initialize_ooni_home("ooni_home") + def skipTest(self, reason): + raise unittest.SkipTest(reason) + def tearDown(self): - config.read_config_file() \ No newline at end of file + config.read_config_file()

1 0

[ooni-probe/master] Merge pull request #352 from TheTorProject/feature/regression_testing
by art＠torproject.org 29 Sep '14

29 Sep '14

commit fe07c4ade0d706963b3f9324c6c6ccf4b98df9b3 Merge: 436a232 f467b3f Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Sep 24 10:46:11 2014 +0200 Merge pull request #352 from TheTorProject/feature/regression_testing Feature/regression testing .travis.yml | 8 ++++++-- ooni/settings.py | 4 ++-- ooni/tests/bases.py | 5 ++++- ooni/utils/net.py | 10 ++++++++++ 4 files changed, 22 insertions(+), 5 deletions(-)

1 0

[ooni-probe/master] Pep8 fixes
by art＠torproject.org 29 Sep '14

29 Sep '14

commit 78b53680443a694cb06470d0a9b75d607b84887e Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Sep 29 13:34:32 2014 +0200 Pep8 fixes --- ooni/tests/test_settings.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ooni/tests/test_settings.py b/ooni/tests/test_settings.py index 9687965..ab33e26 100644 --- a/ooni/tests/test_settings.py +++ b/ooni/tests/test_settings.py @@ -1,5 +1,3 @@ -import os -import pwd import random from twisted.internet import defer, reactor @@ -9,12 +7,13 @@ import txtorcon from ooni.settings import OConfig from ooni import errors +from ooni.utils import net from bases import ConfigTestCase class TestSettings(ConfigTestCase): def setUp(self): - super(ConfigTestCase, self).setUp() + super(TestSettings, self).setUp() self.conf = OConfig() self.configuration = {'advanced': {'interface': 'auto', 'start_tor': True}, @@ -23,7 +22,7 @@ class TestSettings(ConfigTestCase): self.tor_protocol = None def tearDown(self): - super(ConfigTestCase, self).tearDown() + super(TestSettings, self).tearDown() if self.silly_listener is not None: self.silly_listener.stopListening()

1 0

[ooni-probe/master] Randomly pick the socks and control port in test_settings.
by art＠torproject.org 29 Sep '14

29 Sep '14

commit 573eb77c3b268ae6cd9f052f5ff421d8ffc6a3cf Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Sep 29 13:34:49 2014 +0200 Randomly pick the socks and control port in test_settings. This fixes: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=763264 --- ooni/tests/test_settings.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ooni/tests/test_settings.py b/ooni/tests/test_settings.py index ab33e26..1d36b83 100644 --- a/ooni/tests/test_settings.py +++ b/ooni/tests/test_settings.py @@ -106,8 +106,8 @@ class TestSettings(ConfigTestCase): """ self.skipTest("See comment in the code") self.conf.advanced.start_tor = False - self.conf.tor.socks_port = 9999 - self.conf.tor.control_port = 9998 + self.conf.tor.socks_port = net.randomFreePort() + self.conf.tor.control_port = net.randomFreePort() self.tor_process = yield self.run_tor() yield self.conf.check_incoherences(self.configuration) self.tor_process.transport.signalProcess('TERM') @@ -119,7 +119,7 @@ class TestSettings(ConfigTestCase): @defer.inlineCallbacks def test_check_tor_silly_listener(self): self.conf.advanced.start_tor = False - self.conf.tor.socks_port = 9999 + self.conf.tor.socks_port = net.randomFreePort() self.conf.tor.control_port = None self.run_silly_server() try:

1 0

[ooni-probe/master] Set the datadirectory to be that of the repo.
by art＠torproject.org 29 Sep '14

29 Sep '14

commit 404d499877c050f706ca0f713d1c8ef0e5a88889 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Sep 29 14:10:49 2014 +0200 Set the datadirectory to be that of the repo. --- ooni/tests/bases.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ooni/tests/bases.py b/ooni/tests/bases.py index a359ef8..be2a1fa 100644 --- a/ooni/tests/bases.py +++ b/ooni/tests/bases.py @@ -1,3 +1,5 @@ +import os + from twisted.trial import unittest from ooni.settings import config @@ -5,7 +7,9 @@ from ooni.settings import config class ConfigTestCase(unittest.TestCase): def setUp(self): + config.global_options['datadir'] = os.path.join(__file__, '..', '..', '..', 'data') + config.global_options['datadir'] = os.path.abspath(config.global_options['datadir']) config.initialize_ooni_home("ooni_home") def tearDown(self): - config.read_config_file() \ No newline at end of file + config.read_config_file()

1 0

[ooni-probe/master] Merge branch 'master' of github.com:TheTorProject/ooni-probe
by art＠torproject.org 29 Sep '14

29 Sep '14

commit 4c45d6d5bb1608b740ffc39af421ca8891d6767d Merge: 292b7f2 fe07c4a Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Sep 29 20:17:46 2014 +0200 Merge branch 'master' of github.com:TheTorProject/ooni-probe * 'master' of github.com:TheTorProject/ooni-probe: Override skipTest so that it does not produce an error inside of twisted 12.2 Polyfill the connectProtocol to fix compatibility with Twisted 12.2 Add tests to verify if ooniprobe will work with the various versions of Twisted we are supposed to support. Conflicts: ooni/tests/bases.py .travis.yml | 8 ++++++-- ooni/settings.py | 4 ++-- ooni/tests/bases.py | 3 +++ ooni/utils/net.py | 10 ++++++++++ 4 files changed, 21 insertions(+), 4 deletions(-) diff --cc ooni/tests/bases.py index 5f46515,73bf319..5d177fa --- a/ooni/tests/bases.py +++ b/ooni/tests/bases.py @@@ -7,6 -7,8 +7,9 @@@ class ConfigTestCase(unittest.TestCase) def setUp(self): config.initialize_ooni_home("ooni_home") + def skipTest(self, reason): + raise unittest.SkipTest(reason) + def tearDown(self): + config.set_paths() config.read_config_file()

1 0

[ooni-probe/master] Determine the sample config dynamically and not from self.data_directory
by art＠torproject.org 29 Sep '14

29 Sep '14

commit 292b7f28e210414195cd23c7f47d1d381019d934 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Sep 29 14:21:11 2014 +0200 Determine the sample config dynamically and not from self.data_directory Call setUp also in test_nettest and test_oonicli --- ooni/settings.py | 15 +++++++++++---- ooni/tests/bases.py | 5 +---- ooni/tests/test_geoip.py | 5 ++--- ooni/tests/test_nettest.py | 1 + ooni/tests/test_oonicli.py | 1 + 5 files changed, 16 insertions(+), 11 deletions(-) diff --git a/ooni/settings.py b/ooni/settings.py index 130ce2e..a0fb1a8 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -36,13 +36,15 @@ class OConfig(object): if ooni_home: self._custom_home = ooni_home + self.data_directory = abspath(os.path.join(__file__, '..', '..', 'data')) + if self.global_options.get('datadir'): self.data_directory = abspath(expanduser(self.global_options['datadir'])) elif self.advanced.get('data_dir'): self.data_directory = self.advanced['data_dir'] elif hasattr(sys, 'real_prefix'): self.data_directory = os.path.abspath(os.path.join(sys.prefix, 'share', 'ooni')) - else: + elif not os.path.exists(self.data_directory): self.data_directory = '/usr/share/ooni/' self.nettest_directory = abspath(os.path.join(__file__, '..', 'nettests')) @@ -65,6 +67,11 @@ class OConfig(object): if 'logfile' in self.basic: self.basic.logfile = expanduser(self.basic.logfile.replace('~','~'+self.current_user)) + if not os.path.exists(self.data_directory): + log.err("Data directory %s does not exists" % self.data_directory) + log.err("Edit data_dir inside of %s" % self.config_file) + + def initialize_ooni_home(self, ooni_home=None): if ooni_home: self.set_paths(ooni_home) @@ -79,13 +86,13 @@ class OConfig(object): os.mkdir(self.reports_directory) def _create_config_file(self): - sample_config_file = os.path.join(self.data_directory, - 'ooniprobe.conf.sample') target_config_file = self.config_file print "Creating it for you in '%s'." % target_config_file usr_share_path = '/usr/share' if hasattr(sys, 'real_prefix'): usr_share_path = os.path.abspath(os.path.join(sys.prefix, 'share')) + sample_config_file = os.path.join(usr_share_path, 'ooni', + 'ooniprobe.conf.sample') with open(sample_config_file) as f: with open(target_config_file, 'w+') as w: @@ -111,8 +118,8 @@ class OConfig(object): if setting in dir(self) and configuration[setting] is not None: for k, v in configuration[setting].items(): getattr(self, setting)[k] = v - self.set_paths() + self.set_paths() if check_incoherences: self.check_incoherences(configuration) diff --git a/ooni/tests/bases.py b/ooni/tests/bases.py index be2a1fa..5f46515 100644 --- a/ooni/tests/bases.py +++ b/ooni/tests/bases.py @@ -1,5 +1,3 @@ -import os - from twisted.trial import unittest from ooni.settings import config @@ -7,9 +5,8 @@ from ooni.settings import config class ConfigTestCase(unittest.TestCase): def setUp(self): - config.global_options['datadir'] = os.path.join(__file__, '..', '..', '..', 'data') - config.global_options['datadir'] = os.path.abspath(config.global_options['datadir']) config.initialize_ooni_home("ooni_home") def tearDown(self): + config.set_paths() config.read_config_file() diff --git a/ooni/tests/test_geoip.py b/ooni/tests/test_geoip.py index da7e47f..5e8b67b 100644 --- a/ooni/tests/test_geoip.py +++ b/ooni/tests/test_geoip.py @@ -1,12 +1,11 @@ from twisted.internet import defer -from twisted.trial import unittest -from ooni.tests import is_internet_connected +from ooni.tests import is_internet_connected, bases from ooni import geoip -class TestGeoIP(unittest.TestCase): +class TestGeoIP(bases.ConfigTestCase): def test_ip_to_location(self): location = geoip.IPToLocation('8.8.8.8') assert 'countrycode' in location diff --git a/ooni/tests/test_nettest.py b/ooni/tests/test_nettest.py index 4f7b3e4..e93111b 100644 --- a/ooni/tests/test_nettest.py +++ b/ooni/tests/test_nettest.py @@ -329,6 +329,7 @@ class TestNettestTimeout(ConfigTestCase): @defer.inlineCallbacks def setUp(self): + super(TestNettestTimeout, self).setUp() from twisted.internet.protocol import Protocol, Factory from twisted.internet.endpoints import TCP4ServerEndpoint diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py index ff241ea..9d93cd4 100644 --- a/ooni/tests/test_oonicli.py +++ b/ooni/tests/test_oonicli.py @@ -60,6 +60,7 @@ tor: class TestRunDirector(ConfigTestCase): def setUp(self): + super(TestRunDirector, self).setUp() if not is_internet_connected(): self.skipTest("You must be connected to the internet to run this test") try:

1 0

[oonib/master] Merge pull request #57 from TheTorProject/feature/13232
by art＠torproject.org 29 Sep '14

29 Sep '14

commit 632e9aa72a78a511a6f4e8d8a618135c880f93dd Merge: 93edf6d 7449409 Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Sep 29 16:13:09 2014 +0200 Merge pull request #57 from TheTorProject/feature/13232 Add support for specifying the format of report filenames and directorie... bin/oonib | 1 + oonib.conf.example | 2 +- oonib/bouncer/handlers.py | 20 ++++++++++++++------ oonib/report/handlers.py | 37 +++++++++++++++++++++---------------- oonib/test/test_report.py | 3 +-- 5 files changed, 38 insertions(+), 25 deletions(-)

1 0

[oonib/master] Add support for specifying the format of report filenames and directories in the archive.
by art＠torproject.org 29 Sep '14

29 Sep '14

commit 33b953312a5244afd6295788304faab822cdbaf6 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Sep 29 13:02:53 2014 +0200 Add support for specifying the format of report filenames and directories in the archive. This implements: https://trac.torproject.org/projects/tor/ticket/13232#comment:1 --- bin/oonib | 1 + oonib.conf.example | 2 +- oonib/report/handlers.py | 37 +++++++++++++++++++++---------------- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/bin/oonib b/bin/oonib index fcb0c72..109c74e 100755 --- a/bin/oonib +++ b/bin/oonib @@ -51,5 +51,6 @@ class OBaseRunner(UnixApplicationRunner): temporary_data_dir = None def createOrGetApplication(self): return application + OBaseRunner.loggerFactory = LoggerFactory OBaseRunner(config.main).run() diff --git a/oonib.conf.example b/oonib.conf.example index 6b07414..64eb5c5 100644 --- a/oonib.conf.example +++ b/oonib.conf.example @@ -29,7 +29,7 @@ main: stale_time: 3600 tor_hidden_service: true - + report_file_template: '{test_name}-{iso8601_timestamp}-{probe_asn}-probe.yamloo' helpers: http-return-json-headers: address: null diff --git a/oonib/report/handlers.py b/oonib/report/handlers.py index 4b88647..13cc06e 100644 --- a/oonib/report/handlers.py +++ b/oonib/report/handlers.py @@ -16,13 +16,25 @@ from oonib import randomStr, otime, log from oonib.config import config -def report_file_name(report_details): - timestamp = otime.timestamp( - datetime.fromtimestamp( - report_details['start_time'])) - dst_filename = '{test_name}-{timestamp}-{probe_asn}-probe.yamloo'.format( - timestamp=timestamp, - **report_details) +def report_file_name(archive_dir, report_details): + timestamp = datetime.fromtimestamp(report_details['start_time']) + keys = dict( + report_details.items(), + iso8601_timestamp=otime.timestamp(timestamp), + year=timestamp.strftime("%Y"), + month=timestamp.strftime("%m"), + day=timestamp.strftime("%d"), + hour=timestamp.strftime("%H"), + minute=timestamp.strftime("%M"), + second=timestamp.strftime("%S") + ) + report_file_template = "{test_name}-{iso8601_timestamp}-{probe_asn}-probe.yamloo" + if config.main.report_file_template: + report_file_template = config.main.report_file_template + dst_filename = os.path.join(archive_dir, report_file_template.format(**keys)) + dst_dir = os.path.dirname(dst_filename) + if not os.path.exists(dst_dir): + os.makedirs(dst_dir) return dst_filename @@ -69,15 +81,8 @@ class Report(object): except IOError: raise e.ReportNotFound - dst_filename = report_file_name(report_details) - dst_path = os.path.join(self.archive_dir, - report_details['probe_cc']) - - if not os.path.isdir(dst_path): - os.mkdir(dst_path) - - dst_path = os.path.join(dst_path, dst_filename) - shutil.move(report_filename, dst_path) + dst_filename = report_file_name(self.archive_dir, report_details) + shutil.move(report_filename, dst_filename) if not self.delayed_call.called: self.delayed_call.cancel()

1 0