June 2016 - tor-commits - lists.torproject.org

[tor/maint-0.2.8] Remove useless message about nonexistent onion services after uploading a descriptor
by nickm＠torproject.org 21 Jun '16

21 Jun '16

commit b432efb8381360fb10179c67156b8ed5a05e290b Author: Ivan Markin <twim(a)riseup.net> Date: Mon Jun 20 19:29:27 2016 +0000 Remove useless message about nonexistent onion services after uploading a descriptor --- changes/bug19464 | 6 ++++++ src/or/rendservice.c | 2 -- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/changes/bug19464 b/changes/bug19464 new file mode 100644 index 0000000..22c9e73 --- /dev/null +++ b/changes/bug19464 @@ -0,0 +1,6 @@ + o Minor bugfixes (user interface): + - Remove a warning message "Service [scrubbed] not found after + descriptor upload". This message appears when one uses HSPOST control + command to upload a service descriptor. Since there is only a descriptor + and no service, showing this message is pointless and confusing. + Fixes bug 19464; bugfix on 0.2.7.2-alpha. diff --git a/src/or/rendservice.c b/src/or/rendservice.c index 6f41f3b..b81a01c 100644 --- a/src/or/rendservice.c +++ b/src/or/rendservice.c @@ -3465,8 +3465,6 @@ rend_service_desc_has_uploaded(const rend_data_t *rend_data) service = rend_service_get_by_service_id(rend_data->onion_address); if (service == NULL) { - log_warn(LD_REND, "Service %s not found after descriptor upload", - safe_str_client(rend_data->onion_address)); return; }

1 0

[tor/release-0.2.8] Merge branch 'maint-0.2.8' into release-0.2.8
by nickm＠torproject.org 21 Jun '16

21 Jun '16

commit a839ed5cc1e12aa096191909e67c715f0863a20a Merge: b673f7e f038e9c Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Jun 21 08:54:39 2016 -0400 Merge branch 'maint-0.2.8' into release-0.2.8 changes/bug19454 | 3 +++ src/or/main.c | 12 +++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-)

1 0

[tor/master] Merge branch 'maint-0.2.8'
by nickm＠torproject.org 21 Jun '16

21 Jun '16

commit dd9cebf1099996d3361110fddfc2cf47beb94c5d Merge: 49e8f47 f038e9c Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Jun 21 08:54:49 2016 -0400 Merge branch 'maint-0.2.8' changes/bug19454 | 3 +++ src/or/main.c | 12 +++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-)

1 0

[tor/release-0.2.8] Fix bug when disabling heartbeats.
by nickm＠torproject.org 21 Jun '16

21 Jun '16

commit f038e9cb00b9324b24a100d7cd3576410a7ae7f3 Author: George Kadianakis <desnacked(a)riseup.net> Date: Tue Jun 21 15:05:59 2016 +0300 Fix bug when disabling heartbeats. Callbacks can't return 0. --- changes/bug19454 | 3 +++ src/or/main.c | 12 +++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/changes/bug19454 b/changes/bug19454 new file mode 100644 index 0000000..05650b5 --- /dev/null +++ b/changes/bug19454 @@ -0,0 +1,3 @@ + o Minor bugfixes (heartbeat): + - Fix regression that crashes Tor when disabling heartbeats. Fixes bug + 19454; bugfix on tor-0.2.8.1-alpha. Reported by "kubaku". diff --git a/src/or/main.c b/src/or/main.c index b9fee1d..f585f0b 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -2004,18 +2004,24 @@ check_fw_helper_app_callback(time_t now, const or_options_t *options) return PORT_FORWARDING_CHECK_INTERVAL; } +/** Callback to write heartbeat message in the logs. */ static int heartbeat_callback(time_t now, const or_options_t *options) { static int first = 1; - /* 12. write the heartbeat message */ + + /* Check if heartbeat is disabled */ + if (!options->HeartbeatPeriod) { + return PERIODIC_EVENT_NO_UPDATE; + } + + /* Write the heartbeat message */ if (first) { first = 0; /* Skip the first one. */ } else { log_heartbeat(now); } - /* XXXX This isn't such a good way to handle possible changes in the - * callback event */ + return options->HeartbeatPeriod; }

1 0

[tor/maint-0.2.8] Fix bug when disabling heartbeats.
by nickm＠torproject.org 21 Jun '16

21 Jun '16

commit f038e9cb00b9324b24a100d7cd3576410a7ae7f3 Author: George Kadianakis <desnacked(a)riseup.net> Date: Tue Jun 21 15:05:59 2016 +0300 Fix bug when disabling heartbeats. Callbacks can't return 0. --- changes/bug19454 | 3 +++ src/or/main.c | 12 +++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/changes/bug19454 b/changes/bug19454 new file mode 100644 index 0000000..05650b5 --- /dev/null +++ b/changes/bug19454 @@ -0,0 +1,3 @@ + o Minor bugfixes (heartbeat): + - Fix regression that crashes Tor when disabling heartbeats. Fixes bug + 19454; bugfix on tor-0.2.8.1-alpha. Reported by "kubaku". diff --git a/src/or/main.c b/src/or/main.c index b9fee1d..f585f0b 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -2004,18 +2004,24 @@ check_fw_helper_app_callback(time_t now, const or_options_t *options) return PORT_FORWARDING_CHECK_INTERVAL; } +/** Callback to write heartbeat message in the logs. */ static int heartbeat_callback(time_t now, const or_options_t *options) { static int first = 1; - /* 12. write the heartbeat message */ + + /* Check if heartbeat is disabled */ + if (!options->HeartbeatPeriod) { + return PERIODIC_EVENT_NO_UPDATE; + } + + /* Write the heartbeat message */ if (first) { first = 0; /* Skip the first one. */ } else { log_heartbeat(now); } - /* XXXX This isn't such a good way to handle possible changes in the - * callback event */ + return options->HeartbeatPeriod; }

1 0

[tor/master] Fix bug when disabling heartbeats.
by nickm＠torproject.org 21 Jun '16

21 Jun '16

commit f038e9cb00b9324b24a100d7cd3576410a7ae7f3 Author: George Kadianakis <desnacked(a)riseup.net> Date: Tue Jun 21 15:05:59 2016 +0300 Fix bug when disabling heartbeats. Callbacks can't return 0. --- changes/bug19454 | 3 +++ src/or/main.c | 12 +++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/changes/bug19454 b/changes/bug19454 new file mode 100644 index 0000000..05650b5 --- /dev/null +++ b/changes/bug19454 @@ -0,0 +1,3 @@ + o Minor bugfixes (heartbeat): + - Fix regression that crashes Tor when disabling heartbeats. Fixes bug + 19454; bugfix on tor-0.2.8.1-alpha. Reported by "kubaku". diff --git a/src/or/main.c b/src/or/main.c index b9fee1d..f585f0b 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -2004,18 +2004,24 @@ check_fw_helper_app_callback(time_t now, const or_options_t *options) return PORT_FORWARDING_CHECK_INTERVAL; } +/** Callback to write heartbeat message in the logs. */ static int heartbeat_callback(time_t now, const or_options_t *options) { static int first = 1; - /* 12. write the heartbeat message */ + + /* Check if heartbeat is disabled */ + if (!options->HeartbeatPeriod) { + return PERIODIC_EVENT_NO_UPDATE; + } + + /* Write the heartbeat message */ if (first) { first = 0; /* Skip the first one. */ } else { log_heartbeat(now); } - /* XXXX This isn't such a good way to handle possible changes in the - * callback event */ + return options->HeartbeatPeriod; }

1 0

[ooni-probe/master] Generators do not work well in a concurrent environment.
by art＠torproject.org 21 Jun '16

21 Jun '16

commit 007685b2fbf8ec5e51ed8004af832d0299f4429e Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Jun 15 12:43:23 2016 +0300 Generators do not work well in a concurrent environment. Use a mutex lock on the tasks generator to avoid concurrency issues. --- ooni/managers.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/ooni/managers.py b/ooni/managers.py index 65e4ce1..f139f20 100644 --- a/ooni/managers.py +++ b/ooni/managers.py @@ -1,4 +1,5 @@ import itertools +from twisted.internet import defer from ooni.utils import log from ooni.settings import config @@ -24,6 +25,7 @@ class TaskManager(object): self._tasks = iter(()) self._active_tasks = [] self.failures = 0 + self.task_lock = defer.DeferredLock() def _failed(self, failure, task): """ @@ -55,17 +57,17 @@ class TaskManager(object): Called on test completion and schedules measurements to be run for the available slots. """ + d = self.task_lock.acquire() + d.addCallback(lambda _: self._scheduleNextTask()) + + def _scheduleNextTask(self): for _ in range(self.availableSlots): try: - task = self._tasks.next() + task = next(self._tasks) self._run(task) except StopIteration: break - except ValueError: - # XXX this is a workaround the race condition that leads the - # _tasks generator to throw the exception - # ValueError: generator already called. - continue + self.task_lock.release() def _run(self, task): """

1 0

[ooni-probe/master] Merge branch 'feature/bisection_503'
by art＠torproject.org 21 Jun '16

21 Jun '16

commit 25b317ab8c015940d034fd2682f4da6aef6e70a3 Merge: 7d0cd24 007685b Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Jun 15 12:44:19 2016 +0300 Merge branch 'feature/bisection_503' * feature/bisection_503: Generators do not work well in a concurrent environment. Make inputs a shared iterator across all NetTestCase measurements docs/source/writing_tests.rst | 61 ++++++++++++++++++++++- ooni/managers.py | 14 +++--- ooni/nettest.py | 10 ++-- ooni/tests/test_nettest.py | 109 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 182 insertions(+), 12 deletions(-)

1 0

[ooni-probe/master] Add basic unittests for oonireport command (#545)
by art＠torproject.org 21 Jun '16

21 Jun '16

commit 813a9fcb58fac057b46c1eadebac426f3346fa61 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Jun 17 10:03:28 2016 +0300 Add basic unittests for oonireport command (#545) --- ooni/report/cli.py | 4 +- ooni/tests/test_oonireport.py | 107 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 2 deletions(-) diff --git a/ooni/report/cli.py b/ooni/report/cli.py index f730027..075178c 100644 --- a/ooni/report/cli.py +++ b/ooni/report/cli.py @@ -60,10 +60,10 @@ def tor_check(): sys.exit(1) -def run(): +def run(args=sys.argv): options = Options() try: - options.parseOptions() + options.parseOptions(args) except Exception as exc: print("Error: %s" % exc) print(options) diff --git a/ooni/tests/test_oonireport.py b/ooni/tests/test_oonireport.py new file mode 100644 index 0000000..2275672 --- /dev/null +++ b/ooni/tests/test_oonireport.py @@ -0,0 +1,107 @@ +import yaml + +from mock import patch, MagicMock + +from twisted.internet import defer +from ooni.tests.bases import ConfigTestCase + +from ooni.report import tool + +mock_tor_check = MagicMock(return_value=True) + +class TestOONIReport(ConfigTestCase): + + def _create_reporting_yaml(self, filename): + from ooni.settings import config + with open(config.report_log_file, 'w+') as f: + yaml.dump({ + filename: { + "collector": "httpo://thirteenchars123.onion" + } + }, f) + + def _write_dummy_report(self, filename): + from ooni.reporter import YAMLReporter + from .test_reporter import test_details + reporter = YAMLReporter(test_details, filename) + reporter.createReport() + reporter.writeReportEntry({"spam": "ham"}) + reporter.finish() + + def test_cli_status(self): + mock_tool = MagicMock() + with patch('ooni.report.cli.tool', mock_tool): + from ooni.report import cli + cli.run(["status"]) + self.assertTrue(mock_tool.status.called) + + @patch('ooni.report.cli.tor_check', mock_tor_check) + def test_cli_upload(self): + mock_tool = MagicMock() + with patch('ooni.report.cli.tool', mock_tool): + from ooni.report import cli + cli.run(["upload", "dummy.yaml"]) + self.assertTrue(mock_tool.upload.called) + + @patch('ooni.report.cli.tor_check', mock_tor_check) + def test_cli_upload_all(self): + mock_tool = MagicMock() + with patch('ooni.report.cli.tool', mock_tool): + from ooni.report import cli + cli.run(["upload"]) + self.assertTrue(mock_tool.upload_all.called) + + @patch('ooni.report.tool.CollectorClient') + @patch('ooni.report.tool.OONIBReportLog') + @patch('ooni.report.tool.OONIBReporter') + def test_tool_upload(self, mock_oonib_reporter, mock_oonib_report_log, + mock_collector_client): + + mock_oonib_reporter_i = mock_oonib_reporter.return_value + mock_oonib_reporter_i.createReport.return_value = defer.succeed("fake_id") + mock_oonib_reporter_i.writeReportEntry.return_value = defer.succeed(True) + mock_oonib_reporter_i.finish.return_value = defer.succeed(True) + + mock_oonib_report_log_i = mock_oonib_report_log.return_value + mock_oonib_report_log_i.created.return_value = defer.succeed(True) + mock_oonib_report_log_i.closed.return_value = defer.succeed(True) + + report_name = "dummy_report.yaml" + self._create_reporting_yaml(report_name) + self._write_dummy_report(report_name) + + d = tool.upload(report_name) + @d.addCallback + def cb(result): + mock_oonib_reporter_i.writeReportEntry.assert_called_with( + {"spam": "ham"} + ) + return d + + @patch('ooni.report.tool.CollectorClient') + @patch('ooni.report.tool.OONIBReportLog') + @patch('ooni.report.tool.OONIBReporter') + def test_tool_upload_all(self, mock_oonib_reporter, mock_oonib_report_log, + mock_collector_client): + + mock_oonib_reporter_i = mock_oonib_reporter.return_value + mock_oonib_reporter_i.createReport.return_value = defer.succeed("fake_id") + mock_oonib_reporter_i.writeReportEntry.return_value = defer.succeed(True) + mock_oonib_reporter_i.finish.return_value = defer.succeed(True) + + mock_oonib_report_log_i = mock_oonib_report_log.return_value + mock_oonib_report_log_i.created.return_value = defer.succeed(True) + mock_oonib_report_log_i.closed.return_value = defer.succeed(True) + mock_oonib_report_log_i.reports_to_upload = [("dummy_report.yaml", None)] + + report_name = "dummy_report.yaml" + self._create_reporting_yaml(report_name) + self._write_dummy_report(report_name) + + d = tool.upload_all() + @d.addCallback + def cb(result): + mock_oonib_reporter_i.writeReportEntry.assert_called_with( + {"spam": "ham"} + ) + return d

1 0

[ooni-probe/master] Make inputs a shared iterator across all NetTestCase measurements
by art＠torproject.org 21 Jun '16

21 Jun '16

commit b391f1dc286d318178d0c0b622748b7b604d45d0 Author: seamus tuohy <code(a)seamustuohy.com> Date: Sun Jun 5 13:01:24 2016 -0400 Make inputs a shared iterator across all NetTestCase measurements This commit makes the inputs iterator shared by all the measurement tests within a NetTestCase. Per issue 503, In order to make bisection style testing functional the individual tests need to be able to pass data to the main test_case.inputs generator that seeds each measurement. This provides opportunities for each measurement test in a NetTestCase to communicate with the iterator to do things such as adding additional values to be passed to measurement tests as a later input. --- docs/source/writing_tests.rst | 61 ++++++++++++++++++++++- ooni/nettest.py | 10 ++-- ooni/tests/test_nettest.py | 109 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 174 insertions(+), 6 deletions(-) diff --git a/docs/source/writing_tests.rst b/docs/source/writing_tests.rst index 2132010..aaa4adf 100644 --- a/docs/source/writing_tests.rst +++ b/docs/source/writing_tests.rst @@ -57,14 +57,71 @@ this:: yield x.strip() fp.close() -For example, if you wanted to modify inputProcessor to read enteries from a CSV file, you could use:: - +For example, if you wanted to modify inputProcessor to read entries from a CSV file, you could use:: + def inputProcessor(self, filename): with open(filename) as csvFile: reader = DictReader(csvFile) for entry in reader: yield entry + +The ``inputs`` iterator is unique per ``NetTestCase`` and shared by all its measurement tests. This provides opportunities for each measurement test in a ``NetTestCase`` to communicate with the iterator to do things such as adding additional values to be passed to measurement tests as a later ``input``. + +.. note :: Deleting/removing the current item from the ``inputs`` iterator will not stop other measurement tests from operating on that ``input``. When a ``NetTestCase`` is run a single ``input`` is taken from the ``inputs`` iterator by the OONI test loader and run against all of the individual measurement tests within that ``NetTestCase``. Removing an ``input`` from the ``inputs`` iterator during a measurement will not stop that input from being called on all other measurement tests within the NetTestCase. + +Here is one example of how you can take advantage of shared ``inputs`` iterator when writing your own OONI tests. If you have a list of urls and you want to make sure that you always test the HTTP equivalent of any HTTPS urls provided you can ``send`` values back to a custom generator in your ``postProcessor``. + +To do this the first thing you would need to create is a URL generator that can accept values sent to it. + +:: + class UrlGeneratorWithSend(object): + def __init__(self): + """Create initial list and set generator state.""" + self.urls = ["http://www.torproject.org", + "https://ooni.torproject.org"] + self.current = 0 + + def __iter__(self): + return self + + def __next__(self): + try: + cur = self.urls[self.current] + self.current += 1 + return cur + except IndexError: + raise StopIteration + + # Python 2 & 3 generator compatibility + next = __next__ + + def send(self, returned): + """Appends a value to self.urls when activated""" + if returned is not None: + print("Value {0} sent to generator".format(returned)) + self.urls.append(returned) + +With this generator created you can now assign it as the ``inputs`` to a ``NetTestCase`` and ``send`` values back to it. + +:: + class TestUrlList(nettest.NetTestCase): + + # Adding custom generator here + inputs = UrlGeneratorWithSend() + + def postProcessor(self, measurements): + """If any HTTPS url's are passed send back an HTTP url.""" + if re.match("^https", self.input): + http_version = re.sub("https", "http", self.input, 1) + self.inputs.send(http_version) + return self.report + + def test_url(self): + self.report['tested'] = [self.input] + return defer.succeed(1) + + Setup and command line passing ------------------------------ diff --git a/ooni/nettest.py b/ooni/nettest.py index 8b9556e..35fc2ff 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -585,10 +585,13 @@ class NetTest(object): """ for test_class, test_methods in self.testCases: - # load the input processor as late as possible - for input in test_class.inputs: + # load a singular input processor for all instances + all_inputs = test_class.inputs + for test_input in all_inputs: measurements = [] test_instance = test_class() + # Set each instances inputs to a singular input processor + test_instance.inputs = all_inputs test_instance._setUp() test_instance.summary = self.summary for method in test_methods: @@ -596,7 +599,7 @@ class NetTest(object): measurement = self.makeMeasurement( test_instance, method, - input) + test_input) measurements.append(measurement.done) self.state.taskCreated() yield measurement @@ -721,7 +724,6 @@ class NetTestCase(object): It gets called once for every input. """ self.report = {} - self.inputs = None def requirements(self): """ diff --git a/ooni/tests/test_nettest.py b/ooni/tests/test_nettest.py index da4969f..55134d3 100644 --- a/ooni/tests/test_nettest.py +++ b/ooni/tests/test_nettest.py @@ -1,4 +1,5 @@ import os +import yaml from tempfile import mkstemp from twisted.trial import unittest @@ -147,6 +148,65 @@ class HTTPBasedTest(httpt.HTTPTest): use_tor=False) """ +generator_net_test = """ +from twisted.python import usage +from ooni.nettest import NetTestCase + +class UsageOptions(usage.Options): + optParameters = [['spam', 's', None, 'ham']] + +def input_generator(): + # Generates a list of numbers + # The first value sent back is appended to the list. + received = False + numbers = [i for i in range(10)] + while numbers: + i = numbers.pop() + result = yield i + # Place sent value back in numbers + if result is not None and received is False: + numbers.append(result) + received = True + yield i + +class TestSendGen(NetTestCase): + usageOptions = UsageOptions + inputs = input_generator() + + def test_input_sent_to_generator(self): + # Sends a single value back to the generator + if self.input == 5: + self.inputs.send(self.input) +""" + +generator_id_net_test = """ +from twisted.python import usage +from ooni.nettest import NetTestCase + +class UsageOptions(usage.Options): + optParameters = [['spam', 's', None, 'ham']] + +class DummyTestCaseA(NetTestCase): + + usageOptions = UsageOptions + + def test_a(self): + self.report.setdefault("results", []).append(id(self.inputs)) + + def test_b(self): + self.report.setdefault("results", []).append(id(self.inputs)) + + def test_c(self): + self.report.setdefault("results", []).append(id(self.inputs)) + +class DummyTestCaseB(NetTestCase): + + usageOptions = UsageOptions + + def test_a(self): + self.report.setdefault("results", []).append(id(self.inputs)) +""" + dummyInputs = range(1) dummyArgs = ('--spam', 'notham') dummyOptions = {'spam': 'notham'} @@ -308,6 +368,55 @@ class TestNetTest(unittest.TestCase): for test_class, methods in ntl.getTestCases(): self.assertTrue(test_class.requiresRoot) + def test_singular_input_processor(self): + """ + Verify that all measurements use the same object as their input processor. + """ + ntl = NetTestLoader(dummyArgs) + ntl.loadNetTestString(generator_id_net_test) + ntl.checkOptions() + + director = Director() + self.filename = 'dummy_report.yamloo' + d = director.startNetTest(ntl, self.filename) + + @d.addCallback + def complete(result): + with open(self.filename) as report_file: + all_report_entries = yaml.safe_load_all(report_file) + header = all_report_entries.next() + results_case_a = all_report_entries.next() + aa_test, ab_test, ac_test = results_case_a.get('results', []) + results_case_b = all_report_entries.next() + ba_test = results_case_b.get('results', [])[0] + # Within a NetTestCase an inputs object will be consistent + self.assertEqual(aa_test, ab_test, ac_test) + # An inputs object will be different between different NetTestCases + self.assertNotEqual(aa_test, ba_test) + + return d + + def test_send_to_inputs_generator(self): + """ + Verify that a net test can send information back into an inputs generator. + """ + ntl = NetTestLoader(dummyArgs) + ntl.loadNetTestString(generator_net_test) + ntl.checkOptions() + + director = Director() + self.filename = 'dummy_report.yamloo' + d = director.startNetTest(ntl, self.filename) + + @d.addCallback + def complete(result): + with open(self.filename) as report_file: + all_report_entries = yaml.safe_load_all(report_file) + header = all_report_entries.next() + results = [x['input'] for x in all_report_entries] + self.assertEqual(results, [9, 8, 7, 6, 5, 5, 3, 2, 1, 0]) + + return d class TestNettestTimeout(ConfigTestCase):

1 0