January 2017 - tor-commits - lists.torproject.org

[ooni-probe/master] [ci-debug] only deploy with the Twisted environment variable
by art＠torproject.org 13 Jan '17

13 Jan '17

commit 2df130b5367524b27f67216fed72241a5d73a6c9 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Dec 7 00:27:34 2016 +0000 [ci-debug] only deploy with the Twisted environment variable --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index f45a576..3991d21 100644 --- a/.travis.yml +++ b/.travis.yml @@ -58,6 +58,7 @@ deploy: script: 'scripts/deploy.sh $HOME/.ssh/id_rsa_travis' on: branch: master + condition: $TWISTED=Twisted notifications: irc: channels:

1 0

[ooni-probe/master] Install tor from deb.tpo mirror
by art＠torproject.org 13 Jan '17

13 Jan '17

commit 8849deab425618e1289d87dadb13aadbf6e82ec7 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Dec 7 00:14:29 2016 +0000 Install tor from deb.tpo mirror --- .travis.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index ebec628..f45a576 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,9 +25,12 @@ before_install: - mkdir -p $HOME/.ssh/ - mv secrets/id_rsa_travis $HOME/.ssh/ + # Setup deb.tpo repository to download latest tor version + - gpg --keyserver keys.gnupg.net --recv A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89 + - gpg --export A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89 | sudo apt-key add - + - echo 'deb http://deb.torproject.org/torproject.org trusty main' | sudo tee /etc/apt/sources.list.d/tor.list - sudo apt-get update - - sudo apt-get install tor libpcap-dev libgeoip-dev libdumbnet-dev libffi-dev - libssl-dev + - sudo apt-get install -y libpcap-dev libgeoip-dev libdumbnet-dev libffi-dev libssl-dev tor - sudo /etc/init.d/tor start python: - "2.7"

1 0

[ooni-probe/master] [ci-debug] fix typo
by art＠torproject.org 13 Jan '17

13 Jan '17

commit e2de276b0e7cf6326131800427b511206f6fbe49 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Dec 6 19:05:44 2016 +0000 [ci-debug] fix typo --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6ee1585..fef3ae3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,7 +50,7 @@ after_success: - coveralls deploy: provider: script - script: 'script/deploy.sh $HOME/.ssh/id_rsa_travis' + script: 'scripts/deploy.sh $HOME/.ssh/id_rsa_travis' on: branch: master notifications:

1 0

[ooni-probe/master] [hotfix] Only release the lock if it is locked
by art＠torproject.org 13 Jan '17

13 Jan '17

commit 8cc1e691df2d481157e7d023c894eb9623fbe524 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Dec 7 13:15:15 2016 +0000 [hotfix] Only release the lock if it is locked --- ooni/agent/scheduler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py index bda8510..262062e 100644 --- a/ooni/agent/scheduler.py +++ b/ooni/agent/scheduler.py @@ -88,7 +88,8 @@ class ScheduledTask(object): ) def cancel(self): - self._last_run_lock.release() + if self._last_run_lock.locked: + self._last_run_lock.release() @property def should_run(self):

1 0

[ooni-probe/master] Improvements to docstrings in ooni.agent and ooni.ui.web
by art＠torproject.org 13 Jan '17

13 Jan '17

commit d9f3c7aaf1a4f9cc6b11b4508c36fb8b3d56114d Author: Donald <donald213(a)users.noreply.github.com> Date: Sun Nov 27 00:07:38 2016 +0100 Improvements to docstrings in ooni.agent and ooni.ui.web --- ooni/agent/agent.py | 14 +++++++++++++- ooni/agent/scheduler.py | 29 ++++++++++++++++++++++++++++- ooni/ui/web/web.py | 11 ++++++++++- 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/ooni/agent/agent.py b/ooni/agent/agent.py index 01e2162..9c057e6 100644 --- a/ooni/agent/agent.py +++ b/ooni/agent/agent.py @@ -6,9 +6,17 @@ from ooni.ui.web.web import WebUIService from ooni.agent.scheduler import SchedulerService class AgentService(service.MultiService): + """Manage agent services.""" + + def __init__(self, web_ui_port): - service.MultiService.__init__(self) + """Load configuration or made the default configuration of the service. + + If the advanced configuration is not enabled, the page is not displayed. + Else, if the advanced configuration is enabled, the page is displayed.""" + + service.MultiService.__init__(self) director = Director() self.scheduler_service = SchedulerService(director) @@ -22,7 +30,11 @@ class AgentService(service.MultiService): def startService(self): + """Run a service.""" + service.MultiService.startService(self) def stopService(self): + """Kill a service.""" + service.MultiService.stopService(self) diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py index d63a6b1..41c27f1 100644 --- a/ooni/agent/scheduler.py +++ b/ooni/agent/scheduler.py @@ -35,6 +35,8 @@ class FileSystemlockAndMutex(object): different stacks (threads/fibers) within the same process without races. """ def __init__(self, file_path): + """Allows for an Object-oriented filesystem path a representation to be fired when the lock is acquired. + Create a lock for event driven systems.""" self._fs_lock = defer.DeferredFilesystemLock(file_path) self._mutex = defer.DeferredLock() @@ -44,6 +46,7 @@ class FileSystemlockAndMutex(object): yield self._fs_lock.deferUntilLocked() def release(self): + """Release the locks of the filesystem and the event driven system.""" self._fs_lock.unlock() self._mutex.release() @@ -88,16 +91,25 @@ class ScheduledTask(object): ) def cancel(self): + """Closes the task. + + Releases all locks (releases the locks of the filesystem and the event driven system).""" if self._last_run_lock.locked: self._last_run_lock.release() + @property def should_run(self): + """ + + """ current_time = datetime.utcnow().replace(tzinfo=tz.tzutc()) next_cycle = croniter(self.schedule, self.last_run).get_next(datetime) if next_cycle <= current_time: return True return False + #should_run = property(should_run()) + @property def last_run(self): @@ -110,6 +122,9 @@ class ScheduledTask(object): tzinfo=tz.tzutc()) def _update_last_run(self, last_run_time): + """Update time. + + Write the curent time into a file. If the file already exists, the file is overwrited.""" with self._last_run.open('w') as out_file: out_file.write(last_run_time.strftime(self._time_format)) @@ -409,14 +424,23 @@ class SchedulerService(service.MultiService): self.schedule(RunDeck(self.director, deck_id, schedule)) def _task_did_not_run(self, failure, task): + """React in case of task absent. + + TODO and Write debugging log to report this error.""" failure.trap(DidNotRun) log.debug("Did not run {0}".format(task.identifier)) - def _task_failed(self, failure, task): + def _task_failed(self, failure, task):#is this place the right one to raise error as described in the doc? + """React in case of task faillure during the launching. + + Write debugging logs to report the error.""" log.err("Failed to run {0}".format(task.identifier)) log.exception(failure) def _task_success(self, result, task): + """React in case of successfull launching. + + Writes log about debugging.""" log.debug("Ran {0}".format(task.identifier)) def _should_run(self): @@ -453,5 +477,8 @@ class SchedulerService(service.MultiService): self._looping_call.start(self.interval) def stopService(self): + """Stop the Scheduler. + + Stop the Scheduler's service and kill the graphical user interface loop.""" service.MultiService.stopService(self) self._looping_call.stop() diff --git a/ooni/ui/web/web.py b/ooni/ui/web/web.py index f36e0a2..fe9498a 100644 --- a/ooni/ui/web/web.py +++ b/ooni/ui/web/web.py @@ -6,6 +6,8 @@ from ooni.ui.web.server import WebUIAPI from ooni.settings import config class WebUIService(service.MultiService): + """This multiservice contains the ooniprobe web user interface.""" + def __init__(self, director, scheduler, port_number=8842): service.MultiService.__init__(self) @@ -14,6 +16,10 @@ class WebUIService(service.MultiService): self.port_number = port_number def startService(self): + """Start a web user interface. + + Connects a given protocol factory to the given numeric TCP/IP port and open a page.""" + service.MultiService.startService(self) web_ui_api = WebUIAPI(config, self.director, self.scheduler) @@ -21,9 +27,12 @@ class WebUIService(service.MultiService): self.port_number, server.Site(web_ui_api.app.resource()), interface=config.advanced.webui_address - ) + ) def stopService(self): + """Close the web page. + + Close the service, verify that the connection is really finished. If the program is still listenning, the program kill the connection.""" service.MultiService.stopService(self) if self._port: self._port.stopListening()

1 0

[ooni-probe/master] Use details tags to tuck verbose messages (#711)
by art＠torproject.org 13 Jan '17

13 Jan '17

commit 47799e70c9b520a98f91e9c53771901e555f44fa Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Jan 12 11:04:51 2017 +0000 Use details tags to tuck verbose messages (#711) * Use details tags to tuck verbose messages * Add missing question mark --- .github/ISSUE_TEMPLATE.md | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 3123450..a619e65 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -8,18 +8,36 @@ 2. [Second Step] 3. [and so on...] +<details><summary>Backtrace of the error</summary> + +``` +[paste output here] +``` +</details> + **Expected behavior:** [What you expected to happen] **Actual behavior:** [What actually happened] -### Versions +### Further information **How did you install ooniprobe** -Did you get it via pip, homebrew or debian packages? +Did you install ooniprobe via the **debian package**, **homebrew** or **pip**? -**What version of ooniprobe are you running?** +**What operating system are you running and what version is it?** -You can get this information by running `ooniprobe --version` +<details><summary>Output of `ooniprobe -v --info`</summary> + +``` +[paste output here] +``` +</details> + +<details><summary>Output of `ooniprobe --version`</summary> + +``` +[paste output here] +``` +</details> -**What operating system are you running and what version is it?**

1 0

[ooni-probe/master] Added a space when printing the -g opt (#710)
by art＠torproject.org 13 Jan '17

13 Jan '17

commit 72c66005313ac954ec019d872f8fb012309920af Author: E.Iosifidis <iosifidise(a)gmail.com> Date: Wed Jan 11 19:47:18 2017 +0200 Added a space when printing the -g opt (#710) --- ooni/ui/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py index adaef30..162354c 100644 --- a/ooni/ui/cli.py +++ b/ooni/ui/cli.py @@ -26,7 +26,7 @@ class Options(usage.Options): optFlags = [["help", "h"], ["no-collector", "n", "Disable writing to collector"], ["no-njson", "N", "Disable writing to disk"], - ["no-geoip", "g", "Disable geoip lookup on start." + ["no-geoip", "g", "Disable geoip lookup on start. " "With this option your IP address can be disclosed in the report."], ["list", "s", "List the currently installed ooniprobe " "nettests"],

1 0

[ooni-probe/master] Cleanup of docs from @donald213
by art＠torproject.org 13 Jan '17

13 Jan '17

commit 22720e383093c0d30fd1b6ca44794d3b29031f91 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Dec 14 13:11:42 2016 +0000 Cleanup of docs from @donald213 --- ooni/agent/agent.py | 15 ++++----------- ooni/agent/scheduler.py | 49 +++++++++++++++++++++++-------------------------- ooni/ui/web/web.py | 7 ------- 3 files changed, 27 insertions(+), 44 deletions(-) diff --git a/ooni/agent/agent.py b/ooni/agent/agent.py index 9c057e6..20b9332 100644 --- a/ooni/agent/agent.py +++ b/ooni/agent/agent.py @@ -6,16 +6,13 @@ from ooni.ui.web.web import WebUIService from ooni.agent.scheduler import SchedulerService class AgentService(service.MultiService): - """Manage agent services.""" + """Manage all services related to the ooniprobe-agent daemon.""" def __init__(self, web_ui_port): - """Load configuration or made the default configuration of the service. - - - If the advanced configuration is not enabled, the page is not displayed. - Else, if the advanced configuration is enabled, the page is displayed.""" - + """ + If the advanced->disabled_webui is set to true then we will not start the WebUI. + """ service.MultiService.__init__(self) director = Director() @@ -30,11 +27,7 @@ class AgentService(service.MultiService): def startService(self): - """Run a service.""" - service.MultiService.startService(self) def stopService(self): - """Kill a service.""" - service.MultiService.stopService(self) diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py index 41c27f1..4bf058a 100644 --- a/ooni/agent/scheduler.py +++ b/ooni/agent/scheduler.py @@ -35,8 +35,10 @@ class FileSystemlockAndMutex(object): different stacks (threads/fibers) within the same process without races. """ def __init__(self, file_path): - """Allows for an Object-oriented filesystem path a representation to be fired when the lock is acquired. - Create a lock for event driven systems.""" + """ + Args: + file_path: is the location of where the filesystem based lockfile should be written to. + """ self._fs_lock = defer.DeferredFilesystemLock(file_path) self._mutex = defer.DeferredLock() @@ -46,7 +48,7 @@ class FileSystemlockAndMutex(object): yield self._fs_lock.deferUntilLocked() def release(self): - """Release the locks of the filesystem and the event driven system.""" + """Release the filesystem based and in memory locks.""" self._fs_lock.unlock() self._mutex.release() @@ -91,24 +93,21 @@ class ScheduledTask(object): ) def cancel(self): - """Closes the task. - - Releases all locks (releases the locks of the filesystem and the event driven system).""" + """ + Cancel a currently running task. + If it is locked, then release the lock. + """ if self._last_run_lock.locked: self._last_run_lock.release() @property def should_run(self): - """ - - """ current_time = datetime.utcnow().replace(tzinfo=tz.tzutc()) next_cycle = croniter(self.schedule, self.last_run).get_next(datetime) if next_cycle <= current_time: return True return False - #should_run = property(should_run()) @property @@ -122,9 +121,10 @@ class ScheduledTask(object): tzinfo=tz.tzutc()) def _update_last_run(self, last_run_time): - """Update time. - - Write the curent time into a file. If the file already exists, the file is overwrited.""" + """ + Update the time at which this task ran successfully last, by running + to a file. + """ with self._last_run.open('w') as out_file: out_file.write(last_run_time.strftime(self._time_format)) @@ -424,23 +424,23 @@ class SchedulerService(service.MultiService): self.schedule(RunDeck(self.director, deck_id, schedule)) def _task_did_not_run(self, failure, task): - """React in case of task absent. - - TODO and Write debugging log to report this error.""" + """ + Fired when a tasks did not run. This is not an error. + """ failure.trap(DidNotRun) log.debug("Did not run {0}".format(task.identifier)) - def _task_failed(self, failure, task):#is this place the right one to raise error as described in the doc? - """React in case of task faillure during the launching. - - Write debugging logs to report the error.""" + def _task_failed(self, failure, task): + """ + Fired when a task failed to run due to an error. + """ log.err("Failed to run {0}".format(task.identifier)) log.exception(failure) def _task_success(self, result, task): - """React in case of successfull launching. - - Writes log about debugging.""" + """ + Fired when a task has successfully run. + """ log.debug("Ran {0}".format(task.identifier)) def _should_run(self): @@ -477,8 +477,5 @@ class SchedulerService(service.MultiService): self._looping_call.start(self.interval) def stopService(self): - """Stop the Scheduler. - - Stop the Scheduler's service and kill the graphical user interface loop.""" service.MultiService.stopService(self) self._looping_call.stop() diff --git a/ooni/ui/web/web.py b/ooni/ui/web/web.py index fe9498a..081022a 100644 --- a/ooni/ui/web/web.py +++ b/ooni/ui/web/web.py @@ -16,10 +16,6 @@ class WebUIService(service.MultiService): self.port_number = port_number def startService(self): - """Start a web user interface. - - Connects a given protocol factory to the given numeric TCP/IP port and open a page.""" - service.MultiService.startService(self) web_ui_api = WebUIAPI(config, self.director, self.scheduler) @@ -30,9 +26,6 @@ class WebUIService(service.MultiService): ) def stopService(self): - """Close the web page. - - Close the service, verify that the connection is really finished. If the program is still listenning, the program kill the connection.""" service.MultiService.stopService(self) if self._port: self._port.stopListening()

1 0

[ooni-probe/master] Option -g help updates in order to warn user about IP disclosure in report.
by art＠torproject.org 13 Jan '17

13 Jan '17

commit fdd215907d0c98460daaf88f8ecf5050818624ea Author: ioef <iosifidise(a)gmail.com> Date: Wed Jan 11 14:40:49 2017 +0200 Option -g help updates in order to warn user about IP disclosure in report. --- data/ooniprobe.1 | 1 + docs/source/manual/ooniprobe.rst | 3 ++- ooni/ui/cli.py | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/data/ooniprobe.1 b/data/ooniprobe.1 index 52ec4a8..cee8817 100644 --- a/data/ooniprobe.1 +++ b/data/ooniprobe.1 @@ -52,6 +52,7 @@ Disable writing to disk .TP .B \-g\fP,\fB \-\-no\-geoip Disable geoip lookup on start +Warning: By using this option the IP address of the user executing ooniprobe is not excluded from the report. .TP .B \-s\fP,\fB \-\-list List the currently installed ooniprobe nettests diff --git a/docs/source/manual/ooniprobe.rst b/docs/source/manual/ooniprobe.rst index 7d71795..928b99e 100644 --- a/docs/source/manual/ooniprobe.rst +++ b/docs/source/manual/ooniprobe.rst @@ -26,7 +26,8 @@ Options Disable writing to disk -g, --no-geoip - Disable geoip lookup on start + Disable geoip lookup on start. + Warning: By using this option the IP address of the user executing ooniprobe is not excluded from the report. -s, --list List the currently installed ooniprobe nettests diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py index c545d82..adaef30 100644 --- a/ooni/ui/cli.py +++ b/ooni/ui/cli.py @@ -26,7 +26,8 @@ class Options(usage.Options): optFlags = [["help", "h"], ["no-collector", "n", "Disable writing to collector"], ["no-njson", "N", "Disable writing to disk"], - ["no-geoip", "g", "Disable geoip lookup on start"], + ["no-geoip", "g", "Disable geoip lookup on start." + "With this option your IP address can be disclosed in the report."], ["list", "s", "List the currently installed ooniprobe " "nettests"], ["verbose", "v", "Show more verbose information"],

1 0

[ooni-probe/master] Add support for specifying a settings.ini file via environment variables
by art＠torproject.org 13 Jan '17

13 Jan '17

commit ee68602abac9c5f66ba81c4df95822a3ccd87bda Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Oct 26 17:12:03 2016 +0200 Add support for specifying a settings.ini file via environment variables When doing end to end testing for the homebrew package it's required to have the ability to overwrite the paths used by ooniprobe. --- ooni/settings.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ooni/settings.py b/ooni/settings.py index ef220ca..48dd1f3 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -156,6 +156,9 @@ if hasattr(sys, 'real_prefix'): # These are the the embedded settings _SETTINGS_INI = os.path.join(OONIPROBE_ROOT, 'settings.ini') +# This can be overridden by an environment variable for end to end testing +if 'OONIPROBE_SETTINGS' in os.environ: + _SETTINGS_INI = os.environ['OONIPROBE_SETTINGS'] USR_SHARE_PATH = '/usr/share/ooni' VAR_LIB_PATH = '/var/lib/ooni'

1 0