April 2014 - tor-commits - lists.torproject.org

[ooni-probe/master] Resolved differences between paths (log, config, home) pointing to different users' home folders
by art＠torproject.org 30 Apr '14

30 Apr '14

commit 08d61704cb74f876929b5b42ebcdbf1471d1aadd Author: olafmk <mail(a)olaf-koehler.com> Date: Sat Apr 12 13:31:18 2014 +0200 Resolved differences between paths (log,config,home) pointing to different users' home folders (`logfile` and `config_file` were pointing to 'user/.ooni/...' and `ooni_home` to 'root/.ooni/...') --- ooni/settings.py | 9 +++++---- ooni/utils/log.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ooni/settings.py b/ooni/… [View More]

1 0

[ooni-probe/master] Refactoring of geoip related functionality to have it less coupled
by art＠torproject.org 30 Apr '14

30 Apr '14

commit d46b8791d3ba1e6520a725e94e8ed51763841334 Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Mar 8 13:47:25 2014 +0100 Refactoring of geoip related functionality to have it less coupled --- ooni/director.py | 3 +-- ooni/geoip.py | 41 +++++++++++++++++++++++++++++++++++------ ooni/nettest.py | 38 ++++---------------------------------- ooni/settings.py | 6 +++--- ooni/utils/log.py | 6 +++++- 5 files changed, 48 insertions(+), 46 deletions(-) diff --… [View More]git a/ooni/director.py b/ooni/director.py index e4fc261..27556d6 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -127,7 +127,6 @@ class Director(object): log.msg("Connecting to Tor Control Port...") yield self.getTorState() - config.probe_ip = geoip.ProbeIP() yield config.probe_ip.lookup() @property @@ -222,7 +221,7 @@ class Director(object): """ if config.privacy.includepcap: if not config.reports.pcap: - config.reports.pcap = config.generatePcapFilename(net_test_loader.testDetails) + config.reports.pcap = config.generate_pcap_filename(net_test_loader.testDetails) self.startSniffing() report = Report(reporters, self.reportEntryManager) diff --git a/ooni/geoip.py b/ooni/geoip.py index 2792126..53a1f70 100644 --- a/ooni/geoip.py +++ b/ooni/geoip.py @@ -5,11 +5,9 @@ import random from twisted.web import client, http_headers client._HTTP11ClientFactory.noisy = False -from ooni.utils.net import userAgents, BodyReceiver from twisted.internet import reactor, defer, protocol from ooni.utils import log, net, checkForRoot -from ooni.settings import config from ooni import errors try: @@ -26,6 +24,8 @@ class GeoIPDataFilesNotFound(Exception): pass def IPToLocation(ipaddr): + from ooni.settings import config + city_file = os.path.join(config.advanced.geoip_data_dir, 'GeoLiteCity.dat') country_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIP.dat') asn_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIPASNum.dat') @@ -67,6 +67,8 @@ class HTTPGeoIPLookupper(object): self.agent = self._agent(reactor) def _response(self, response): + from ooni.utils.net import BodyReceiver + content_length = response.headers.getRawHeaders('content-length') finished = defer.Deferred() @@ -88,6 +90,8 @@ class HTTPGeoIPLookupper(object): return failure def lookup(self): + from ooni.utils.net import userAgents + headers = {} headers['User-Agent'] = [random.choice(userAgents)] @@ -117,16 +121,37 @@ class ProbeIP(object): address = None def __init__(self): - self.tor_state = config.tor_state - self.geoIPServices = {'ubuntu': UbuntuGeoIP, + self.geoIPServices = { + 'ubuntu': UbuntuGeoIP, 'torproject': TorProjectGeoIP } + self.geodata = { + 'asn': 'AS0', + 'city': None, + 'countrycode': 'ZZ', + 'ip': '127.0.0.1' + } + + def resolveGeodata(self): + from ooni.settings import config + + self.geodata = IPToLocation(self.address) + self.geodata['ip'] = self.address + if not config.privacy.includeasn: + self.geodata['asn'] = 'AS0' + if not config.privacy.includecity: + self.geodata['city'] = None + if not config.privacy.includecountry: + self.geodata['countrycode'] = 'ZZ' + if not config.privacy.includeip: + self.geodata['ip'] = '127.0.0.1' @defer.inlineCallbacks def lookup(self): try: yield self.askTor() log.msg("Found your IP via Tor %s" % self.address) + self.resolveGeodata() defer.returnValue(self.address) except errors.TorStateNotFound: log.debug("Tor is not running. Skipping IP lookup via Tor.") @@ -136,6 +161,7 @@ class ProbeIP(object): try: yield self.askTraceroute() log.msg("Found your IP via Traceroute %s" % self.address) + self.resolveGeodata() defer.returnValue(self.address) except errors.InsufficientPrivileges: log.debug("Cannot determine the probe IP address with a traceroute, becase of insufficient priviledges") @@ -145,6 +171,7 @@ class ProbeIP(object): try: yield self.askGeoIPService() log.msg("Found your IP via a GeoIP service: %s" % self.address) + self.resolveGeodata() defer.returnValue(self.address) except Exception, e: log.msg("Unable to lookup the probe IP via GeoIPService") @@ -183,8 +210,10 @@ class ProbeIP(object): XXX this lookup method is currently broken when there are cached descriptors or consensus documents see: https://trac.torproject.org/projects/tor/ticket/8214 """ - if self.tor_state: - d = self.tor_state.protocol.get_info("address") + from ooni.settings import config + + if config.tor_state: + d = config.tor_state.protocol.get_info("address") @d.addCallback def cb(result): self.strategy = 'tor_get_info_address' diff --git a/ooni/nettest.py b/ooni/nettest.py index ce7831e..2ea0522 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -238,45 +238,15 @@ class NetTestLoader(object): def testDetails(self): from ooni import __version__ as software_version - client_geodata = { - 'city': None, - 'countrycode': 'ZZ', - 'asn': 'AS0', - 'ip': '127.0.0.1' - } - - if config.probe_ip and config.probe_ip.address and (config.privacy.includeip or \ - config.privacy.includeasn or \ - config.privacy.includecountry or \ - config.privacy.includecity): - log.msg("We will include some geo data in the report") - client_geodata = geoip.IPToLocation(config.probe_ip.address) - - if config.privacy.includeip: - client_geodata['ip'] = config.probe_ip.address - else: - client_geodata['ip'] = "127.0.0.1" - - # Here we unset all the client geodata if the option to not include then - # has been specified - if not config.privacy.includeasn: - client_geodata['asn'] = 'AS0' - - if not config.privacy.includecity: - client_geodata['city'] = None - - if not config.privacy.includecountry: - client_geodata['countrycode'] = None - input_file_hashes = [] for input_file in self.inputFiles: input_file_hashes.append(input_file['hash']) test_details = {'start_time': time.time(), - 'probe_asn': client_geodata['asn'], - 'probe_cc': client_geodata['countrycode'], - 'probe_ip': client_geodata['ip'], - 'probe_city': client_geodata['city'], + 'probe_asn': config.probe_ip.geodata['asn'], + 'probe_cc': config.probe_ip.geodata['countrycode'], + 'probe_ip': config.probe_ip.geodata['ip'], + 'probe_city': config.probe_ip.geodata['city'], 'test_name': self.testName, 'test_version': self.testVersion, 'software_name': 'ooniprobe', diff --git a/ooni/settings.py b/ooni/settings.py index c1071b9..0fed604 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -8,7 +8,7 @@ from os.path import abspath, expanduser from twisted.internet import reactor, threads, defer -from ooni import otime +from ooni import otime, geoip from ooni.utils import Storage class OConfig(object): @@ -19,7 +19,7 @@ class OConfig(object): self.scapyFactory = None self.tor_state = None # This is used to store the probes IP address obtained via Tor - self.probe_ip = None + self.probe_ip = geoip.ProbeIP() # This is used to keep track of the state of the sniffer self.sniffer_running = None self.logging = True @@ -91,7 +91,7 @@ class OConfig(object): pass self.set_paths() - def generatePcapFilename(self, testDetails): + def generate_pcap_filename(self, testDetails): test_name, start_time = testDetails['test_name'], testDetails['start_time'] start_time = otime.epochToTimestamp(start_time) return "report-%s-%s.%s" % (test_name, start_time, "pcap") diff --git a/ooni/utils/log.py b/ooni/utils/log.py index 379b5c7..773673b 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -9,7 +9,6 @@ from twisted.python.failure import Failure from twisted.python.logfile import DailyLogFile from ooni import otime -from ooni.settings import config ## Get rid of the annoying "No route found for ## IPv6 destination warnings": @@ -26,6 +25,8 @@ class LogWithNoPrefix(txlog.FileLogObserver): class OONILogger(object): def start(self, logfile=None, application_name="ooniprobe"): + from ooni.settings import config + daily_logfile = None if not logfile: @@ -58,14 +59,17 @@ def stop(): oonilogger.stop() def msg(msg, *arg, **kw): + from ooni.settings import config if config.logging: print "%s" % msg def debug(msg, *arg, **kw): + from ooni.settings import config if config.advanced.debug and config.logging: print "[D] %s" % msg def err(msg, *arg, **kw): + from ooni.settings import config if config.logging: print "[!] %s" % msg [View Less]

1 0

[ooni-probe/master] Merge pull request #301 from olafmk/master
by art＠torproject.org 30 Apr '14

30 Apr '14

commit 2fb54faf3b4e6c08270aba6daa4d212dc9328252 Merge: b705eae 08d6170 Author: Arturo Filastò <hellais(a)users.noreply.github.com> Date: Thu Apr 17 14:08:50 2014 +0200 Merge pull request #301 from olafmk/master Fixes Issue 300 Endless recursion 'Configuration file does not exist' HACKING | 10 +++++----- ooni/settings.py | 12 +++++++----- ooni/utils/log.py | 2 +- 3 files changed, 13 insertions(+), 11 deletions(-)

1 0

[ooni-probe/master] Fixes Issue 300 Endless recursion 'Configuration file does not exist'
by art＠torproject.org 30 Apr '14

30 Apr '14

commit a76b3b049f43e237fe6c958212399674752ec368 Author: olafmk <mail(a)olaf-koehler.com> Date: Fri Apr 11 18:17:24 2014 +0200 Fixes Issue 300 Endless recursion 'Configuration file does not exist' --- HACKING | 10 +++++----- ooni/settings.py | 7 ++++--- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/HACKING b/HACKING index aad7ebb..89b9edf 100644 --- a/HACKING +++ b/HACKING @@ -45,11 +45,6 @@ Code Structure - ooni/ Contains the main ooni probe … [View More]

1 0

[ooni-probe/master] Fix vulnerability that allowed arbitrary files specified in decks to be executed.
by art＠torproject.org 30 Apr '14

30 Apr '14

commit 572e68a859f3241ca257f940c415d22a81bf45d0 Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Apr 22 21:44:23 2014 +0200 Fix vulnerability that allowed arbitrary files specified in decks to be executed. --- ooni/deck.py | 15 ++++++++++----- ooni/oonicli.py | 2 +- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/ooni/deck.py b/ooni/deck.py index cfd5d73..8a26d0c 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -6,6 +6,7 @@ from ooni.settings import … [View More]

1 0

[ooni-probe/master] In unittests no longer specify the test path as relative.
by art＠torproject.org 30 Apr '14

30 Apr '14

commit e3a8aa554d5d65c4888c15bed7cdeac34b8506c4 Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Apr 22 22:13:10 2014 +0200 In unittests no longer specify the test path as relative. --- ooni/tests/test_deck.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/ooni/tests/test_deck.py b/ooni/tests/test_deck.py index 7c7bc35..fbfbef3 100644 --- a/ooni/tests/test_deck.py +++ b/ooni/tests/test_deck.py @@ -39,9 +39,9 @@ class BaseTestCase(… [View More]

1 0

[ooni-probe/master] Add ThreatModel page to ooni documentation (#7761)
by art＠torproject.org 30 Apr '14

30 Apr '14

commit 10ca73fc0ec0bf77339707775f28abfe851c7989 Author: anadahz <kojgelo(a)inbox.com> Date: Mon Apr 28 14:30:18 2014 +0300 Add ThreatModel page to ooni documentation (#7761) Port the ThreatModel (https://trac.torproject.org/projects/tor/wiki/doc/OONI/ThreatModel) documentation to docs. https://trac.torproject.org/projects/tor/ticket/7761 --- docs/source/index.rst | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/source/… [View More]

1 0

[ooni-probe/master] Add OONI installation instructions (#7760)
by art＠torproject.org 30 Apr '14

30 Apr '14

commit a9fb1414b8a5c3f6806b532ed341991c5c41d965 Author: anadahz <kojgelo(a)inbox.com> Date: Sat Apr 26 17:53:38 2014 +0300 Add OONI installation instructions (#7760) Add installation instructions for OONI to the documentation. https://trac.torproject.org/projects/tor/ticket/7760 --- docs/source/index.rst | 198 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 198 insertions(+) diff --git a/docs/source/index.rst b/docs/source/index.rst index e425c4a..… [View More]5be19c1 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -35,6 +35,204 @@ component for running the test. blocked from the probes network point of view. As such they usually require to have specified an input list for running the test. +Installation +------------ + +**Read this before running ooniprobe!** + +Running ooniprobe is a potentially risky activity. This greatly depends on the +jurisdiction in which you are in and which test you are running. It is +technically possible for a person observing your internet connection to be +aware of the fact that you are running ooniprobe. This means that if running +network measurement tests is something considered to be illegal in your country +then you could be spotted. + +Futhermore, ooniprobe takes no precautions to protect the install target machine +from forensics analysis. If the fact that you have installed or used ooni +probe is a liability for you, please be aware of this risk. + +Debian based systems +.................... + +`sudo sh -c 'echo "deb http://deb.ooni.nu/ooni wheezy main" >> /etc/apt/sources.list'` + +`gpg --keyserver pgp.mit.edu --recv-key 0x49B8CDF4` + +`gpg --export 89AB86D4788F3785FE9EDA31F9E2D9B049B8CDF4 | sudo apt-key add -` + +`sudo apt-get update && sudo apt-get install ooniprobe` + +Linux +..... + +We believe that ooniprobe runs reasonably well on Debian GNU/Linux wheezy as +well as versions of Ubuntu such as natty and later releases. Running ooniprobe +without installing it is supported with the following commands: + +`git clone https://git.torproject.org/ooni-probe.git` + +`cd ooni-probe` + +`./setup-dependencies.sh` + +`python setup.py install` + +Setting up development environment +.................................. + +On debian based systems this can be done with: + +`Vsudo apt-get install libgeoip-dev python-virtualenv virtualenvwrapper` + +`mkvirtualenv ooniprobe` + +`python setup.py install` + +`pip install -r requirements-dev.txt` + +Other platforms (with Vagrant) +.............................. + +`Install Vagrant <https://www.vagrantup.com/downloads.html>`_ +and `Install Virtualbox <https://www.virtualbox.org/wiki/Downloads>`_ + +**On OSX:** + +If you don't have it install `homebrew <http://mxcl.github.io/homebrew/>`_ + +`brew install git` + +**On debian/ubuntu:** + +`sudo apt-get install git` + +1. Open a Terminal and run: + +`git clone https://git.torproject.org/ooni-probe.git` + +`cd ooni-probe/` + +`vagrant up` + +2. Login to the box with: + +`vagrant ssh` + +ooniprobe will be installed in `/ooni`. + +3. You can run tests with: + +`ooniprobe blocking/http_requests -f /ooni/inputs/input-pack/alexa-top-1k.txt` + +Using ooniprobe +--------------- + +**Net test** is a set of measurements to assess what kind of internet censorship is occurring. + +**Decks** are collections of ooniprobe nettests with some associated inputs. + +**Collector** is a service used to report the results of measurements. + +**Test helper** is a service used by a probe for successfully performing its measurements. + +**Bouncer** is a service used to discover the addresses of test helpers and collectors. + +Configuring ooniprobe +..................... + +You may edit the configuration for ooniprobe by editing the configuration file +found inside of `~/.ooni/ooniprobe.conf`. + +By default ooniprobe will not include personal identifying information in the +test result, nor create a pcap file. This behavior can be personalized. + +Running decks +............. + +You will find all the installed decks inside of `/usr/share/ooni/decks`. + +You may then run a deck by using the command line option `-i`: + +As root: + +`ooniprobe -i /usr/share/ooni/decks/mlab.deck` + +Or as a user: + +`ooniprobe -i /usr/share/ooni/decks/mlab_no_root.deck` + +Or: + +As root: + +`ooniprobe -i /usr/share/ooni/decks/complete.deck` + +Or as a user: + +`ooniprobe -i /usr/share/ooni/decks/complete_no_root.deck` + +The above tests will require around 20-30 minutes to complete depending on your network speed. + +If you would prefer to run some faster tests you should run: +As root: + +`ooniprobe -i /usr/share/ooni/decks/fast.deck` + +Or as a user: + +`ooniprobe -i /usr/share/ooni/decks/fast_no_root.deck` + +Running net tests +................. + +You may list all the installed stable net tests with: + +`ooniprobe -s` + +You may then run a nettest by specifying its name for example: + +`ooniprobe manipulation/http_header_field_manipulation` + +It is also possible to specify inputs to tests as URLs: + + +`ooniprobe blocking/http_requests -f httpo://ihiderha53f36lsd.onion/input/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1` + +You can find the result of the test in your current working directory. + +By default the report result will be collected by the default ooni collector +and the addresses of test helpers will be obtained from the default bouncer. + +You may also specify your own collector or bouncer with the options `-c` and +`-b`. + +(Optional) Install obfsproxy +---------------------------- + +Install the latest version of obfsproxy for your platform. + +`Download Obfsproxy <https://www.torproject.org/projects/obfsproxy.html.en>`_ + +Bridges and obfsproxy bridges +----------------------------- + +ooniprobe submits reports to oonib report collectors through Tor to a hidden +service endpoint. By default, ooniprobe uses the installed system Tor, but can +also be configured to launch Tor (see the advanced.start_tor option in +ooniprobe.conf), and ooniprobe supports bridges (and obfsproxy bridges, if +obfsproxy is installed). The tor.bridges option in ooniprobe.conf sets the path +to a file that should contain a set of "bridge" lines (of the same format as +used in torrc, and as returned by https://bridges.torproject.org) If obfsproxy +bridges are to be used, the path to the obfsproxy binary must be configured. +See option advanced.obfsproxy_binary, in ooniprobe.conf. + +Setting capabilities on your virtualenv python binary +----------------------------------------------------- + +If your distributation supports capabilities you can avoid needing to run OONI as root: + +`setcap cap_net_admin,cap_net_raw+eip /path/to/your/virtualenv's/python` + Core ooniprobe Tests -------------------- [View Less]

1 0

[ooni-probe/master] Merge pull request #309 from anadahz/trac_7760
by art＠torproject.org 30 Apr '14

30 Apr '14

commit ecba7cd5725ff459964eeedd8b349726eb3c73db Merge: 2fb54fa a9fb141 Author: Arturo Filastò <hellais(a)users.noreply.github.com> Date: Mon Apr 28 10:49:29 2014 +0200 Merge pull request #309 from anadahz/trac_7760 Add OONI installation instructions (#7760) docs/source/index.rst | 198 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 198 insertions(+)

1 0

[ooni-probe/master] Add required packages is setup-dependencies.sh
by art＠torproject.org 30 Apr '14

30 Apr '14

commit c00f369afd7009762922bf207932b126b0a297f4 Author: anadahz <kojgelo(a)inbox.com> Date: Tue Apr 29 02:22:11 2014 +0300 Add required packages is setup-dependencies.sh Add packages in setup-dependencies.sh required for ooniprobe installation script: * Add package libssl-dev * Add package libffi-dev --- setup-dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-dependencies.sh b/setup-dependencies.sh index 2aac715..… [View More]

1 0