commit 71e2b411377319f88b5c19f25e0339158dfcc677 Author: Arturo Filastò <arturo@filasto.net> Date: Fri Jul 8 17:09:11 2016 +0200 Change the default measurement concurrency to 4 and reporting to 7. Rewrite the configuration for lepidopter installations of ooniprobe. * Clean up the data/ tree --- MANIFEST.in | 2 ++ data/Makefile | 10 ---------- data/inputs/.gitignore | 2 ++ data/inputs/Makefile | 13 ------------- data/inputs/README | 47 ---------------------------------------------- data/ooniprobe.conf.sample | 10 +++++----- data/resources/README | 3 ++- setup.py | 22 ++++++++++++++++++++-- 8 files changed, 31 insertions(+), 78 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 920f933..e2903fb 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -6,4 +6,6 @@ include data/ooniprobe.1 include data/oonireport.1 include data/ooniresources.1 include data/ooniprobe.conf.sample +include data/configs/lepidopter-ooniprobe.conf +include data/configs/lepidopter-oonireport.conf include ooni/settings.ini diff --git a/data/Makefile b/data/Makefile deleted file mode 100644 index 969cc65..0000000 --- a/data/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -geoip: - wget -N http://www.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz - gunzip -f GeoIPASNum.dat.gz - wget -N http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.... - gunzip -f GeoIP.dat.gz - wget -N http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz - gunzip -f GeoLiteCity.dat.gz -clean: - rm -f *.dat - diff --git a/data/inputs/.gitignore b/data/inputs/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/data/inputs/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/data/inputs/Makefile b/data/inputs/Makefile deleted file mode 100644 index ff74cf6..0000000 --- a/data/inputs/Makefile +++ /dev/null @@ -1,13 +0,0 @@ -# This file will fetch a set of inputs that are of use to ooni-probe -all: whatheaders lists - -lists: - curl -O https://ooni.torproject.org/inputs/input-pack.tar.gz - tar xzf input-pack.tar.gz - rm input-pack.tar.gz - -whatheaders: - wget http://s3.amazonaws.com/data.whatheaders.com/whatheaders-latest.xml.zip - unzip whatheaders-latest.xml.zip - mv whatheaders*.xml whatheaders.xml - rm whatheaders-latest.xml.zip diff --git a/data/inputs/README b/data/inputs/README deleted file mode 100644 index 9a55f67..0000000 --- a/data/inputs/README +++ /dev/null @@ -1,47 +0,0 @@ -In here you will find some very simple input lists that are useful for testing -the correct functionality of the various OONIProbe tests. - -# DNS Consistency - -./ooniprobe-dev -o dns_tamper_test.yamloo data/nettests/blocking/dns_consistency.py -t -example_inputs/dns_tamper_test_resolvers.txt -f example_inputs/dns_tamper_file.txt - -less dns_tamper_test.yamloo - -# Captive Portal - -./ooniprobe-dev -o captive_portal_test.yamloo data/nettests/core/captiveportal.py - -less captive_portal_test.yamloo - -# HTTP Host - -./ooniprobe-dev -o http_host.yamloo data/nettests/manipulation/http_host.py -b http://ooni.nu/test -f example_inputs/http_host_file.txt - -less http_host.yamloo - -# Keyword filtering - -./ooniprobe-dev -o keyword_filtering.yamloo data/nettests/core/keyword_filtering.py -b http://ooni.nu/test/ -f test_inputs/keyword_filtering_file.txt - -less keyword_filtering.yamloo - -# URL List - -./ooniprobe-dev -o url_lists.yamloo data/nettests/core/url_list.py -f test_inputs/url_lists_file.txt - -less url_lists.yamloo - -# Squid transparent proxy - -./ooniprobe-dev -o squid.yamloo data/nettests/core/squid.py - -less squid.yamloo - -# HTTP Requests - -Not Implemented - -# Traceroute - -Not Implemented diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample index e310ddc..cde4c00 100644 --- a/data/ooniprobe.conf.sample +++ b/data/ooniprobe.conf.sample @@ -33,17 +33,17 @@ advanced: # explicitly set the control port and SOCKS port start_tor: true # After how many seconds we should give up on a particular measurement - measurement_timeout: 60 + measurement_timeout: 120 # After how many retries we should give up on a measurement measurement_retries: 2 # How many measurements to perform concurrently - measurement_concurrency: 10 + measurement_concurrency: 4 # After how may seconds we should give up reporting - reporting_timeout: 80 + reporting_timeout: 360 # After how many retries to give up on reporting - reporting_retries: 3 + reporting_retries: 5 # How many reports to perform concurrently - reporting_concurrency: 15 + reporting_concurrency: 7 oonid_api_port: 8042 report_log_file: null inputs_dir: null diff --git a/data/resources/README b/data/resources/README index 5157b33..fb768f4 100644 --- a/data/resources/README +++ b/data/resources/README @@ -1 +1,2 @@ -In here shall be stored all the resources needed for using ooniresources and oonideckgen. +In here shall be stored all the resources downloaded via ooniresources to be +used by oonideckgen. diff --git a/setup.py b/setup.py index 1138ee3..1a8138c 100644 --- a/setup.py +++ b/setup.py @@ -86,6 +86,8 @@ To setup a daily cronjob run this: Have fun! """ +from __future__ import print_function + from ooni import __version__, __author__ import os import sys @@ -106,13 +108,18 @@ TEST_LISTS_URL = "https://github.com/citizenlab/test-lists/archive/master.zip" def run_command(args, cwd=None): try: p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd) - except EnvironmentError as e: + except EnvironmentError: return None stdout = p.communicate()[0].strip() if p.returncode != 0: return None return stdout +def is_lepidopter(): + if os.path.exists("/etc/default/lepidopter"): + return True + return False + class OoniInstall(install): def gen_config(self, share_path): config_file = pj(tempfile.mkdtemp(), "ooniprobe.conf.sample") @@ -175,11 +182,22 @@ class OoniInstall(install): stderr=sys.stderr.fileno()) process.wait() + def update_lepidopter_config(self): + try: + shutil.copyfile("data/configs/lepidopter-ooniprobe.conf", + "/etc/ooniprobe/ooniprobe.conf") + shutil.copyfile("data/configs/lepidopter-oonireport.conf", + "/etc/ooniprobe/oonireport.conf") + except Exception: + print("ERR: Failed to copy configuration files to /etc/ooniprobe/") + def run(self): prefix = os.path.abspath(self.prefix) self.set_data_files(prefix) self.do_egg_install() self.ooniresources() + if is_lepidopter(): + self.update_lepidopter_config() class ExecutableNotFound(Exception): pass @@ -226,7 +244,7 @@ class CreateOoniResources(Command): try: self.find_executables() except ExecutableNotFound as enf: - print("Could not find '{0}'".format(enf.message)) + print("ERR: Could not find '{0}'".format(enf.message)) return tmp_dir = tempfile.mkdtemp()