[tor-commits] [ooni-probe/master] Change the default measurement concurrency to 4 and reporting to 7.

art at torproject.org art at torproject.org
Sun Jul 10 20:22:57 UTC 2016


commit 71e2b411377319f88b5c19f25e0339158dfcc677
Author: Arturo Filastò <arturo at filasto.net>
Date:   Fri Jul 8 17:09:11 2016 +0200

    Change the default measurement concurrency to 4 and reporting to 7.
    
    Rewrite the configuration for lepidopter installations of ooniprobe.
    
    * Clean up the data/ tree
---
 MANIFEST.in                |  2 ++
 data/Makefile              | 10 ----------
 data/inputs/.gitignore     |  2 ++
 data/inputs/Makefile       | 13 -------------
 data/inputs/README         | 47 ----------------------------------------------
 data/ooniprobe.conf.sample | 10 +++++-----
 data/resources/README      |  3 ++-
 setup.py                   | 22 ++++++++++++++++++++--
 8 files changed, 31 insertions(+), 78 deletions(-)

diff --git a/MANIFEST.in b/MANIFEST.in
index 920f933..e2903fb 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -6,4 +6,6 @@ include data/ooniprobe.1
 include data/oonireport.1
 include data/ooniresources.1
 include data/ooniprobe.conf.sample
+include data/configs/lepidopter-ooniprobe.conf
+include data/configs/lepidopter-oonireport.conf
 include ooni/settings.ini
diff --git a/data/Makefile b/data/Makefile
deleted file mode 100644
index 969cc65..0000000
--- a/data/Makefile
+++ /dev/null
@@ -1,10 +0,0 @@
-geoip:
-	wget -N http://www.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz
-	gunzip -f GeoIPASNum.dat.gz
-	wget -N http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz
-	gunzip -f GeoIP.dat.gz
-	wget -N http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz
-	gunzip -f GeoLiteCity.dat.gz
-clean:
-	rm -f *.dat
-
diff --git a/data/inputs/.gitignore b/data/inputs/.gitignore
new file mode 100644
index 0000000..d6b7ef3
--- /dev/null
+++ b/data/inputs/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
diff --git a/data/inputs/Makefile b/data/inputs/Makefile
deleted file mode 100644
index ff74cf6..0000000
--- a/data/inputs/Makefile
+++ /dev/null
@@ -1,13 +0,0 @@
-# This file will fetch a set of inputs that are of use to ooni-probe
-all: whatheaders lists
-
-lists:
-	curl -O https://ooni.torproject.org/inputs/input-pack.tar.gz
-	tar xzf input-pack.tar.gz
-	rm input-pack.tar.gz
-
-whatheaders:
-	wget http://s3.amazonaws.com/data.whatheaders.com/whatheaders-latest.xml.zip
-	unzip whatheaders-latest.xml.zip
-	mv whatheaders*.xml whatheaders.xml
-	rm whatheaders-latest.xml.zip
diff --git a/data/inputs/README b/data/inputs/README
deleted file mode 100644
index 9a55f67..0000000
--- a/data/inputs/README
+++ /dev/null
@@ -1,47 +0,0 @@
-In here you will find some very simple input lists that are useful for testing
-the correct functionality of the various OONIProbe tests.
-
-# DNS Consistency
-
-./ooniprobe-dev -o dns_tamper_test.yamloo data/nettests/blocking/dns_consistency.py -t
-example_inputs/dns_tamper_test_resolvers.txt -f example_inputs/dns_tamper_file.txt
-
-less dns_tamper_test.yamloo
-
-# Captive Portal
-
-./ooniprobe-dev -o captive_portal_test.yamloo data/nettests/core/captiveportal.py
-
-less captive_portal_test.yamloo
-
-# HTTP Host
-
-./ooniprobe-dev -o http_host.yamloo data/nettests/manipulation/http_host.py -b http://ooni.nu/test -f example_inputs/http_host_file.txt
-
-less http_host.yamloo
-
-# Keyword filtering
-
-./ooniprobe-dev -o keyword_filtering.yamloo data/nettests/core/keyword_filtering.py -b http://ooni.nu/test/ -f test_inputs/keyword_filtering_file.txt
-
-less keyword_filtering.yamloo
-
-# URL List
-
-./ooniprobe-dev -o url_lists.yamloo data/nettests/core/url_list.py -f test_inputs/url_lists_file.txt
-
-less url_lists.yamloo
-
-# Squid transparent proxy
-
-./ooniprobe-dev -o squid.yamloo data/nettests/core/squid.py
-
-less squid.yamloo
-
-# HTTP Requests
-
-Not Implemented
-
-# Traceroute
-
-Not Implemented
diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample
index e310ddc..cde4c00 100644
--- a/data/ooniprobe.conf.sample
+++ b/data/ooniprobe.conf.sample
@@ -33,17 +33,17 @@ advanced:
     # explicitly set the control port and SOCKS port
     start_tor: true
     # After how many seconds we should give up on a particular measurement
-    measurement_timeout: 60
+    measurement_timeout: 120
     # After how many retries we should give up on a measurement
     measurement_retries: 2
     # How many measurements to perform concurrently
-    measurement_concurrency: 10
+    measurement_concurrency: 4
     # After how may seconds we should give up reporting
-    reporting_timeout: 80
+    reporting_timeout: 360
     # After how many retries to give up on reporting
-    reporting_retries: 3
+    reporting_retries: 5
     # How many reports to perform concurrently
-    reporting_concurrency: 15
+    reporting_concurrency: 7
     oonid_api_port: 8042
     report_log_file: null
     inputs_dir: null
diff --git a/data/resources/README b/data/resources/README
index 5157b33..fb768f4 100644
--- a/data/resources/README
+++ b/data/resources/README
@@ -1 +1,2 @@
-In here shall be stored all the resources needed for using ooniresources and oonideckgen.
+In here shall be stored all the resources downloaded via ooniresources to be
+used by oonideckgen.
diff --git a/setup.py b/setup.py
index 1138ee3..1a8138c 100644
--- a/setup.py
+++ b/setup.py
@@ -86,6 +86,8 @@ To setup a daily cronjob run this:
 Have fun!
 """
 
+from __future__ import print_function
+
 from ooni import __version__, __author__
 import os
 import sys
@@ -106,13 +108,18 @@ TEST_LISTS_URL = "https://github.com/citizenlab/test-lists/archive/master.zip"
 def run_command(args, cwd=None):
     try:
         p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd)
-    except EnvironmentError as e:
+    except EnvironmentError:
         return None
     stdout = p.communicate()[0].strip()
     if p.returncode != 0:
         return None
     return stdout
 
+def is_lepidopter():
+    if os.path.exists("/etc/default/lepidopter"):
+        return True
+    return False
+
 class OoniInstall(install):
     def gen_config(self, share_path):
         config_file = pj(tempfile.mkdtemp(), "ooniprobe.conf.sample")
@@ -175,11 +182,22 @@ class OoniInstall(install):
                                    stderr=sys.stderr.fileno())
         process.wait()
 
+    def update_lepidopter_config(self):
+        try:
+            shutil.copyfile("data/configs/lepidopter-ooniprobe.conf",
+                            "/etc/ooniprobe/ooniprobe.conf")
+            shutil.copyfile("data/configs/lepidopter-oonireport.conf",
+                            "/etc/ooniprobe/oonireport.conf")
+        except Exception:
+            print("ERR: Failed to copy configuration files to /etc/ooniprobe/")
+
     def run(self):
         prefix = os.path.abspath(self.prefix)
         self.set_data_files(prefix)
         self.do_egg_install()
         self.ooniresources()
+        if is_lepidopter():
+            self.update_lepidopter_config()
 
 class ExecutableNotFound(Exception):
     pass
@@ -226,7 +244,7 @@ class CreateOoniResources(Command):
         try:
             self.find_executables()
         except ExecutableNotFound as enf:
-            print("Could not find '{0}'".format(enf.message))
+            print("ERR: Could not find '{0}'".format(enf.message))
             return
 
         tmp_dir = tempfile.mkdtemp()





More information about the tor-commits mailing list