[tor-commits] [ooni-probe/master] Review of deckgen branch
art at torproject.org
art at torproject.org
Tue Sep 2 23:20:51 UTC 2014
commit 198681983304f77c0cc20a40b3c31ab66fc699d9
Author: kudrom <kudrom at riseup.net>
Date: Wed Aug 27 17:40:28 2014 +0200
Review of deckgen branch
---
ooni/deckgen/cli.py | 15 +++--------
ooni/deckgen/processors/citizenlab_test_lists.py | 29 +++++++++++-----------
ooni/deckgen/processors/namebench_dns_servers.py | 1 -
ooni/resources/cli.py | 2 --
ooni/resources/update.py | 9 +++++++
ooni/utils/__init__.py | 4 ++-
6 files changed, 30 insertions(+), 30 deletions(-)
diff --git a/ooni/deckgen/cli.py b/ooni/deckgen/cli.py
index 95e0c98..77f5933 100644
--- a/ooni/deckgen/cli.py
+++ b/ooni/deckgen/cli.py
@@ -5,7 +5,7 @@ import errno
import yaml
-from twisted.internet import defer, reactor
+from twisted.internet import defer
from twisted.python import usage
from ooni.geoip import ProbeIP
@@ -36,7 +36,7 @@ class Options(usage.Options):
sys.exit(0)
-class Deck():
+class Deck(object):
_base_entry = {
"options": {
"collector": None,
@@ -105,7 +105,8 @@ def generate_deck(options):
deck.add_test('manipulation/http_invalid_request_line')
deck.add_test('manipulation/http_header_field_manipulation')
# deck.add_test('manipulation/traceroute')
- deck.pprint()
+ if config.advanced.debug:
+ deck.pprint()
deck_filename = os.path.join(options['output'],
"%s-%s-user.deck" % (__version__,
options['country-code']))
@@ -163,11 +164,3 @@ def run():
raise
generate_deck(options)
-
-if __name__ == "__main__":
- d = run()
-
- @d.addBoth
- def cb(_):
- reactor.stop()
- reactor.start()
diff --git a/ooni/deckgen/processors/citizenlab_test_lists.py b/ooni/deckgen/processors/citizenlab_test_lists.py
index 645d714..b36e726 100644
--- a/ooni/deckgen/processors/citizenlab_test_lists.py
+++ b/ooni/deckgen/processors/citizenlab_test_lists.py
@@ -2,6 +2,17 @@ import os
import csv
from ooni.settings import config
+
+def load_input(file_input, file_output):
+ fw = open(file_output, "w+")
+ with open(file_input) as f:
+ csvreader = csv.reader(f)
+ csvreader.next()
+ for row in csvreader:
+ fw.write("%s\n" % row[0])
+ fw.close()
+
+
def generate_country_input(country_code, dst):
"""
Write to dst/citizenlab-urls-{country_code}.txt
@@ -14,7 +25,6 @@ def generate_country_input(country_code, dst):
country_code = country_code.lower()
filename = os.path.join(dst, "citizenlab-urls-%s.txt" % country_code)
- fw = open(filename, "w+")
input_list = os.path.join(config.resources_directory,
"citizenlab-test-lists",
@@ -24,30 +34,19 @@ def generate_country_input(country_code, dst):
if not os.path.exists(input_list):
raise Exception("Could not find list for country %s" % country_code)
- with open(input_list) as f:
- csvreader = csv.reader(f)
- csvreader.next()
- for row in csvreader:
- fw.write("%s\n" % row[0])
+ load_input(input_list, filename)
- fw.close()
return filename
def generate_global_input(dst):
-
filename = os.path.join(dst, "citizenlab-urls-global.txt")
- fw = open(filename, "w+")
input_list = os.path.join(config.resources_directory,
"citizenlab-test-lists",
"test-lists-master",
"csv", "global.csv")
- with open(input_list) as f:
- csvreader = csv.reader(f)
- csvreader.next()
- for row in csvreader:
- fw.write("%s\n" % row[0])
- fw.close()
+ load_input(input_list, filename)
+
return filename
diff --git a/ooni/deckgen/processors/namebench_dns_servers.py b/ooni/deckgen/processors/namebench_dns_servers.py
index 1079335..e434f72 100644
--- a/ooni/deckgen/processors/namebench_dns_servers.py
+++ b/ooni/deckgen/processors/namebench_dns_servers.py
@@ -14,7 +14,6 @@ class GeoIPDB(object):
self.__dict__ = self._borg
if not self.country:
try:
- print config.advanced.geoip_data_dir
country_file = os.path.join(config.advanced.geoip_data_dir,
'GeoIP.dat')
self.country = GeoIP.open(country_file,
diff --git a/ooni/resources/cli.py b/ooni/resources/cli.py
index a140c8d..cab9700 100644
--- a/ooni/resources/cli.py
+++ b/ooni/resources/cli.py
@@ -4,7 +4,6 @@ from twisted.internet import defer
from twisted.python import usage
from ooni.utils import log
-from ooni.settings import config
from ooni.resources import __version__
from ooni.resources import update
@@ -26,7 +25,6 @@ class Options(usage.Options):
@defer.inlineCallbacks
def run():
- config.read_config_file()
options = Options()
try:
options.parseOptions()
diff --git a/ooni/resources/update.py b/ooni/resources/update.py
index a2919fc..6479aa9 100644
--- a/ooni/resources/update.py
+++ b/ooni/resources/update.py
@@ -13,6 +13,15 @@ def download_resource(resources):
print "Downloading %s" % filename
filename = os.path.join(config.resources_directory, filename)
+ if not os.path.exists(filename):
+ directory = os.path.dirname(filename)
+ if not os.path.isdir(directory):
+ os.makedirs(directory)
+ f = open(filename, 'w')
+ f.close()
+ elif not os.path.isfile(filename):
+ print "[!] %s must be a file." % filename
+ defer.returnValue(False)
yield downloadPage(resource['url'], filename)
if resource['action'] is not None:
diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py
index 30a93d5..180b5b3 100644
--- a/ooni/utils/__init__.py
+++ b/ooni/utils/__init__.py
@@ -135,6 +135,7 @@ def generate_filename(testDetails, prefix=None, extension=None, filename=None):
return final_filename
+
def sanitize_options(options):
"""
Strips all possible user identifying information from the ooniprobe test
@@ -147,8 +148,8 @@ def sanitize_options(options):
sanitized_options.append(option)
return sanitized_options
-def unzip(filename, dst):
+def unzip(filename, dst):
assert filename.endswith('.zip')
dst_path = os.path.join(
dst,
@@ -159,6 +160,7 @@ def unzip(filename, dst):
zip_file.extractall(dst_path)
return dst_path
+
def gunzip(filename, dst):
assert filename.endswith(".gz")
dst_path = os.path.join(
More information about the tor-commits
mailing list