September 2014 - tor-commits - lists.torproject.org

r26941: {} New RecommendedVersions file location for TBB. (website/trunk/projects/torbrowser)
by Mike Perry 03 Sep '14

03 Sep '14

Author: mikeperry Date: 2014-09-03 05:26:53 +0000 (Wed, 03 Sep 2014) New Revision: 26941 Added: website/trunk/projects/torbrowser/RecommendedTBBVersions Log: New RecommendedVersions file location for TBB. Added: website/trunk/projects/torbrowser/RecommendedTBBVersions =================================================================== --- website/trunk/projects/torbrowser/RecommendedTBBVersions (rev 0) +++ website/trunk/projects/torbrowser/RecommendedTBBVersions 2014-09-03 05:26:53 UTC (rev 26941) @@ -0,0 +1,8 @@ +[ +"3.6.5-Linux", +"3.6.5-MacOS", +"3.6.5-Windows", +"4.0-alpha-2-Linux", +"4.0-alpha-2-MacOS", +"4.0-alpha-2-Windows" +]

1 0

[ooni-probe/master] If no country code is specified lookup the country code of the user.
by art＠torproject.org 02 Sep '14

02 Sep '14

commit 4c09c008e3f634a12fd971263d8ade77d87565db Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Aug 13 16:02:34 2014 +0200 If no country code is specified lookup the country code of the user. --- bin/oonideckgen | 7 +++- ooni/deckgen/cli.py | 104 +++++++++++++++++++++++++++++++-------------------- 2 files changed, 68 insertions(+), 43 deletions(-) diff --git a/bin/oonideckgen b/bin/oonideckgen index c990160..74ea8de 100755 --- a/bin/oonideckgen +++ b/bin/oonideckgen @@ -1,6 +1,7 @@ #!/usr/bin/env python import os import sys +import exceptions sys.path[:] = map(os.path.abspath, sys.path) sys.path.insert(0, os.path.abspath(os.getcwd())) @@ -11,8 +12,10 @@ from ooni.utils import log from ooni.deckgen import cli def failed(failure): - log.err("Failed to run oonideckgen") - log.exception(failure) + r = failure.trap(exceptions.SystemExit) + if r != exceptions.SystemExit: + log.err("Failed to run oonideckgen") + log.exception(failure) reactor.stop() def done(result): diff --git a/ooni/deckgen/cli.py b/ooni/deckgen/cli.py index 406417d..95e0c98 100644 --- a/ooni/deckgen/cli.py +++ b/ooni/deckgen/cli.py @@ -5,16 +5,19 @@ import errno import yaml +from twisted.internet import defer, reactor from twisted.python import usage -from . import __version__ -from ooni.resources import inputs +from ooni.geoip import ProbeIP from ooni.settings import config +from ooni.deckgen import __version__ +from ooni.resources import inputs + class Options(usage.Options): synopsis = """%s [options] - """ + """ % sys.argv[0] optParameters = [ ["country-code", "c", @@ -32,23 +35,6 @@ class Options(usage.Options): print("oonideckgen version: %s" % __version__) sys.exit(0) - def postOptions(self): - if not self['output'] or not self['country-code']: - raise usage.UsageError( - "Both --output and --country-code are required" - ) - if len(self['country-code']) != 2: - raise usage.UsageError("--country-code must be 2 characters") - if not os.path.isdir(self['output']): - raise usage.UsageError("%s is not a directory" % self['output']) - - self['country-code'] = self['country-code'].lower() - - output_dir = os.path.abspath(self['output']) - output_dir = os.path.join(output_dir, - "deck-%s" % self['country-code']) - self['output'] = output_dir - class Deck(): _base_entry = { @@ -82,26 +68,7 @@ class Deck(): f.write(yaml.safe_dump(self.deck)) -def usage(): - print "%s <two letter country code> <output dir>" % sys.argv[0] - - -def run(): - options = Options() - try: - options.parseOptions() - except usage.UsageError as error_message: - print "%s: %s" % (sys.argv[0], error_message) - print "%s: Try --help for usage details." % (sys.argv[0]) - sys.exit(1) - - config.read_config_file() - - try: - os.makedirs(options['output']) - except OSError as exception: - if exception.errno != errno.EEXIST: - raise +def generate_deck(options): dns_servers_processor = inputs['namebench-dns-servers.csv']['processor'] url_lists_processor = inputs['citizenlab-test-lists.zip']['processor'] @@ -147,5 +114,60 @@ def run(): print "Run ooniprobe like so:" print "ooniprobe -i %s" % deck_filename + +(a)defer.inlineCallbacks +def get_user_country_code(): + probe_ip = ProbeIP() + yield probe_ip.lookup() + defer.returnValue(probe_ip.geodata['countrycode']) + + +(a)defer.inlineCallbacks +def run(): + options = Options() + try: + options.parseOptions() + except usage.UsageError as error_message: + print "%s: %s" % (sys.argv[0], error_message) + print options + sys.exit(1) + + if not options['output']: + options['output'] = os.getcwd() + + if not options['country-code']: + options['country-code'] = yield get_user_country_code() + + if len(options['country-code']) != 2: + print "%s: --country-code must be 2 characters" % sys.argv[0] + sys.exit(2) + + if not os.path.isdir(options['output']): + print "%s: %s is not a directory" % (sys.argv[0], + options['output']) + sys.exit(3) + + options['country-code'] = options['country-code'].lower() + + output_dir = os.path.abspath(options['output']) + output_dir = os.path.join(output_dir, + "deck-%s" % options['country-code']) + options['output'] = output_dir + + config.read_config_file() + + try: + os.makedirs(options['output']) + except OSError as exception: + if exception.errno != errno.EEXIST: + raise + + generate_deck(options) + if __name__ == "__main__": - run() + d = run() + + @d.addBoth + def cb(_): + reactor.stop() + reactor.start()

1 0

[ooni-probe/master] Simplify the code for downloading resources.
by art＠torproject.org 02 Sep '14

02 Sep '14

commit 8545423373dee1f4b801375922b67bc2417cb426 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Aug 18 13:28:36 2014 +0200 Simplify the code for downloading resources. Use downloadPage instead of our own class. --- ooni/resources/update.py | 30 +++--------------------------- 1 file changed, 3 insertions(+), 27 deletions(-) diff --git a/ooni/resources/update.py b/ooni/resources/update.py index 4cb0f49..a2919fc 100644 --- a/ooni/resources/update.py +++ b/ooni/resources/update.py @@ -1,32 +1,11 @@ import os -from twisted.internet import reactor, defer, protocol -from twisted.web.client import RedirectAgent, Agent +from twisted.internet import defer +from twisted.web.client import downloadPage from ooni.settings import config from ooni.resources import inputs, geoip -agent = RedirectAgent(Agent(reactor)) - - -class SaveToFile(protocol.Protocol): - def __init__(self, finished, filesize, filename): - self.finished = finished - self.remaining = filesize - self.outfile = open(filename, 'wb') - - def dataReceived(self, bytes): - if self.remaining: - display = bytes[:self.remaining] - self.outfile.write(display) - self.remaining -= len(display) - else: - self.outfile.close() - - def connectionLost(self, reason): - self.outfile.close() - self.finished.callback(None) - @defer.inlineCallbacks def download_resource(resources): @@ -34,10 +13,7 @@ def download_resource(resources): print "Downloading %s" % filename filename = os.path.join(config.resources_directory, filename) - response = yield agent.request("GET", resource['url']) - finished = defer.Deferred() - response.deliverBody(SaveToFile(finished, response.length, filename)) - yield finished + yield downloadPage(resource['url'], filename) if resource['action'] is not None: yield defer.maybeDeferred(resource['action'],

1 0

[ooni-probe/master] Simplify the commands in the 5 minute ooni command
by art＠torproject.org 02 Sep '14

02 Sep '14

commit 135e683b26f6bbaa65f80740679ec678613a57a4 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Aug 13 20:28:36 2014 +0200 Simplify the commands in the 5 minute ooni command --- README.rst | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index d85e352..bb4c59c 100644 --- a/README.rst +++ b/README.rst @@ -62,23 +62,21 @@ On unix systems:: The run:: - YOUR_COUNTRY_CODE="it" - PATH_TO_MY_DECK=`pwd` mkdir my_decks sudo ooniresource --update-inputs --update-geoip - oonideckgen --country-code $YOUR_COUNTRY_CODE -o my_decks/ - ooniprobe -i my_decks/deck-$YOUR_COUNTRY_CODE/0.0.1-$YOUR_COUNTRY_CODE-user.deck + oonideckgen -o my_decks/ -The output from oonideckgen should give you the full path to the test deck. +The output from the last command will tell you how to run ooniprobe to perform +the measurment. If you would like to contribute measurements to OONI daily you can also add this to your crontab:: - @daily ooni /usr/bin/ooniprobe -i REPLACE_WITH_PATH_TO_THE_DECK + @daily ooni $THE_OONI_COMMAND -This command should automatically do it if you did not quit the shell:: +Run this command to automatically update your crontab:: - (crontab -l 2>/dev/null; echo "@daily ooni /usr/bin/ooniprobe -i $PATH_TO_MY_DECK/my_decks/$YOUR_COUNTRY_CODE/0.0.1-$YOUR_COUNTRY_CODE-user.deck") | crontab - + (crontab -l 2>/dev/null; echo "@daily ooni $THE_OONI_COMMAND") | crontab - Installation ============

1 0

[ooni-probe/master] Merge pull request #347 from kudrom/feature/deckgen
by art＠torproject.org 02 Sep '14

02 Sep '14

commit aeccb188dd083cc23da3ba74b5c105b05467f5dd Merge: 8545423 1986819 Author: aagbsn <aagbsn(a)extc.org> Date: Tue Sep 2 15:48:58 2014 +0000 Merge pull request #347 from kudrom/feature/deckgen Review of deckgen branch ooni/deckgen/cli.py | 15 +++-------- ooni/deckgen/processors/citizenlab_test_lists.py | 29 +++++++++++----------- ooni/deckgen/processors/namebench_dns_servers.py | 1 - ooni/resources/cli.py | 2 -- ooni/resources/update.py | 9 +++++++ ooni/utils/__init__.py | 4 ++- 6 files changed, 30 insertions(+), 30 deletions(-)

1 0

[ooni-probe/master] Add resources/ directory
by art＠torproject.org 02 Sep '14

02 Sep '14

commit abb1e315e4d0e8d956eda7770461836cc33043a8 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Aug 13 18:35:07 2014 +0200 Add resources/ directory --- data/resources/README | 1 + 1 file changed, 1 insertion(+) diff --git a/data/resources/README b/data/resources/README new file mode 100644 index 0000000..5157b33 --- /dev/null +++ b/data/resources/README @@ -0,0 +1 @@ +In here shall be stored all the resources needed for using ooniresources and oonideckgen.

1 0

[ooni-probe/master] Review of deckgen branch
by art＠torproject.org 02 Sep '14

02 Sep '14

commit 198681983304f77c0cc20a40b3c31ab66fc699d9 Author: kudrom <kudrom(a)riseup.net> Date: Wed Aug 27 17:40:28 2014 +0200 Review of deckgen branch --- ooni/deckgen/cli.py | 15 +++-------- ooni/deckgen/processors/citizenlab_test_lists.py | 29 +++++++++++----------- ooni/deckgen/processors/namebench_dns_servers.py | 1 - ooni/resources/cli.py | 2 -- ooni/resources/update.py | 9 +++++++ ooni/utils/__init__.py | 4 ++- 6 files changed, 30 insertions(+), 30 deletions(-) diff --git a/ooni/deckgen/cli.py b/ooni/deckgen/cli.py index 95e0c98..77f5933 100644 --- a/ooni/deckgen/cli.py +++ b/ooni/deckgen/cli.py @@ -5,7 +5,7 @@ import errno import yaml -from twisted.internet import defer, reactor +from twisted.internet import defer from twisted.python import usage from ooni.geoip import ProbeIP @@ -36,7 +36,7 @@ class Options(usage.Options): sys.exit(0) -class Deck(): +class Deck(object): _base_entry = { "options": { "collector": None, @@ -105,7 +105,8 @@ def generate_deck(options): deck.add_test('manipulation/http_invalid_request_line') deck.add_test('manipulation/http_header_field_manipulation') # deck.add_test('manipulation/traceroute') - deck.pprint() + if config.advanced.debug: + deck.pprint() deck_filename = os.path.join(options['output'], "%s-%s-user.deck" % (__version__, options['country-code'])) @@ -163,11 +164,3 @@ def run(): raise generate_deck(options) - -if __name__ == "__main__": - d = run() - - @d.addBoth - def cb(_): - reactor.stop() - reactor.start() diff --git a/ooni/deckgen/processors/citizenlab_test_lists.py b/ooni/deckgen/processors/citizenlab_test_lists.py index 645d714..b36e726 100644 --- a/ooni/deckgen/processors/citizenlab_test_lists.py +++ b/ooni/deckgen/processors/citizenlab_test_lists.py @@ -2,6 +2,17 @@ import os import csv from ooni.settings import config + +def load_input(file_input, file_output): + fw = open(file_output, "w+") + with open(file_input) as f: + csvreader = csv.reader(f) + csvreader.next() + for row in csvreader: + fw.write("%s\n" % row[0]) + fw.close() + + def generate_country_input(country_code, dst): """ Write to dst/citizenlab-urls-{country_code}.txt @@ -14,7 +25,6 @@ def generate_country_input(country_code, dst): country_code = country_code.lower() filename = os.path.join(dst, "citizenlab-urls-%s.txt" % country_code) - fw = open(filename, "w+") input_list = os.path.join(config.resources_directory, "citizenlab-test-lists", @@ -24,30 +34,19 @@ def generate_country_input(country_code, dst): if not os.path.exists(input_list): raise Exception("Could not find list for country %s" % country_code) - with open(input_list) as f: - csvreader = csv.reader(f) - csvreader.next() - for row in csvreader: - fw.write("%s\n" % row[0]) + load_input(input_list, filename) - fw.close() return filename def generate_global_input(dst): - filename = os.path.join(dst, "citizenlab-urls-global.txt") - fw = open(filename, "w+") input_list = os.path.join(config.resources_directory, "citizenlab-test-lists", "test-lists-master", "csv", "global.csv") - with open(input_list) as f: - csvreader = csv.reader(f) - csvreader.next() - for row in csvreader: - fw.write("%s\n" % row[0]) - fw.close() + load_input(input_list, filename) + return filename diff --git a/ooni/deckgen/processors/namebench_dns_servers.py b/ooni/deckgen/processors/namebench_dns_servers.py index 1079335..e434f72 100644 --- a/ooni/deckgen/processors/namebench_dns_servers.py +++ b/ooni/deckgen/processors/namebench_dns_servers.py @@ -14,7 +14,6 @@ class GeoIPDB(object): self.__dict__ = self._borg if not self.country: try: - print config.advanced.geoip_data_dir country_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIP.dat') self.country = GeoIP.open(country_file, diff --git a/ooni/resources/cli.py b/ooni/resources/cli.py index a140c8d..cab9700 100644 --- a/ooni/resources/cli.py +++ b/ooni/resources/cli.py @@ -4,7 +4,6 @@ from twisted.internet import defer from twisted.python import usage from ooni.utils import log -from ooni.settings import config from ooni.resources import __version__ from ooni.resources import update @@ -26,7 +25,6 @@ class Options(usage.Options): @defer.inlineCallbacks def run(): - config.read_config_file() options = Options() try: options.parseOptions() diff --git a/ooni/resources/update.py b/ooni/resources/update.py index a2919fc..6479aa9 100644 --- a/ooni/resources/update.py +++ b/ooni/resources/update.py @@ -13,6 +13,15 @@ def download_resource(resources): print "Downloading %s" % filename filename = os.path.join(config.resources_directory, filename) + if not os.path.exists(filename): + directory = os.path.dirname(filename) + if not os.path.isdir(directory): + os.makedirs(directory) + f = open(filename, 'w') + f.close() + elif not os.path.isfile(filename): + print "[!] %s must be a file." % filename + defer.returnValue(False) yield downloadPage(resource['url'], filename) if resource['action'] is not None: diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index 30a93d5..180b5b3 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -135,6 +135,7 @@ def generate_filename(testDetails, prefix=None, extension=None, filename=None): return final_filename + def sanitize_options(options): """ Strips all possible user identifying information from the ooniprobe test @@ -147,8 +148,8 @@ def sanitize_options(options): sanitized_options.append(option) return sanitized_options -def unzip(filename, dst): +def unzip(filename, dst): assert filename.endswith('.zip') dst_path = os.path.join( dst, @@ -159,6 +160,7 @@ def unzip(filename, dst): zip_file.extractall(dst_path) return dst_path + def gunzip(filename, dst): assert filename.endswith(".gz") dst_path = os.path.join(

1 0

[ooni-probe/master] Merge pull request #341 from TheTorProject/feature/deckgen
by art＠torproject.org 02 Sep '14

02 Sep '14

commit 49849e525cfe26075593973cba7b6c03336b1ca8 Merge: 8c56b16 aeccb18 Author: aagbsn <aagbsn(a)extc.org> Date: Tue Sep 2 15:56:57 2014 +0000 Merge pull request #341 from TheTorProject/feature/deckgen Feature/deckgen README.rst | 91 ++++++++++-- bin/oonideckgen | 30 ++++ bin/oonireport | 3 +- bin/ooniresources | 27 ++++ data/resources/README | 1 + ooni/deckgen/__init__.py | 1 + ooni/deckgen/cli.py | 166 ++++++++++++++++++++++ ooni/deckgen/processors/citizenlab_test_lists.py | 52 +++++++ ooni/deckgen/processors/namebench_dns_servers.py | 52 +++++++ ooni/geoip.py | 19 +-- ooni/resources/__init__.py | 41 ++++++ ooni/resources/cli.py | 55 +++++++ ooni/resources/update.py | 39 +++++ ooni/settings.py | 1 + ooni/utils/__init__.py | 29 ++++ setup.py | 5 +- 16 files changed, 591 insertions(+), 21 deletions(-)

1 0

[ooni-probe/master] Fix bug that lead to the transport name not being correct when the pluggable transport was not installed.
by art＠torproject.org 02 Sep '14

02 Sep '14

commit 90aa08c979ae484488f6b33a6c85e9e2ceef1db2 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Sep 3 01:10:11 2014 +0200 Fix bug that lead to the transport name not being correct when the pluggable transport was not installed. Thanks to asn for spotting it. --- ooni/nettests/blocking/bridge_reachability.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ooni/nettests/blocking/bridge_reachability.py b/ooni/nettests/blocking/bridge_reachability.py index 9957430..e0d0d8a 100644 --- a/ooni/nettests/blocking/bridge_reachability.py +++ b/ooni/nettests/blocking/bridge_reachability.py @@ -28,7 +28,7 @@ class BridgeReachability(nettest.NetTestCase): description = "A test for checking if bridges are reachable " \ "from a given location." author = "Arturo Filastò" - version = "0.1.1" + version = "0.1.2" usageOptions = UsageOptions @@ -124,11 +124,14 @@ class BridgeReachability(nettest.NetTestCase): (self.bridge, onion.tor_details['version'])) transport_name = onion.transport_name(self.bridge) + + if transport_name: + self.report['transport_name'] = transport_name + if transport_name and transport_name == 'fte' and self.fteproxy_bin: config.ClientTransportPlugin = "%s exec %s --managed" % ( transport_name, self.fteproxy_bin) log.debug("Using fte from %s" % self.fteproxy_bin) - self.report['transport_name'] = transport_name self.report['bridge_address'] = self.bridge.split(' ')[1] elif transport_name and transport_name == 'fte'\ and not self.fteproxy_bin: @@ -145,7 +148,6 @@ class BridgeReachability(nettest.NetTestCase): self.report['error'] = 'old-obfsproxy' return log.debug("Using pyobfsproxy from %s" % self.pyobfsproxy_bin) - self.report['transport_name'] = transport_name self.report['bridge_address'] = self.bridge.split(' ')[1] elif transport_name and not self.pyobfsproxy_bin: log.err(

1 0

[ooni-probe/master] Merge pull request #331 from anadahz/setup_dep.fix
by art＠torproject.org 02 Sep '14

02 Sep '14

commit 8a9fcd57d4126a9ef485d282085c81580d235c5c Merge: 49849e5 d1436ab Author: aagbsn <aagbsn(a)extc.org> Date: Tue Sep 2 17:04:35 2014 +0000 Merge pull request #331 from anadahz/setup_dep.fix Add Debian jessie support in setup script setup-dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

1 0