[tor-commits] [ooni-probe/master] Merge branch 'twisted' of github.com:hellais/ooni-probe
art at torproject.org
art at torproject.org
Sat Jun 2 18:34:28 UTC 2012
commit baab66b10eccb5a3bc40addc84de0e45f2897497
Merge: abfcf7c 59f1b4e
Author: Arturo Filastò <hellais at torproject.org>
Date: Tue May 29 03:23:01 2012 +0200
Merge branch 'twisted' of github.com:hellais/ooni-probe
HACKING | 36 +-
assets/example.txt | 2 -
assets/tcpscan.txt | 1 -
config.py | 53 -
docs/writing_tests.txt | 12 +
install.txt | 4 -
lib/traceroute.py | 32 -
lib/txtraceroute.py | 389 -
logo.py | 214 -
nodes.conf | 53 -
old-to-be-ported-code/AUTHORS | 3 +
old-to-be-ported-code/CHANGES.yaml | 5 +
old-to-be-ported-code/HACKING | 21 +
old-to-be-ported-code/INSTALL | 8 +
old-to-be-ported-code/LICENSE | 28 +
old-to-be-ported-code/MANIFEST.in | 6 +
old-to-be-ported-code/Makefile | 182 +
old-to-be-ported-code/README.API | 29 +
old-to-be-ported-code/README.examples | 10 +
old-to-be-ported-code/README.rst | 40 +
old-to-be-ported-code/RFA.md | 101 +
old-to-be-ported-code/TODO | 418 +
old-to-be-ported-code/TODO.plgoons | 79 +
old-to-be-ported-code/bin/ooni-probe | 10 +
old-to-be-ported-code/keyword-lists/Makefile | 3 +
old-to-be-ported-code/keyword-lists/keyfile | 256 +
old-to-be-ported-code/list/list.txt | 2 +
old-to-be-ported-code/ooni-probe.diff | 358 +
old-to-be-ported-code/ooni/.DS_Store | Bin 0 -> 15364 bytes
old-to-be-ported-code/ooni/__init__.py | 12 +
old-to-be-ported-code/ooni/captive_portal.py | 62 +
old-to-be-ported-code/ooni/command.py | 250 +
old-to-be-ported-code/ooni/common.py | 139 +
old-to-be-ported-code/ooni/dns_cc_check.py | 48 +
old-to-be-ported-code/ooni/dns_poisoning.py | 43 +
old-to-be-ported-code/ooni/dnsooni.py | 356 +
old-to-be-ported-code/ooni/empty.txt | 9 +
old-to-be-ported-code/ooni/helpers.py | 38 +
old-to-be-ported-code/ooni/http.py | 306 +
old-to-be-ported-code/ooni/input.py | 33 +
old-to-be-ported-code/ooni/namecheck.py | 39 +
old-to-be-ported-code/ooni/output.py | 21 +
.../ooni/plugins/captiveportal_plgoo.py | 55 +
.../ooni/plugins/dnstest_plgoo.py | 84 +
old-to-be-ported-code/ooni/plugins/http_plgoo.py | 70 +
old-to-be-ported-code/ooni/plugins/marco_plgoo.py | 377 +
.../ooni/plugins/netalyzr_plgoo.py | 30 +
.../ooni/plugins/old_stuff_to_convert/README | 47 +
.../plugins/old_stuff_to_convert/README.automation | 11 +
.../ooni/plugins/old_stuff_to_convert/TODO | 6 +
.../plugins/old_stuff_to_convert/cached-consensus |11724 ++++++++++++++++++++
.../plugins/old_stuff_to_convert/connectback.sh | 56 +
.../plugins/old_stuff_to_convert/dirconntest.sh | 52 +
.../plugins/old_stuff_to_convert/dns-checker.sh | 7 +
.../old_stuff_to_convert/generic-host-test.sh | 33 +
.../ooni/plugins/old_stuff_to_convert/host-prep.sh | 20 +
.../plugins/old_stuff_to_convert/install-probe.sh | 27 +
.../ooni/plugins/old_stuff_to_convert/run-tests.sh | 11 +
.../plugins/old_stuff_to_convert/twitter-test.sh | 33 +
old-to-be-ported-code/ooni/plugins/proxy_plgoo.py | 69 +
.../ooni/plugins/simple_dns_plgoo.py | 35 +
old-to-be-ported-code/ooni/plugins/skel_plgoo.py | 17 +
old-to-be-ported-code/ooni/plugins/skel_plgoo.yaml | 33 +
old-to-be-ported-code/ooni/plugins/tcpcon_plgoo.py | 278 +
old-to-be-ported-code/ooni/plugins/tor.py | 80 +
old-to-be-ported-code/ooni/plugins/torrc | 9 +
old-to-be-ported-code/ooni/plugooni.py | 106 +
old-to-be-ported-code/ooni/report.py | 22 +
old-to-be-ported-code/ooni/transparenthttp.py | 41 +
old-to-be-ported-code/ooni/yamlooni.py | 40 +
old-to-be-ported-code/proxy-lists/Makefile | 11 +
old-to-be-ported-code/proxy-lists/README | 9 +
old-to-be-ported-code/proxy-lists/italy-dns.txt | 46 +
.../proxy-lists/italy-http-block-pages-notes.txt | 62 +
.../proxy-lists/italy-http-block-pages.txt | 5 +
.../proxy-lists/parse-trusted-xff.sh | 16 +
old-to-be-ported-code/proxy-lists/trusted-xff.html | 7789 +++++++++++++
old-to-be-ported-code/reports/marco-1.yamlooni | 86 +
old-to-be-ported-code/reports/marco-2.yamlooni | 60 +
old-to-be-ported-code/reports/marco.log | 4672 ++++++++
old-to-be-ported-code/reports/marco.yamlooni | 241 +
old-to-be-ported-code/reports/marco_certs-1-2.out | 307 +
old-to-be-ported-code/reports/marco_certs-1.out | 5850 ++++++++++
old-to-be-ported-code/reports/marco_certs-2.out | 1182 ++
old-to-be-ported-code/reports/marco_certs.out | 1900 ++++
old-to-be-ported-code/setup.py | 10 +
old-to-be-ported-code/spec/proxooni-spec.txt | 65 +
old-to-be-ported-code/test-lists/Makefile | 7 +
old-to-be-ported-code/tests/test_import.py | 10 +
old-to-be-ported-code/third-party/Makefile | 3 +
old-to-be-ported-code/third-party/README | 14 +
old/AUTHORS | 3 -
old/CHANGES.yaml | 5 -
old/HACKING | 21 -
old/INSTALL | 8 -
old/LICENSE | 28 -
old/MANIFEST.in | 6 -
old/Makefile | 182 -
old/README.API | 29 -
old/README.examples | 10 -
old/README.rst | 40 -
old/RFA.md | 101 -
old/TODO | 418 -
old/TODO.plgoons | 79 -
old/bin/ooni-probe | 10 -
old/keyword-lists/Makefile | 3 -
old/keyword-lists/keyfile | 256 -
old/list/list.txt | 2 -
old/ooni-probe.diff | 358 -
old/ooni/.DS_Store | Bin 15364 -> 0 bytes
old/ooni/__init__.py | 12 -
old/ooni/captive_portal.py | 62 -
old/ooni/command.py | 250 -
old/ooni/common.py | 139 -
old/ooni/dns_cc_check.py | 48 -
old/ooni/dns_poisoning.py | 43 -
old/ooni/dnsooni.py | 356 -
old/ooni/empty.txt | 9 -
old/ooni/helpers.py | 38 -
old/ooni/http.py | 306 -
old/ooni/input.py | 33 -
old/ooni/namecheck.py | 39 -
old/ooni/output.py | 21 -
old/ooni/plugins/captiveportal_plgoo.py | 55 -
old/ooni/plugins/dnstest_plgoo.py | 84 -
old/ooni/plugins/http_plgoo.py | 70 -
old/ooni/plugins/marco_plgoo.py | 377 -
old/ooni/plugins/netalyzr_plgoo.py | 30 -
old/ooni/plugins/old_stuff_to_convert/README | 47 -
.../plugins/old_stuff_to_convert/README.automation | 11 -
old/ooni/plugins/old_stuff_to_convert/TODO | 6 -
.../plugins/old_stuff_to_convert/cached-consensus |11724 --------------------
.../plugins/old_stuff_to_convert/connectback.sh | 56 -
.../plugins/old_stuff_to_convert/dirconntest.sh | 52 -
.../plugins/old_stuff_to_convert/dns-checker.sh | 7 -
.../old_stuff_to_convert/generic-host-test.sh | 33 -
old/ooni/plugins/old_stuff_to_convert/host-prep.sh | 20 -
.../plugins/old_stuff_to_convert/install-probe.sh | 27 -
old/ooni/plugins/old_stuff_to_convert/run-tests.sh | 11 -
.../plugins/old_stuff_to_convert/twitter-test.sh | 33 -
old/ooni/plugins/proxy_plgoo.py | 69 -
old/ooni/plugins/simple_dns_plgoo.py | 35 -
old/ooni/plugins/skel_plgoo.py | 17 -
old/ooni/plugins/skel_plgoo.yaml | 33 -
old/ooni/plugins/tcpcon_plgoo.py | 278 -
old/ooni/plugins/tor.py | 80 -
old/ooni/plugins/torrc | 9 -
old/ooni/plugooni.py | 106 -
old/ooni/report.py | 22 -
old/ooni/transparenthttp.py | 41 -
old/ooni/yamlooni.py | 40 -
old/proxy-lists/Makefile | 11 -
old/proxy-lists/README | 9 -
old/proxy-lists/italy-dns.txt | 46 -
old/proxy-lists/italy-http-block-pages-notes.txt | 62 -
old/proxy-lists/italy-http-block-pages.txt | 5 -
old/proxy-lists/parse-trusted-xff.sh | 16 -
old/proxy-lists/trusted-xff.html | 7789 -------------
old/reports/marco-1.yamlooni | 86 -
old/reports/marco-2.yamlooni | 60 -
old/reports/marco.log | 4672 --------
old/reports/marco.yamlooni | 241 -
old/reports/marco_certs-1-2.out | 307 -
old/reports/marco_certs-1.out | 5850 ----------
old/reports/marco_certs-2.out | 1182 --
old/reports/marco_certs.out | 1900 ----
old/setup.py | 10 -
old/spec/proxooni-spec.txt | 65 -
old/test-lists/Makefile | 7 -
old/tests/test_import.py | 10 -
old/third-party/Makefile | 3 -
old/third-party/README | 14 -
ooni-probe.conf | 77 -
ooni/assets/bridgetests.txt | 3 +
ooni/assets/example.txt | 2 +
ooni/assets/tcpscan.txt | 1 +
ooni/config.py | 53 +
ooni/lib/Makefile | 14 +
ooni/logo.py | 214 +
ooni/nodes.conf | 53 +
ooni/ooni-probe.conf | 77 +
ooni/oonicli.py | 139 +
ooni/ooniprobe.log | 2 +
ooni/ooniprobe.py | 191 +
ooni/oonitests/bridget.py | 373 +
ooni/oonitests/dnstamper.py | 156 +
ooni/oonitests/httphost.py | 135 +
ooni/oonitests/squid.py | 113 +
ooni/oonitests/tcpscan.py | 86 +
ooni/oonitests/template.py | 65 +
ooni/oonitests/traceroute.py | 110 +
ooni/plugins/.bridget.py.swp | Bin 0 -> 12288 bytes
ooni/plugins/.dnstamper.py.swp | Bin 0 -> 12288 bytes
ooni/plugins/__init__.py | 3 +
ooni/plugins/bridget.py | 36 +
ooni/plugins/dnstamper.py | 22 +
ooni/plugins/dropin.cache | 76 +
ooni/plugins/skel.py | 21 +
ooni/plugoo/__init__.py | 80 +
ooni/plugoo/assets.py | 56 +
ooni/plugoo/interface.py | 2 +
ooni/plugoo/nodes.py | 180 +
ooni/plugoo/reports.py | 175 +
ooni/plugoo/tests.py | 229 +
ooni/plugoo/work.py | 162 +
ooni/skel.py | 10 +
ooni/tests/worker_test.py | 27 +
ooni/utils.py | 146 +
oonib/dnsbackend.py | 16 +
oonib/httpbackend.py | 74 +
oonib/oonibackend.py | 34 +
oonicli.py | 142 -
ooniprobe.py | 191 -
plugins/__init__.py | 3 -
plugins/dropin.cache | 48 -
plugins/skel.py | 20 -
plugoo/__init__.py | 80 -
plugoo/assets.py | 56 -
plugoo/interface.py | 2 -
plugoo/nodes.py | 180 -
plugoo/reports.py | 175 -
plugoo/tests.py | 213 -
plugoo/work.py | 162 -
skel.py | 10 -
tests/bridget.py | 378 -
tests/dnstamper.py | 156 -
tests/httphost.py | 135 -
tests/squid.py | 113 -
tests/tcpscan.py | 86 -
tests/template.py | 65 -
tests/traceroute.py | 110 -
unittest/tests.py | 27 -
utils.py | 146 -
233 files changed, 41804 insertions(+), 41969 deletions(-)
diff --cc ooni/ooni-probe.conf
index 0000000,336af99..d95e410
mode 000000,100644..100644
--- a/ooni/ooni-probe.conf
+++ b/ooni/ooni-probe.conf
@@@ -1,0 -1,72 +1,77 @@@
+ # ooni-probe
+ #
+ # These are the global configuration parameters necessary to
+ # make ooni-probe work
+ [main]
+ reportdir = reports/
+ logfile = ooniprobe.log
+ assetdir = assets/
+ testdir = oonitests/
+
+ loglevel = DEBUG
+ consoleloglevel = DEBUG
+ proxyaddress = 127.0.0.1:9050
+
+ # The following configurations are for searching for PlanetLab
+ # nodes, adding them to a slice, and PlanetLab general API
+ # authentication:
+ pl_username = yourusername
+ pl_password = yourpassword
+
+ # These are configurations specific to the tests that should be
+ # run by ooni-probe
+ [tests]
+ run = dnstamper
+ ### DNS testing related config parameters
+
+ # This is the list of hostnames that must be looked up
+ dns_experiment = top-1m.txt
+
+ # This is the dns servers to be tested
+ dns_experiment_dns = dns_servers.txt
+
+ # This is the control known good DNS server
-dns_control_server = 8.8.8.8
++dns_control_server = 91.191.136.152
++
++# Specify whether the dnstamper test should attempt to remove
++# GeoIP-based false positives by doing a reverse DNS resolve
++# on positive results.
++dns_reverse_lookup = true
+
+ ### traceroute testing related config parameters
+
+ # This is the list of ips to traceroute to
+ traceroute = example_exp_list.txt
+
+ # This is the list of ports that should be used
+ # src_x,src_y,src_z|dst_x,dst_y,dst_z
+ traceroute_ports = 0,53,80,123,443|0,53,80,123,443
+
+ # The protocol to be used in the scan
+ traceroute_proto = UDP, TCP, ICMP
+
+ ### keyword injection related tests
+
+ # List of keywords
+ keywords = keywordlist.txt
+
+ # hosts
+ keywords_hosts = hostslist.txt
+
+ # Methods to be used for testing
+ keyword_method = http,telnet
+
+ ### Tor bridge testing
+
+ tor_bridges = bridgetests.txt
+ tor_bridges_timeout = 40
+
+ [report]
+ file = report.log
+ timestamp = true
+ #ssh = 127.0.0.1:22
+ #ssh_user = theusername
+ #ssh_password = thepassword
+ #ssh_keyfile = ~/.ssh/mykey_rsa
+ #ssh_rpath = ~/ooni-probe/
+ #tcp = "127.0.0.1:9088"
diff --cc ooni/oonitests/dnstamper.py
index 0000000,68be12d..498ba04
mode 000000,100644..100644
--- a/ooni/oonitests/dnstamper.py
+++ b/ooni/oonitests/dnstamper.py
@@@ -1,0 -1,70 +1,156 @@@
++# -*- coding: utf-8 -*-
++"""
++ dnstamper
++ *********
++
++ This test resolves DNS for a list of domain names, one per line, in the
++ file specified in the ooni-config under the setting "dns_experiment". If
++ the file is top-1m.txt, the test will be run using Amazon's list of top
++ one million domains. The experimental dns servers to query should
++ be specified one per line in assets/dns_servers.txt.
++
++ The test reports censorship if the cardinality of the intersection of
++ the query result set from the control server and the query result set
++ from the experimental server is zero, which is to say, if the two sets
++ have no matching results whatsoever.
++
++ NOTE: This test frequently results in false positives due to GeoIP-based
++ load balancing on major global sites such as google, facebook, and
++ youtube, etc.
++
++ :copyright: (c) 2012 Arturo Filastò, Isis Lovecruft
++ :license: see LICENSE for more details
++"""
++
++try:
++ from dns import resolver, reversename
++except:
++ print "Error: dnspython is not installed! (http://www.dnspython.org/)"
+ try:
- from dns import resolver
++ import gevent
+ except:
- print "Error dnspython is not installed! (http://www.dnspython.org/)"
-import gevent
++ print "Error: gevent is not installed! (http://www.gevent.org/)"
++
+ import os
++
+ import plugoo
+ from plugoo.assets import Asset
+ from plugoo.tests import Test
+
-
+ __plugoo__ = "DNST"
+ __desc__ = "DNS censorship detection test"
+
++class Top1MAsset(Asset):
++ """
++ Class for parsing top-1m.txt as an asset.
++ """
++ def __init__(self, file=None):
++ self = Asset.__init__(self, file)
++
++ def parse_line(self, line):
++ self = Asset.parse_line(self, line)
++ return line.split(',')[1].replace('\n','')
++
+ class DNSTAsset(Asset):
++ """
++ Creates DNS testing specific Assets.
++ """
+ def __init__(self, file=None):
- self = asset.__init__(self, file)
++ self = Asset.__init__(self, file)
+
+ class DNST(Test):
+ def lookup(self, hostname, ns):
++ """
++ Resolves a hostname through a DNS nameserver, ns, to the corresponding
++ IP address(es).
++ """
+ res = resolver.Resolver(configure=False)
+ res.nameservers = [ns]
+ answer = res.query(hostname)
+
+ ret = []
+
+ for data in answer:
+ ret.append(data.address)
+
+ return ret
+
++ def reverse_lookup(self, ip, ns):
++ """
++ Attempt to do a reverse DNS lookup to determine if the control and exp
++ sets from a positive result resolve to the same domain, in order to
++ remove false positives due to GeoIP load balancing.
++ """
++ res = resolver.Resolver(configure=False)
++ res.nameservers = [ns]
++ n = reversename.from_address(ip)
++ revn = res.query(n, "PTR").__iter__().next().to_text()[:-1]
++
++ return revn
++
+ def experiment(self, *a, **kw):
++ """
++ Compares the lookup() sets of the control and experiment groups.
++ """
+ # this is just a dirty hack
+ address = kw['data'][0]
+ ns = kw['data'][1]
+
+ config = self.config
++ ctrl_ns = config.tests.dns_control_server
+
+ print "ADDRESS: %s" % address
+ print "NAMESERVER: %s" % ns
+
+ exp = self.lookup(address, ns)
- control = self.lookup(address, config.tests.dns_control_server)
++ control = self.lookup(address, ctrl_ns)
++
++ result = []
+
+ if len(set(exp) & set(control)) > 0:
- print "%s : no tampering on %s" % (address, ns)
- return (address, ns, False)
++ print "Address %s has not tampered with on DNS server %s\n" % (address, ns)
++ result = (address, ns, exp, control, False)
++ return result
+ else:
- print "%s : possible tampering on %s (%s, %s)" % (address, ns, exp, control)
- return (address, ns, exp, control, True)
++ print "Address %s has possibly been tampered on %s:\nDNS resolution through %s yeilds:\n%s\nAlthough the control group DNS servers resolve to:\n%s" % (address, ns, ns, exp, control)
++ result = (address, ns, exp, control, True)
++
++ if config.tests.dns_reverse_lookup:
++
++ exprevn = [self.reverse_lookup(ip, ns) for ip in exp]
++ ctrlrevn = [self.reverse_lookup(ip, ctrl_ns)
++ for ip in control]
++
++ if len(set(exprevn) & set(ctrlrevn)) > 0:
++ print "Further testing has eliminated this as a false positive."
++ else:
++ print "Reverse DNS on the results returned by %s returned:\n%s\nWhich does not match the expected domainname:\n%s\n" % (ns, exprevn, ctrlrevn)
++ return result
++
++ else:
++ print "\n"
++ return result
+
+ def run(ooni):
- """Run the test
++ """
++ Run the test.
+ """
+ config = ooni.config
+ urls = []
+
- dns_experiment = DNSTAsset(os.path.join(config.main.assetdir, \
- config.tests.dns_experiment))
- dns_experiment_dns = DNSTAsset(os.path.join(config.main.assetdir, \
++ if (config.tests.dns_experiment == "top-1m.txt"):
++ dns_experiment = Top1MAsset(os.path.join(config.main.assetdir,
++ config.tests.dns_experiment))
++ else:
++ dns_experiment = DNSTAsset(os.path.join(config.main.assetdir,
++ config.tests.dns_experiment))
++ dns_experiment_dns = DNSTAsset(os.path.join(config.main.assetdir,
+ config.tests.dns_experiment_dns))
+
+ assets = [dns_experiment, dns_experiment_dns]
+
+ dnstest = DNST(ooni)
- ooni.logger.info("starting test")
- dnstest.run(assets)
- ooni.logger.info("finished")
-
++ ooni.logger.info("Beginning dnstamper test...")
++ dnstest.run(assets, {'index': 1})
++ ooni.logger.info("Dnstamper test completed!")
+
diff --cc ooni/oonitests/httphost.py
index 0000000,6446e1f..25adff2
mode 000000,100644..100644
--- a/ooni/oonitests/httphost.py
+++ b/ooni/oonitests/httphost.py
@@@ -1,0 -1,132 +1,135 @@@
+ """
+ HTTP Host based filtering
+ *************************
+
+ This test detect HTTP Host field
+ based filtering.
+ It is used to detect censorship on
+ performed with Web Guard (used by
+ T-Mobile US).
+ """
+ import os
+ from datetime import datetime
+ from gevent import monkey
+
+ import urllib2
+ import httplib
+ # WARNING! Using gevent's socket
+ # introduces the 0x20 DNS "feature".
+ # This will result is weird DNS requests
+ # appearing on the wire.
+ monkey.patch_socket()
+
-from BeautifulSoup import BeautifulSoup
++try:
++ from BeautifulSoup import BeautifulSoup
++except:
++ print "BeautifulSoup-3.2.1 is missing. Please see https://crate.io/packages/BeautifulSoup/"
+
+ from plugoo.assets import Asset
+ from plugoo.tests import Test
+
+ __plugoo__ = "HTTP Host"
+ __desc__ = "This detects HTTP Host field based filtering"
+
+ class HTTPHostAsset(Asset):
+ """
+ This is the asset that should be used by the Test. It will
+ contain all the code responsible for parsing the asset file
+ and should be passed on instantiation to the test.
+ """
+ def __init__(self, file=None):
+ self = Asset.__init__(self, file)
+
+ def parse_line(self, line):
+ return line.split(',')[1].replace('\n','')
+
+ class HTTPHost(Test):
+ """
+ The main Test class
+ """
+
+ def check_response(self, response):
+ soup = BeautifulSoup(response)
+ if soup.head.title.string == "Content Filtered":
+ # Response indicates censorship
+ return True
+ else:
+ # Response does not indicate censorship
+ return False
+
+
+ def is_censored(self, response):
+ if response:
+ soup = BeautifulSoup(response)
+ censored = self.check_response(response)
+ else:
+ censored = "unreachable"
+ return censored
+
+ def urllib2_test(self, control_server, host):
+ req = urllib2.Request(control_server)
+ req.add_header('Host', host)
+ try:
+ r = urllib2.urlopen(req)
+ response = r.read()
+ censored = self.is_censored(response)
+ except Exception, e:
+ censored = "Error! %s" % e
+
+ return censored
+
+ def httplib_test(self, control_server, host):
+ try:
+ conn = httplib.HTTPConnection(control_server)
+ conn.putrequest("GET", "", skip_host=True, skip_accept_encoding=True)
+ conn.putheader("Host", host)
+ conn.putheader("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6")
+ conn.endheaders()
+ r = conn.getresponse()
+ response = r.read()
+ censored = self.is_censored(response)
+ except Exception, e:
+ censored = "Error! %s" % e
+
+ return censored
+
+
+ def experiment(self, *a, **kw):
+ """
+ Try to connect to the control server with
+ the specified host field.
+ """
+ host = kw['data']
+ control_server = kw['control_server']
+ self.logger.info("Testing %s (%s)" % (host, control_server))
+
+ #censored = self.urllib2_test(control_server, host)
+ censored = self.httplib_test(control_server, host)
+
+ self.logger.info("%s: %s" % (host, censored))
+ return {'Time': datetime.now(),
+ 'Host': host,
+ 'Censored': censored}
+
+
+ def run(ooni):
+ """
+ This is the function that will be called by OONI
+ and it is responsible for instantiating and passing
+ the arguments to the Test class.
+ """
+ config = ooni.config
+
+ # This the assets array to be passed to the run function of
+ # the test
+ assets = [HTTPHostAsset(os.path.join(config.main.assetdir, \
+ "top-1m.csv"))]
+
+ # Instantiate the Test
+ thetest = HTTPHost(ooni)
+ ooni.logger.info("starting HTTP Host Test...")
+ # Run the test with argument assets
+ thetest.run(assets, {'index': 5825, 'control_server': '195.85.254.203:8080'})
+ ooni.logger.info("finished.")
+
+
diff --cc ooni/plugoo/reports.py
index 0000000,c099456..c3e7c23
mode 000000,100644..100644
--- a/ooni/plugoo/reports.py
+++ b/ooni/plugoo/reports.py
@@@ -1,0 -1,175 +1,175 @@@
+ import os
+ from datetime import datetime
+ import yaml
+
+ import logging
+ import itertools
+ import gevent
+
+ class Report:
+ """This is the ooni-probe reporting mechanism. It allows
+ reporting to multiple destinations and file formats.
+
+ :scp the string of <host>:<port> of an ssh server
+
+ :yaml the filename of a the yaml file to write
+
+ :file the filename of a simple txt file to write
+
+ :tcp the <host>:<port> of a TCP server that will just listen for
+ inbound connection and accept a stream of data (think of it
+ as a `nc -l -p <port> > filename.txt`)
+ """
+ def __init__(self, ooni,
+ scp="127.0.0.1:22",
+ file="test.report",
+ tcp="127.0.0.1:9000"):
+
+ self.file = file
+ self.tcp = tcp
+ self.scp = scp
+ self.config = ooni.config.report
+ self.logger = ooni.logger
+
+ if self.config.timestamp:
+ tmp = self.file.split('.')
+ self.file = '.'.join(tmp[:-1]) + "-" + \
+ datetime.now().isoformat('-') + '.' + \
+ tmp[-1]
+ print self.file
+
+ try:
+ import paramiko
+ except:
+ self.scp = None
- self.logger.warn("Could not import paramiko. SCP will not be disabled")
++ self.logger.warn("Could not import paramiko. SCP will be disabled")
+
+ def __call__(self, data):
+ """
+ This should be invoked every time you wish to write some
+ data to the reporting system
+ """
+ #print "Writing report(s)"
+ #dump = '--- \n'
+ dump = yaml.dump([data])
+ #dump += yaml.dump(data)
+ reports = []
+
+ if self.file:
+ reports.append("file")
+
+ if self.tcp:
+ reports.append("tcp")
+
+ if self.scp:
+ reports.append("scp")
+
+ jobs = [gevent.spawn(self.send_report, *(dump, report)) for report in reports]
+ gevent.joinall(jobs)
+ ret = []
+ for job in jobs:
+ #print job.value
+ ret.append(job.value)
+ return ret
+
+ def file_report(self, data, file=None, mode='a+'):
+ """
+ This reports to a file in YAML format
+ """
+ if not file:
+ file = self.file
+ try:
+ f = open(file, mode)
+ f.write(data)
+ except Exception, e:
+ raise e
+ finally:
+ f.close()
+
+
+ def tcp_report(self, data):
+ """
+ This connect to the specified tcp server
+ and writes the data passed as argument.
+ """
+ host, port = self.tcp.split(":")
+ tcp = socket.getprotobyname('tcp')
+ send_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM, tcp)
+ try:
+ send_socket.connect((host, int(port)))
+ send_socket.send(data)
+
+ except Exception, e:
+ raise e
+
+ finally:
+ send_socket.close()
+
+
+ def scp_report(self, data, rfile=None, mode='a+'):
+ """
+ Push data to the remote ssh server.
+
+ :rfile the remote filename to write
+ :data the raw data content that should be written
+ :mode in what mode the file should be created
+ """
+ if not rfile:
+ rfile = self.file
+ host, port = self.scp.split(":")
+ transport = paramiko.Transport((host, port))
+
+ # The remote path of the remote file to write
+ rfpath = os.path.join(self.config.ssh_rpath, rfile)
+
+ try:
+ username = self.config.ssh_username
+ except:
+ raise "No username provided"
+
+ # Load the local known host key file
+ transport.load_host_keys(os.path.expanduser("~/.ssh/known_hosts"))
+
+ # We prefer to use an ssh keyfile fo authentication
+ if self.config.ssh_keyfile:
+ keyfile = os.path.expanduser(self.config.ssh_keyfile)
+ key = paramiko.RSAKey.from_private_key_file(keylocfile)
+ try:
+ transport.connect(username=username, pkey=key)
+ except Exception, e:
+ raise e
+
+ # If not even a password is fine
+ elif self.config.ssh_password:
+ try:
+ transport.connect(username=username, password=self.config.ssh_password)
+ except Exception, e:
+ raise e
+
+ # ... but no authentication, that is madness!
+ else:
+ raise "No key or password provided for ssh"
+
+ sftp = paramiko.SFTPClient.from_transport(transport)
+ try:
+ sftp = ssh.open_sftp()
+ remote_file = sftp.file(rfile, mode)
+ remote_file.set_pipelined(True)
+ remote_file.write(data)
+
+ except Exception, e:
+ raise e
+ sftp.close()
+ transport.close()
+
+
+ def send_report(self, data, type):
+ """
+ This sends the report using the
+ specified type.
+ """
+ #print "Reporting %s to %s" % (data, type)
+ self.logger.info("Reporting to %s" % type)
+ getattr(self, type+"_report").__call__(data)
+
+
More information about the tor-commits
mailing list