commit baab66b10eccb5a3bc40addc84de0e45f2897497 Merge: abfcf7c 59f1b4e Author: Arturo Filastò hellais@torproject.org Date: Tue May 29 03:23:01 2012 +0200

Merge branch 'twisted' of github.com:hellais/ooni-probe

HACKING | 36 +- assets/example.txt | 2 - assets/tcpscan.txt | 1 - config.py | 53 - docs/writing_tests.txt | 12 + install.txt | 4 - lib/traceroute.py | 32 - lib/txtraceroute.py | 389 - logo.py | 214 - nodes.conf | 53 - old-to-be-ported-code/AUTHORS | 3 + old-to-be-ported-code/CHANGES.yaml | 5 + old-to-be-ported-code/HACKING | 21 + old-to-be-ported-code/INSTALL | 8 + old-to-be-ported-code/LICENSE | 28 + old-to-be-ported-code/MANIFEST.in | 6 + old-to-be-ported-code/Makefile | 182 + old-to-be-ported-code/README.API | 29 + old-to-be-ported-code/README.examples | 10 + old-to-be-ported-code/README.rst | 40 + old-to-be-ported-code/RFA.md | 101 + old-to-be-ported-code/TODO | 418 + old-to-be-ported-code/TODO.plgoons | 79 + old-to-be-ported-code/bin/ooni-probe | 10 + old-to-be-ported-code/keyword-lists/Makefile | 3 + old-to-be-ported-code/keyword-lists/keyfile | 256 + old-to-be-ported-code/list/list.txt | 2 + old-to-be-ported-code/ooni-probe.diff | 358 + old-to-be-ported-code/ooni/.DS_Store | Bin 0 -> 15364 bytes old-to-be-ported-code/ooni/__init__.py | 12 + old-to-be-ported-code/ooni/captive_portal.py | 62 + old-to-be-ported-code/ooni/command.py | 250 + old-to-be-ported-code/ooni/common.py | 139 + old-to-be-ported-code/ooni/dns_cc_check.py | 48 + old-to-be-ported-code/ooni/dns_poisoning.py | 43 + old-to-be-ported-code/ooni/dnsooni.py | 356 + old-to-be-ported-code/ooni/empty.txt | 9 + old-to-be-ported-code/ooni/helpers.py | 38 + old-to-be-ported-code/ooni/http.py | 306 + old-to-be-ported-code/ooni/input.py | 33 + old-to-be-ported-code/ooni/namecheck.py | 39 + old-to-be-ported-code/ooni/output.py | 21 + .../ooni/plugins/captiveportal_plgoo.py | 55 + .../ooni/plugins/dnstest_plgoo.py | 84 + old-to-be-ported-code/ooni/plugins/http_plgoo.py | 70 + old-to-be-ported-code/ooni/plugins/marco_plgoo.py | 377 + .../ooni/plugins/netalyzr_plgoo.py | 30 + .../ooni/plugins/old_stuff_to_convert/README | 47 + .../plugins/old_stuff_to_convert/README.automation | 11 + .../ooni/plugins/old_stuff_to_convert/TODO | 6 + .../plugins/old_stuff_to_convert/cached-consensus |11724 ++++++++++++++++++++ .../plugins/old_stuff_to_convert/connectback.sh | 56 + .../plugins/old_stuff_to_convert/dirconntest.sh | 52 + .../plugins/old_stuff_to_convert/dns-checker.sh | 7 + .../old_stuff_to_convert/generic-host-test.sh | 33 + .../ooni/plugins/old_stuff_to_convert/host-prep.sh | 20 + .../plugins/old_stuff_to_convert/install-probe.sh | 27 + .../ooni/plugins/old_stuff_to_convert/run-tests.sh | 11 + .../plugins/old_stuff_to_convert/twitter-test.sh | 33 + old-to-be-ported-code/ooni/plugins/proxy_plgoo.py | 69 + .../ooni/plugins/simple_dns_plgoo.py | 35 + old-to-be-ported-code/ooni/plugins/skel_plgoo.py | 17 + old-to-be-ported-code/ooni/plugins/skel_plgoo.yaml | 33 + old-to-be-ported-code/ooni/plugins/tcpcon_plgoo.py | 278 + old-to-be-ported-code/ooni/plugins/tor.py | 80 + old-to-be-ported-code/ooni/plugins/torrc | 9 + old-to-be-ported-code/ooni/plugooni.py | 106 + old-to-be-ported-code/ooni/report.py | 22 + old-to-be-ported-code/ooni/transparenthttp.py | 41 + old-to-be-ported-code/ooni/yamlooni.py | 40 + old-to-be-ported-code/proxy-lists/Makefile | 11 + old-to-be-ported-code/proxy-lists/README | 9 + old-to-be-ported-code/proxy-lists/italy-dns.txt | 46 + .../proxy-lists/italy-http-block-pages-notes.txt | 62 + .../proxy-lists/italy-http-block-pages.txt | 5 + .../proxy-lists/parse-trusted-xff.sh | 16 + old-to-be-ported-code/proxy-lists/trusted-xff.html | 7789 +++++++++++++ old-to-be-ported-code/reports/marco-1.yamlooni | 86 + old-to-be-ported-code/reports/marco-2.yamlooni | 60 + old-to-be-ported-code/reports/marco.log | 4672 ++++++++ old-to-be-ported-code/reports/marco.yamlooni | 241 + old-to-be-ported-code/reports/marco_certs-1-2.out | 307 + old-to-be-ported-code/reports/marco_certs-1.out | 5850 ++++++++++ old-to-be-ported-code/reports/marco_certs-2.out | 1182 ++ old-to-be-ported-code/reports/marco_certs.out | 1900 ++++ old-to-be-ported-code/setup.py | 10 + old-to-be-ported-code/spec/proxooni-spec.txt | 65 + old-to-be-ported-code/test-lists/Makefile | 7 + old-to-be-ported-code/tests/test_import.py | 10 + old-to-be-ported-code/third-party/Makefile | 3 + old-to-be-ported-code/third-party/README | 14 + old/AUTHORS | 3 - old/CHANGES.yaml | 5 - old/HACKING | 21 - old/INSTALL | 8 - old/LICENSE | 28 - old/MANIFEST.in | 6 - old/Makefile | 182 - old/README.API | 29 - old/README.examples | 10 - old/README.rst | 40 - old/RFA.md | 101 - old/TODO | 418 - old/TODO.plgoons | 79 - old/bin/ooni-probe | 10 - old/keyword-lists/Makefile | 3 - old/keyword-lists/keyfile | 256 - old/list/list.txt | 2 - old/ooni-probe.diff | 358 - old/ooni/.DS_Store | Bin 15364 -> 0 bytes old/ooni/__init__.py | 12 - old/ooni/captive_portal.py | 62 - old/ooni/command.py | 250 - old/ooni/common.py | 139 - old/ooni/dns_cc_check.py | 48 - old/ooni/dns_poisoning.py | 43 - old/ooni/dnsooni.py | 356 - old/ooni/empty.txt | 9 - old/ooni/helpers.py | 38 - old/ooni/http.py | 306 - old/ooni/input.py | 33 - old/ooni/namecheck.py | 39 - old/ooni/output.py | 21 - old/ooni/plugins/captiveportal_plgoo.py | 55 - old/ooni/plugins/dnstest_plgoo.py | 84 - old/ooni/plugins/http_plgoo.py | 70 - old/ooni/plugins/marco_plgoo.py | 377 - old/ooni/plugins/netalyzr_plgoo.py | 30 - old/ooni/plugins/old_stuff_to_convert/README | 47 - .../plugins/old_stuff_to_convert/README.automation | 11 - old/ooni/plugins/old_stuff_to_convert/TODO | 6 - .../plugins/old_stuff_to_convert/cached-consensus |11724 -------------------- .../plugins/old_stuff_to_convert/connectback.sh | 56 - .../plugins/old_stuff_to_convert/dirconntest.sh | 52 - .../plugins/old_stuff_to_convert/dns-checker.sh | 7 - .../old_stuff_to_convert/generic-host-test.sh | 33 - old/ooni/plugins/old_stuff_to_convert/host-prep.sh | 20 - .../plugins/old_stuff_to_convert/install-probe.sh | 27 - old/ooni/plugins/old_stuff_to_convert/run-tests.sh | 11 - .../plugins/old_stuff_to_convert/twitter-test.sh | 33 - old/ooni/plugins/proxy_plgoo.py | 69 - old/ooni/plugins/simple_dns_plgoo.py | 35 - old/ooni/plugins/skel_plgoo.py | 17 - old/ooni/plugins/skel_plgoo.yaml | 33 - old/ooni/plugins/tcpcon_plgoo.py | 278 - old/ooni/plugins/tor.py | 80 - old/ooni/plugins/torrc | 9 - old/ooni/plugooni.py | 106 - old/ooni/report.py | 22 - old/ooni/transparenthttp.py | 41 - old/ooni/yamlooni.py | 40 - old/proxy-lists/Makefile | 11 - old/proxy-lists/README | 9 - old/proxy-lists/italy-dns.txt | 46 - old/proxy-lists/italy-http-block-pages-notes.txt | 62 - old/proxy-lists/italy-http-block-pages.txt | 5 - old/proxy-lists/parse-trusted-xff.sh | 16 - old/proxy-lists/trusted-xff.html | 7789 ------------- old/reports/marco-1.yamlooni | 86 - old/reports/marco-2.yamlooni | 60 - old/reports/marco.log | 4672 -------- old/reports/marco.yamlooni | 241 - old/reports/marco_certs-1-2.out | 307 - old/reports/marco_certs-1.out | 5850 ---------- old/reports/marco_certs-2.out | 1182 -- old/reports/marco_certs.out | 1900 ---- old/setup.py | 10 - old/spec/proxooni-spec.txt | 65 - old/test-lists/Makefile | 7 - old/tests/test_import.py | 10 - old/third-party/Makefile | 3 - old/third-party/README | 14 - ooni-probe.conf | 77 - ooni/assets/bridgetests.txt | 3 + ooni/assets/example.txt | 2 + ooni/assets/tcpscan.txt | 1 + ooni/config.py | 53 + ooni/lib/Makefile | 14 + ooni/logo.py | 214 + ooni/nodes.conf | 53 + ooni/ooni-probe.conf | 77 + ooni/oonicli.py | 139 + ooni/ooniprobe.log | 2 + ooni/ooniprobe.py | 191 + ooni/oonitests/bridget.py | 373 + ooni/oonitests/dnstamper.py | 156 + ooni/oonitests/httphost.py | 135 + ooni/oonitests/squid.py | 113 + ooni/oonitests/tcpscan.py | 86 + ooni/oonitests/template.py | 65 + ooni/oonitests/traceroute.py | 110 + ooni/plugins/.bridget.py.swp | Bin 0 -> 12288 bytes ooni/plugins/.dnstamper.py.swp | Bin 0 -> 12288 bytes ooni/plugins/__init__.py | 3 + ooni/plugins/bridget.py | 36 + ooni/plugins/dnstamper.py | 22 + ooni/plugins/dropin.cache | 76 + ooni/plugins/skel.py | 21 + ooni/plugoo/__init__.py | 80 + ooni/plugoo/assets.py | 56 + ooni/plugoo/interface.py | 2 + ooni/plugoo/nodes.py | 180 + ooni/plugoo/reports.py | 175 + ooni/plugoo/tests.py | 229 + ooni/plugoo/work.py | 162 + ooni/skel.py | 10 + ooni/tests/worker_test.py | 27 + ooni/utils.py | 146 + oonib/dnsbackend.py | 16 + oonib/httpbackend.py | 74 + oonib/oonibackend.py | 34 + oonicli.py | 142 - ooniprobe.py | 191 - plugins/__init__.py | 3 - plugins/dropin.cache | 48 - plugins/skel.py | 20 - plugoo/__init__.py | 80 - plugoo/assets.py | 56 - plugoo/interface.py | 2 - plugoo/nodes.py | 180 - plugoo/reports.py | 175 - plugoo/tests.py | 213 - plugoo/work.py | 162 - skel.py | 10 - tests/bridget.py | 378 - tests/dnstamper.py | 156 - tests/httphost.py | 135 - tests/squid.py | 113 - tests/tcpscan.py | 86 - tests/template.py | 65 - tests/traceroute.py | 110 - unittest/tests.py | 27 - utils.py | 146 - 233 files changed, 41804 insertions(+), 41969 deletions(-)

diff --cc ooni/ooni-probe.conf index 0000000,336af99..d95e410 mode 000000,100644..100644 --- a/ooni/ooni-probe.conf +++ b/ooni/ooni-probe.conf @@@ -1,0 -1,72 +1,77 @@@ + # ooni-probe + # + # These are the global configuration parameters necessary to + # make ooni-probe work + [main] + reportdir = reports/ + logfile = ooniprobe.log + assetdir = assets/ + testdir = oonitests/ + + loglevel = DEBUG + consoleloglevel = DEBUG + proxyaddress = 127.0.0.1:9050 + + # The following configurations are for searching for PlanetLab + # nodes, adding them to a slice, and PlanetLab general API + # authentication: + pl_username = yourusername + pl_password = yourpassword + + # These are configurations specific to the tests that should be + # run by ooni-probe + [tests] + run = dnstamper + ### DNS testing related config parameters + + # This is the list of hostnames that must be looked up + dns_experiment = top-1m.txt + + # This is the dns servers to be tested + dns_experiment_dns = dns_servers.txt + + # This is the control known good DNS server -dns_control_server = 8.8.8.8 ++dns_control_server = 91.191.136.152 ++ ++# Specify whether the dnstamper test should attempt to remove ++# GeoIP-based false positives by doing a reverse DNS resolve ++# on positive results. ++dns_reverse_lookup = true + + ### traceroute testing related config parameters + + # This is the list of ips to traceroute to + traceroute = example_exp_list.txt + + # This is the list of ports that should be used + # src_x,src_y,src_z|dst_x,dst_y,dst_z + traceroute_ports = 0,53,80,123,443|0,53,80,123,443 + + # The protocol to be used in the scan + traceroute_proto = UDP, TCP, ICMP + + ### keyword injection related tests + + # List of keywords + keywords = keywordlist.txt + + # hosts + keywords_hosts = hostslist.txt + + # Methods to be used for testing + keyword_method = http,telnet + + ### Tor bridge testing + + tor_bridges = bridgetests.txt + tor_bridges_timeout = 40 + + [report] + file = report.log + timestamp = true + #ssh = 127.0.0.1:22 + #ssh_user = theusername + #ssh_password = thepassword + #ssh_keyfile = ~/.ssh/mykey_rsa + #ssh_rpath = ~/ooni-probe/ + #tcp = "127.0.0.1:9088" diff --cc ooni/oonitests/dnstamper.py index 0000000,68be12d..498ba04 mode 000000,100644..100644 --- a/ooni/oonitests/dnstamper.py +++ b/ooni/oonitests/dnstamper.py @@@ -1,0 -1,70 +1,156 @@@ ++# -*- coding: utf-8 -*- ++""" ++ dnstamper ++ ********* ++ ++ This test resolves DNS for a list of domain names, one per line, in the ++ file specified in the ooni-config under the setting "dns_experiment". If ++ the file is top-1m.txt, the test will be run using Amazon's list of top ++ one million domains. The experimental dns servers to query should ++ be specified one per line in assets/dns_servers.txt. ++ ++ The test reports censorship if the cardinality of the intersection of ++ the query result set from the control server and the query result set ++ from the experimental server is zero, which is to say, if the two sets ++ have no matching results whatsoever. ++ ++ NOTE: This test frequently results in false positives due to GeoIP-based ++ load balancing on major global sites such as google, facebook, and ++ youtube, etc. ++ ++ :copyright: (c) 2012 Arturo Filastò, Isis Lovecruft ++ :license: see LICENSE for more details ++""" ++ ++try: ++ from dns import resolver, reversename ++except: ++ print "Error: dnspython is not installed! (http://www.dnspython.org/)" + try: - from dns import resolver ++ import gevent + except: - print "Error dnspython is not installed! (http://www.dnspython.org/)" -import gevent ++ print "Error: gevent is not installed! (http://www.gevent.org/)" ++ + import os ++ + import plugoo + from plugoo.assets import Asset + from plugoo.tests import Test + - + __plugoo__ = "DNST" + __desc__ = "DNS censorship detection test" + ++class Top1MAsset(Asset): ++ """ ++ Class for parsing top-1m.txt as an asset. ++ """ ++ def __init__(self, file=None): ++ self = Asset.__init__(self, file) ++ ++ def parse_line(self, line): ++ self = Asset.parse_line(self, line) ++ return line.split(',')[1].replace('\n','') ++ + class DNSTAsset(Asset): ++ """ ++ Creates DNS testing specific Assets. ++ """ + def __init__(self, file=None): - self = asset.__init__(self, file) ++ self = Asset.__init__(self, file) + + class DNST(Test): + def lookup(self, hostname, ns): ++ """ ++ Resolves a hostname through a DNS nameserver, ns, to the corresponding ++ IP address(es). ++ """ + res = resolver.Resolver(configure=False) + res.nameservers = [ns] + answer = res.query(hostname) + + ret = [] + + for data in answer: + ret.append(data.address) + + return ret + ++ def reverse_lookup(self, ip, ns): ++ """ ++ Attempt to do a reverse DNS lookup to determine if the control and exp ++ sets from a positive result resolve to the same domain, in order to ++ remove false positives due to GeoIP load balancing. ++ """ ++ res = resolver.Resolver(configure=False) ++ res.nameservers = [ns] ++ n = reversename.from_address(ip) ++ revn = res.query(n, "PTR").__iter__().next().to_text()[:-1] ++ ++ return revn ++ + def experiment(self, *a, **kw): ++ """ ++ Compares the lookup() sets of the control and experiment groups. ++ """ + # this is just a dirty hack + address = kw['data'][0] + ns = kw['data'][1] + + config = self.config ++ ctrl_ns = config.tests.dns_control_server + + print "ADDRESS: %s" % address + print "NAMESERVER: %s" % ns + + exp = self.lookup(address, ns) - control = self.lookup(address, config.tests.dns_control_server) ++ control = self.lookup(address, ctrl_ns) ++ ++ result = [] + + if len(set(exp) & set(control)) > 0: - print "%s : no tampering on %s" % (address, ns) - return (address, ns, False) ++ print "Address %s has not tampered with on DNS server %s\n" % (address, ns) ++ result = (address, ns, exp, control, False) ++ return result + else: - print "%s : possible tampering on %s (%s, %s)" % (address, ns, exp, control) - return (address, ns, exp, control, True) ++ print "Address %s has possibly been tampered on %s:\nDNS resolution through %s yeilds:\n%s\nAlthough the control group DNS servers resolve to:\n%s" % (address, ns, ns, exp, control) ++ result = (address, ns, exp, control, True) ++ ++ if config.tests.dns_reverse_lookup: ++ ++ exprevn = [self.reverse_lookup(ip, ns) for ip in exp] ++ ctrlrevn = [self.reverse_lookup(ip, ctrl_ns) ++ for ip in control] ++ ++ if len(set(exprevn) & set(ctrlrevn)) > 0: ++ print "Further testing has eliminated this as a false positive." ++ else: ++ print "Reverse DNS on the results returned by %s returned:\n%s\nWhich does not match the expected domainname:\n%s\n" % (ns, exprevn, ctrlrevn) ++ return result ++ ++ else: ++ print "\n" ++ return result + + def run(ooni): - """Run the test ++ """ ++ Run the test. + """ + config = ooni.config + urls = [] + - dns_experiment = DNSTAsset(os.path.join(config.main.assetdir, \ - config.tests.dns_experiment)) - dns_experiment_dns = DNSTAsset(os.path.join(config.main.assetdir, \ ++ if (config.tests.dns_experiment == "top-1m.txt"): ++ dns_experiment = Top1MAsset(os.path.join(config.main.assetdir, ++ config.tests.dns_experiment)) ++ else: ++ dns_experiment = DNSTAsset(os.path.join(config.main.assetdir, ++ config.tests.dns_experiment)) ++ dns_experiment_dns = DNSTAsset(os.path.join(config.main.assetdir, + config.tests.dns_experiment_dns)) + + assets = [dns_experiment, dns_experiment_dns] + + dnstest = DNST(ooni) - ooni.logger.info("starting test") - dnstest.run(assets) - ooni.logger.info("finished") - ++ ooni.logger.info("Beginning dnstamper test...") ++ dnstest.run(assets, {'index': 1}) ++ ooni.logger.info("Dnstamper test completed!") + diff --cc ooni/oonitests/httphost.py index 0000000,6446e1f..25adff2 mode 000000,100644..100644 --- a/ooni/oonitests/httphost.py +++ b/ooni/oonitests/httphost.py @@@ -1,0 -1,132 +1,135 @@@ + """ + HTTP Host based filtering + ************************* + + This test detect HTTP Host field + based filtering. + It is used to detect censorship on + performed with Web Guard (used by + T-Mobile US). + """ + import os + from datetime import datetime + from gevent import monkey + + import urllib2 + import httplib + # WARNING! Using gevent's socket + # introduces the 0x20 DNS "feature". + # This will result is weird DNS requests + # appearing on the wire. + monkey.patch_socket() + -from BeautifulSoup import BeautifulSoup ++try: ++ from BeautifulSoup import BeautifulSoup ++except: ++ print "BeautifulSoup-3.2.1 is missing. Please see https://crate.io/packages/BeautifulSoup/" + + from plugoo.assets import Asset + from plugoo.tests import Test + + __plugoo__ = "HTTP Host" + __desc__ = "This detects HTTP Host field based filtering" + + class HTTPHostAsset(Asset): + """ + This is the asset that should be used by the Test. It will + contain all the code responsible for parsing the asset file + and should be passed on instantiation to the test. + """ + def __init__(self, file=None): + self = Asset.__init__(self, file) + + def parse_line(self, line): + return line.split(',')[1].replace('\n','') + + class HTTPHost(Test): + """ + The main Test class + """ + + def check_response(self, response): + soup = BeautifulSoup(response) + if soup.head.title.string == "Content Filtered": + # Response indicates censorship + return True + else: + # Response does not indicate censorship + return False + + + def is_censored(self, response): + if response: + soup = BeautifulSoup(response) + censored = self.check_response(response) + else: + censored = "unreachable" + return censored + + def urllib2_test(self, control_server, host): + req = urllib2.Request(control_server) + req.add_header('Host', host) + try: + r = urllib2.urlopen(req) + response = r.read() + censored = self.is_censored(response) + except Exception, e: + censored = "Error! %s" % e + + return censored + + def httplib_test(self, control_server, host): + try: + conn = httplib.HTTPConnection(control_server) + conn.putrequest("GET", "", skip_host=True, skip_accept_encoding=True) + conn.putheader("Host", host) + conn.putheader("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6") + conn.endheaders() + r = conn.getresponse() + response = r.read() + censored = self.is_censored(response) + except Exception, e: + censored = "Error! %s" % e + + return censored + + + def experiment(self, *a, **kw): + """ + Try to connect to the control server with + the specified host field. + """ + host = kw['data'] + control_server = kw['control_server'] + self.logger.info("Testing %s (%s)" % (host, control_server)) + + #censored = self.urllib2_test(control_server, host) + censored = self.httplib_test(control_server, host) + + self.logger.info("%s: %s" % (host, censored)) + return {'Time': datetime.now(), + 'Host': host, + 'Censored': censored} + + + def run(ooni): + """ + This is the function that will be called by OONI + and it is responsible for instantiating and passing + the arguments to the Test class. + """ + config = ooni.config + + # This the assets array to be passed to the run function of + # the test + assets = [HTTPHostAsset(os.path.join(config.main.assetdir, \ + "top-1m.csv"))] + + # Instantiate the Test + thetest = HTTPHost(ooni) + ooni.logger.info("starting HTTP Host Test...") + # Run the test with argument assets + thetest.run(assets, {'index': 5825, 'control_server': '195.85.254.203:8080'}) + ooni.logger.info("finished.") + + diff --cc ooni/plugoo/reports.py index 0000000,c099456..c3e7c23 mode 000000,100644..100644 --- a/ooni/plugoo/reports.py +++ b/ooni/plugoo/reports.py @@@ -1,0 -1,175 +1,175 @@@ + import os + from datetime import datetime + import yaml + + import logging + import itertools + import gevent + + class Report: + """This is the ooni-probe reporting mechanism. It allows + reporting to multiple destinations and file formats. + + :scp the string of <host>:<port> of an ssh server + + :yaml the filename of a the yaml file to write + + :file the filename of a simple txt file to write + + :tcp the <host>:<port> of a TCP server that will just listen for + inbound connection and accept a stream of data (think of it + as a `nc -l -p <port> > filename.txt`) + """ + def __init__(self, ooni, + scp="127.0.0.1:22", + file="test.report", + tcp="127.0.0.1:9000"): + + self.file = file + self.tcp = tcp + self.scp = scp + self.config = ooni.config.report + self.logger = ooni.logger + + if self.config.timestamp: + tmp = self.file.split('.') + self.file = '.'.join(tmp[:-1]) + "-" + \ + datetime.now().isoformat('-') + '.' + \ + tmp[-1] + print self.file + + try: + import paramiko + except: + self.scp = None - self.logger.warn("Could not import paramiko. SCP will not be disabled") ++ self.logger.warn("Could not import paramiko. SCP will be disabled") + + def __call__(self, data): + """ + This should be invoked every time you wish to write some + data to the reporting system + """ + #print "Writing report(s)" + #dump = '--- \n' + dump = yaml.dump([data]) + #dump += yaml.dump(data) + reports = [] + + if self.file: + reports.append("file") + + if self.tcp: + reports.append("tcp") + + if self.scp: + reports.append("scp") + + jobs = [gevent.spawn(self.send_report, *(dump, report)) for report in reports] + gevent.joinall(jobs) + ret = [] + for job in jobs: + #print job.value + ret.append(job.value) + return ret + + def file_report(self, data, file=None, mode='a+'): + """ + This reports to a file in YAML format + """ + if not file: + file = self.file + try: + f = open(file, mode) + f.write(data) + except Exception, e: + raise e + finally: + f.close() + + + def tcp_report(self, data): + """ + This connect to the specified tcp server + and writes the data passed as argument. + """ + host, port = self.tcp.split(":") + tcp = socket.getprotobyname('tcp') + send_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM, tcp) + try: + send_socket.connect((host, int(port))) + send_socket.send(data) + + except Exception, e: + raise e + + finally: + send_socket.close() + + + def scp_report(self, data, rfile=None, mode='a+'): + """ + Push data to the remote ssh server. + + :rfile the remote filename to write + :data the raw data content that should be written + :mode in what mode the file should be created + """ + if not rfile: + rfile = self.file + host, port = self.scp.split(":") + transport = paramiko.Transport((host, port)) + + # The remote path of the remote file to write + rfpath = os.path.join(self.config.ssh_rpath, rfile) + + try: + username = self.config.ssh_username + except: + raise "No username provided" + + # Load the local known host key file + transport.load_host_keys(os.path.expanduser("~/.ssh/known_hosts")) + + # We prefer to use an ssh keyfile fo authentication + if self.config.ssh_keyfile: + keyfile = os.path.expanduser(self.config.ssh_keyfile) + key = paramiko.RSAKey.from_private_key_file(keylocfile) + try: + transport.connect(username=username, pkey=key) + except Exception, e: + raise e + + # If not even a password is fine + elif self.config.ssh_password: + try: + transport.connect(username=username, password=self.config.ssh_password) + except Exception, e: + raise e + + # ... but no authentication, that is madness! + else: + raise "No key or password provided for ssh" + + sftp = paramiko.SFTPClient.from_transport(transport) + try: + sftp = ssh.open_sftp() + remote_file = sftp.file(rfile, mode) + remote_file.set_pipelined(True) + remote_file.write(data) + + except Exception, e: + raise e + sftp.close() + transport.close() + + + def send_report(self, data, type): + """ + This sends the report using the + specified type. + """ + #print "Reporting %s to %s" % (data, type) + self.logger.info("Reporting to %s" % type) + getattr(self, type+"_report").__call__(data) + +