commit d5fdae906d37f86c8b2adf084cc58fa2a9a53094 Author: Arturo Filastò <hellais@gmail.com> Date: Fri Mar 16 20:17:19 2012 -0700 Implement HTTP Host based filtering detection Used to detect censorship on T-Mobile --- tests/httphost.py | 132 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 132 insertions(+), 0 deletions(-) diff --git a/tests/httphost.py b/tests/httphost.py new file mode 100644 index 0000000..6446e1f --- /dev/null +++ b/tests/httphost.py @@ -0,0 +1,132 @@ +""" + HTTP Host based filtering + ************************* + + This test detect HTTP Host field + based filtering. + It is used to detect censorship on + performed with Web Guard (used by + T-Mobile US). +""" +import os +from datetime import datetime +from gevent import monkey + +import urllib2 +import httplib +# WARNING! Using gevent's socket +# introduces the 0x20 DNS "feature". +# This will result is weird DNS requests +# appearing on the wire. +monkey.patch_socket() + +from BeautifulSoup import BeautifulSoup + +from plugoo.assets import Asset +from plugoo.tests import Test + +__plugoo__ = "HTTP Host" +__desc__ = "This detects HTTP Host field based filtering" + +class HTTPHostAsset(Asset): + """ + This is the asset that should be used by the Test. It will + contain all the code responsible for parsing the asset file + and should be passed on instantiation to the test. + """ + def __init__(self, file=None): + self = Asset.__init__(self, file) + + def parse_line(self, line): + return line.split(',')[1].replace('\n','') + +class HTTPHost(Test): + """ + The main Test class + """ + + def check_response(self, response): + soup = BeautifulSoup(response) + if soup.head.title.string == "Content Filtered": + # Response indicates censorship + return True + else: + # Response does not indicate censorship + return False + + + def is_censored(self, response): + if response: + soup = BeautifulSoup(response) + censored = self.check_response(response) + else: + censored = "unreachable" + return censored + + def urllib2_test(self, control_server, host): + req = urllib2.Request(control_server) + req.add_header('Host', host) + try: + r = urllib2.urlopen(req) + response = r.read() + censored = self.is_censored(response) + except Exception, e: + censored = "Error! %s" % e + + return censored + + def httplib_test(self, control_server, host): + try: + conn = httplib.HTTPConnection(control_server) + conn.putrequest("GET", "", skip_host=True, skip_accept_encoding=True) + conn.putheader("Host", host) + conn.putheader("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6") + conn.endheaders() + r = conn.getresponse() + response = r.read() + censored = self.is_censored(response) + except Exception, e: + censored = "Error! %s" % e + + return censored + + + def experiment(self, *a, **kw): + """ + Try to connect to the control server with + the specified host field. + """ + host = kw['data'] + control_server = kw['control_server'] + self.logger.info("Testing %s (%s)" % (host, control_server)) + + #censored = self.urllib2_test(control_server, host) + censored = self.httplib_test(control_server, host) + + self.logger.info("%s: %s" % (host, censored)) + return {'Time': datetime.now(), + 'Host': host, + 'Censored': censored} + + +def run(ooni): + """ + This is the function that will be called by OONI + and it is responsible for instantiating and passing + the arguments to the Test class. + """ + config = ooni.config + + # This the assets array to be passed to the run function of + # the test + assets = [HTTPHostAsset(os.path.join(config.main.assetdir, \ + "top-1m.csv"))] + + # Instantiate the Test + thetest = HTTPHost(ooni) + ooni.logger.info("starting HTTP Host Test...") + # Run the test with argument assets + thetest.run(assets, {'index': 5825, 'control_server': '195.85.254.203:8080'}) + ooni.logger.info("finished.") + +