March 2014 - tor-commits - lists.torproject.org

[bridgedb/master] Add a mocked DummyIPBasedDistributor class to test_HTTPServer.py.
by isis＠torproject.org 26 Mar '14

26 Mar '14

commit 6fcf0121296067f7e47721af315ca5b85021e8b8 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Mar 19 17:56:26 2014 +0000 Add a mocked DummyIPBasedDistributor class to test_HTTPServer.py. A distributor is a required parameter to the `HTTPServer.WebResourceBridges` class. This one merely creates N random `DummyBridges`, where N is the number of bridges we're told to return to the client. --- lib/bridgedb/test/test_HTTPServer.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/lib/bridgedb/test/test_HTTPServer.py b/lib/bridgedb/test/test_HTTPServer.py index 05e041c..e2856c7 100644 --- a/lib/bridgedb/test/test_HTTPServer.py +++ b/lib/bridgedb/test/test_HTTPServer.py @@ -418,3 +418,30 @@ class DummyBridge(object): bridgeLine = " ".join([item for item in line]) #print "Created config line: %r" % bridgeLine return bridgeLine + + +class DummyIPBasedDistributor(object): + """A mocked :class:`bridgedb.Dist.IPBasedDistributor` which is used to test + :class:`bridgedb.HTTPServer.WebResourceBridges. + """ + + def _dumbAreaMapper(ip): return ip + + def __init__(self, areaMapper=None, nClusters=None, key=None, + ipCategories=None, answerParameters=None): + """None of the parameters are really used, they are just there to retain + an identical method signature. + """ + self.areaMapper = self._dumbAreaMapper + self.nClusters = 3 + self.nBridgesToGive = 3 + self.key = self.__class__.__name__ + self.ipCategories = ipCategories + self.answerParameters = answerParameters + + def getBridgesForIP(self, ip=None, epoch=None, N=1, + countyCode=None, bridgeFilterRules=None): + """Needed because it's called in + :meth:`WebResourceBridges.getBridgesForIP`. + """ + return [DummyBridge() for _ in xrange(N)]

1 0

[bridgedb/master] Use the class wrapper DummyRequest in unittests in test_HTTPServer.py.
by isis＠torproject.org 26 Mar '14

26 Mar '14

commit 3d6a4ca6f0aaba53785ab935ddc036549e925a7e Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Mar 19 17:58:48 2014 +0000 Use the class wrapper DummyRequest in unittests in test_HTTPServer.py. --- lib/bridgedb/test/test_HTTPServer.py | 36 +++++----------------------------- 1 file changed, 5 insertions(+), 31 deletions(-) diff --git a/lib/bridgedb/test/test_HTTPServer.py b/lib/bridgedb/test/test_HTTPServer.py index 73297de..cd55b46 100644 --- a/lib/bridgedb/test/test_HTTPServer.py +++ b/lib/bridgedb/test/test_HTTPServer.py @@ -65,7 +65,7 @@ class CaptchaProtectedResourceTests(unittest.TestCase): """render_GET() should return a page without a CAPTCHA, which has the image alt text. """ - request = requesthelper.DummyRequest([self.pagename]) + request = DummyRequest([self.pagename]) request.method = b'GET' page = self.captchaResource.render_GET(request) self.assertSubstring( @@ -78,7 +78,7 @@ class CaptchaProtectedResourceTests(unittest.TestCase): oldLookup = HTTPServer.lookup try: HTTPServer.lookup = None - request = requesthelper.DummyRequest([self.pagename]) + request = DummyRequest([self.pagename]) request.method = b'GET' page = self.captchaResource.render_GET(request) errorPage = HTTPServer.replaceErrorPage(Exception('kablam')) @@ -104,18 +104,8 @@ class CaptchaProtectedResourceTests(unittest.TestCase): """render_POST() with a wrong 'captcha_response_field' should return a redirect to the CaptchaProtectedResource page. """ - pagename = 'captcha.html' - self.root.putChild(pagename, self.captchaResource) - - def redirect(request): - newRequest = type(request) - newRequest.uri = pagename - return newRequest - - request = requesthelper.DummyRequest(['captcha.html']) + request = DummyRequest([self.pagename]) request.method = b'POST' - request.redirect = redirect(request) - page = self.captchaResource.render_POST(request) self.assertEqual(BeautifulSoup(page).find('meta')['http-equiv'], 'refresh') @@ -149,15 +139,7 @@ class GimpCaptchaProtectedResourceTests(unittest.TestCase): self.root.putChild(self.pagename, self.captchaResource) # Set up the basic parts of our faked request: - self.request = requesthelper.DummyRequest([self.pagename]) - self.request.URLPath = lambda: request.uri # Fake the URLPath too - self.request.redirect = self.doRedirect(self.request) - - def doRedirect(self, request): - """Stub method to add a redirect() to DummyResponse.""" - newRequest = type(request) - newRequest.uri = self.pagename - return newRequest + self.request = DummyRequest([self.pagename]) def tearDown(self): """Delete the cached CAPTCHA directory if it still exists.""" @@ -283,15 +265,7 @@ class ReCaptchaProtectedResourceTests(unittest.TestCase): self.root.putChild(self.pagename, self.captchaResource) # Set up the basic parts of our faked request: - self.request = requesthelper.DummyRequest([self.pagename]) - self.request.URLPath = lambda: request.uri # Fake the URLPath too - self.request.redirect = self.doRedirect(self.request) - - def doRedirect(self, request): - """Stub method to add a redirect() to DummyResponse.""" - newRequest = type(request) - newRequest.uri = self.pagename - return newRequest + self.request = DummyRequest([self.pagename]) def tearDown(self): """Cleanup method for removing timed out connections on the reactor.

1 0

[bridgedb/master] Add a mocked DummyBridge class to test_HTTPServer.py.
by isis＠torproject.org 26 Mar '14

26 Mar '14

commit f0ca59c8d1792d2fcbea54b0eb29feb10ecacdb0 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Mar 19 17:46:32 2014 +0000 Add a mocked DummyBridge class to test_HTTPServer.py. This is used to test that bridges are actually returned on 'bridges.html' results pages, and is missing all of the real class's methods except for `getConfigLine()`. The latter method needs to be mocked because is used in `HTTPServer.WebResourceBridges.render()` to get the bridge line to display to a client. --- lib/bridgedb/test/test_HTTPServer.py | 49 ++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/lib/bridgedb/test/test_HTTPServer.py b/lib/bridgedb/test/test_HTTPServer.py index 119d2fc..05e041c 100644 --- a/lib/bridgedb/test/test_HTTPServer.py +++ b/lib/bridgedb/test/test_HTTPServer.py @@ -11,8 +11,11 @@ """Unittests for :mod:`bridgedb.HTTPServer`.""" +from __future__ import print_function + import logging import os +import random import shutil import ipaddr @@ -354,7 +357,6 @@ class ReCaptchaProtectedResourceTests(unittest.TestCase): self.request.addArg('captcha_response_field', '') page = self.captchaResource.render_POST(self.request) - print page self.assertEqual(BeautifulSoup(page).find('meta')['http-equiv'], 'refresh') @@ -370,6 +372,49 @@ class ReCaptchaProtectedResourceTests(unittest.TestCase): self.request.addArg('captcha_response_field', expectedResponse) page = self.captchaResource.render_POST(self.request) - print page self.assertEqual(BeautifulSoup(page).find('meta')['http-equiv'], 'refresh') + + +class DummyBridge(object): + """A mock :class:`bridgedb.Bridges.Bridge` which only supports a mocked + ``getConfigLine`` method.""" + + def _randORPort(self): return random.randint(9001, 9999) + def _randPTPort(self): return random.randint(6001, 6666) + def _returnFour(self): return random.randint(2**24, 2**32-1) + def _returnSix(self): return random.randint(2**24, 2**128-1) + + def __init__(self, transports=[]): + """Create a mocked bridge suitable for testing distributors and web + resource rendering. + """ + self.nickname = "bridge-{0}".format(self._returnFour()) + self.ip = ipaddr.IPv4Address(self._returnFour()) + self.orport = self._randORPort() + self.transports = transports + self.running = True + self.stable = True + self.blockingCountries = {} + self.desc_digest = None + self.ei_digest = None + self.verified = False + self.fingerprint = "".join(random.choice('abcdef0123456789') + for _ in xrange(40)) + self.or_addresses = {ipaddr.IPv6Address(self._returnSix()): + self._randORPort()} + + def getConfigLine(self, includeFingerprint=True, + addressClass=ipaddr.IPv4Address, + transport=None, + request=None): + """Get a "torrc" bridge config line to give to a client.""" + line = [] + if transport is not None: + #print("getConfigLine got transport=%r" % transport) + line.append(str(transport)) + line.append("%s:%s" % (self.ip, self.orport)) + if includeFingerprint is True: line.append(self.fingerprint) + bridgeLine = " ".join([item for item in line]) + #print "Created config line: %r" % bridgeLine + return bridgeLine

1 0

[bridgedb/master] Add class wrapper for requesthelper.DummyRequest to support redirects.
by isis＠torproject.org 26 Mar '14

26 Mar '14

commit ad774181faa5cd367e4173ce14a4449a3164abeb Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Mar 19 17:57:09 2014 +0000 Add class wrapper for requesthelper.DummyRequest to support redirects. --- lib/bridgedb/test/test_HTTPServer.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/lib/bridgedb/test/test_HTTPServer.py b/lib/bridgedb/test/test_HTTPServer.py index e2856c7..73297de 100644 --- a/lib/bridgedb/test/test_HTTPServer.py +++ b/lib/bridgedb/test/test_HTTPServer.py @@ -445,3 +445,23 @@ class DummyIPBasedDistributor(object): :meth:`WebResourceBridges.getBridgesForIP`. """ return [DummyBridge() for _ in xrange(N)] + + +class DummyRequest(requesthelper.DummyRequest): + """Wrapper for :api:`twisted.test.requesthelper.DummyRequest` to add + redirect support. + """ + + def __init__(self, *args, **kwargs): + requesthelper.DummyRequest.__init__(self, *args, **kwargs) + self.redirect = self._redirect(self) + + def URLPath(self): + """Fake the missing Request.URLPath too.""" + return self.uri + + def _redirect(self, request): + """Stub method to add a redirect() method to DummyResponse.""" + newRequest = type(request) + newRequest.uri = request.uri + return newRequest

1 0

[bridgedb/master] Add a steaming pile of XXXs and FIXMEs to problems in WebResourceBridges.
by isis＠torproject.org 26 Mar '14

26 Mar '14

commit 17f9650dfdcd0a0a00bd245295fa957efe3e7dfc Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Mar 19 18:27:01 2014 +0000 Add a steaming pile of XXXs and FIXMEs to problems in WebResourceBridges. --- lib/bridgedb/HTTPServer.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/bridgedb/HTTPServer.py b/lib/bridgedb/HTTPServer.py index 27dd61e..0de984b 100644 --- a/lib/bridgedb/HTTPServer.py +++ b/lib/bridgedb/HTTPServer.py @@ -426,7 +426,6 @@ class ReCaptchaProtectedResource(CaptchaProtectedResource): logging.debug("Captcha from %r. Parameters: %r" % (Util.logSafely(clientIP), request.args)) - def checkResponse(solution, clientIP): if solution.is_valid: logging.info("Valid CAPTCHA solution from %r." @@ -564,10 +563,14 @@ class WebResourceBridges(twisted.web.resource.Resource): :rtype: str :returns: A plaintext or HTML response to serve. """ + # XXX why are we getting the interval if our distributor might be + # using bridgedb.Time.NoSchedule? interval = self.schedule.getInterval(time.time()) bridges = ( ) ip = None countryCode = None + + # XXX this code is duplicated in CaptchaProtectedResource if self.useForwardedHeader: h = request.getHeader("X-Forwarded-For") if h: @@ -578,6 +581,8 @@ class WebResourceBridges(twisted.web.resource.Resource): else: ip = request.getClientIP() + # XXX This can also be a separate function + # XXX if the ip is None, this throws an exception if geoip: countryCode = geoip.country_code_by_addr(ip) if countryCode: @@ -587,6 +592,7 @@ class WebResourceBridges(twisted.web.resource.Resource): if rtl: logging.debug("Rendering RTL response.") + # XXX separate function again format = request.args.get("format", None) if format and len(format): format = format[0] # choose the first arg @@ -596,6 +602,9 @@ class WebResourceBridges(twisted.web.resource.Resource): ipv6 = request.args.get("ipv6", False) if ipv6: ipv6 = True # if anything after ?ipv6= + # XXX oh dear hell. why not check for the '?transport=' arg before + # regex'ing? And why not compile the regex once, somewhere outside + # this function and class? try: # validate method name transport = re.match('[_a-zA-Z][_a-zA-Z0-9]*', @@ -673,6 +682,9 @@ class WebResourceBridges(twisted.web.resource.Resource): else: request.setHeader("Content-Type", "text/html; charset=utf-8") try: + # XXX FIXME the returned page from + # ``WebResourceBridgesTests.test_render_GET_RTLlang`` + # is in Arabic and has `<html lang="en">`! Doh. template = lookup.get_template('bridges.html') rendered = template.render(answer=bridgeLines, rtl=rtl) except Exception as err:

1 0

[bridgedb/master] Fix txrecaptcha.submit() to always return Deferreds.
by isis＠torproject.org 26 Mar '14

26 Mar '14

commit b4f88aded4fd6cc60be0129e1b6d0bd708d392c7 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Mar 19 18:30:12 2014 +0000 Fix txrecaptcha.submit() to always return Deferreds. All returned Deferreds callback with RecaptchaResponse objects. This way, we do not need additional code to check if the result was a Deferred (because, in that case, we would need to add a callback function to extract the RecaptchaResponse object from it and check it), or if it directly returned a RecaptchaResponse. This function now *always* returns Deferred. --- lib/bridgedb/txrecaptcha.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/bridgedb/txrecaptcha.py b/lib/bridgedb/txrecaptcha.py index e1e3234..2185112 100644 --- a/lib/bridgedb/txrecaptcha.py +++ b/lib/bridgedb/txrecaptcha.py @@ -213,12 +213,12 @@ def submit(recaptcha_challenge_field, recaptcha_response_field, :returns: A :api:`~twisted.internet.defer.Deferred` which will callback with a ``recaptcha.RecaptchaResponse`` for the request. """ - if not (recaptcha_response_field and - recaptcha_challenge_field and - len(recaptcha_response_field) and - len(recaptcha_challenge_field)): - return RecaptchaResponse(is_valid=False, - error_code='incorrect-captcha-sol') + if not (recaptcha_response_field and len(recaptcha_response_field) and + recaptcha_challenge_field and len(recaptcha_challenge_field)): + d = defer.Deferred() + d.addBoth(_ebRequest) # We want `is_valid=False` + d.errback(failure.Failure(ValueError('incorrect-captcha-sol'))) + return d params = urllib.urlencode({ 'privatekey': _encodeIfNecessary(private_key),

1 0

[bridgedb/master] Give the PluggableTransports class a real docstring.
by isis＠torproject.org 26 Mar '14

26 Mar '14

commit 18d8d30bf67c13396516cb980cd9523720aa2b40 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Mar 20 12:46:17 2014 +0000 Give the PluggableTransports class a real docstring. --- lib/bridgedb/Bridges.py | 52 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index 9dbafdf..2fb784a 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -466,12 +466,56 @@ re_ipv4 = re.compile("((?:\d{1,3}\.?){4}):(.*$)") class PluggableTransport(object): - """ - an object that represents a pluggable-transport method - and a reference to the relevant bridge - """ + """A PT with reference to the parent bridge on which it is running.""" + def __init__(self, bridge, methodname, address, port, argdict=None): + """Create a ``PluggableTransport`` describing a PT running on a bridge. + + Pluggable transports are described within a bridge's ``@type + bridge-extrainfo`` descriptor, see the ``Specifications: Client + behavior`` section and the ``TOR_PT_SERVER_TRANSPORT_OPTIONS`` + description in pt-spec.txt_ for additional specification. + + :type bridge: :class:`Bridge` + :param bridge: The parent bridge running this pluggable transport + instance, i.e. the main ORPort bridge whose + ``@type bridge-server-descriptor`` contains a hash digest for a + ``@type bridge-extrainfo-document``, the latter of which contains + the parameter of this pluggable transport in its ``transport`` + line. + + :param str methodname: The canonical "name" for this pluggable + transport, i.e. the one which would be specified in a torrc + file. For example, ``"obfs2"``, ``"obfs3"``, ``"scramblesuit"`` + would all be pluggable transport method names. + + :param str address: The IP address of the transport. Currently (as of + 20 March 2014), there are no known, widely-deployed pluggable + transports which support IPv6. Ergo, this is very likely going to + be an IPv4 address. + :param int port: A integer specifying the port which this pluggable + transport is listening on. (This should likely be whatever port the + bridge specified in its ``ServerTransportPlugin`` torrc line, + unless the pluggable transport is running in "managed" mode.) + + :param dict argdict: Some PTs can take additional arguments, which + must be distributed to the client out-of-band. These are present + in the ``@type bridge-extrainfo-document``, in the ``transport`` + line like so:: + + METHOD SP ADDR ":" PORT SP [K=V[,K=V[,K=V[…]]]] + + where K is the **argdict** key, and V is the value. For example, + in the case of ``scramblesuit``, for which the client must supply + a shared secret to the ``scramblesuit`` instance running on the + bridge, the **argdict** would be something like:: + + {'password': 'NEQGQYLUMUQGK5TFOJ4XI2DJNZTS4LRO'} + + .. _pt-spec.txt: + https://gitweb.torproject.org/torspec.git/blob/HEAD:/pt-spec.txt + """ #XXX: assert are disabled with python -O assert isinstance(bridge, Bridge) assert type(address) in (ipaddr.IPv4Address, ipaddr.IPv6Address)

1 0

[bridgedb/master] Merge branch 'fix/11231-additional-bug-fixes' into develop
by isis＠torproject.org 26 Mar '14

26 Mar '14

commit 00eaf9160f65c16c725ed7f05ac0583b9da63dc0 Merge: 7fdde31 b4f88ad Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Mar 21 22:30:10 2014 +0000 Merge branch 'fix/11231-additional-bug-fixes' into develop lib/bridgedb/HTTPServer.py | 14 +++++++++++++- lib/bridgedb/txrecaptcha.py | 12 ++++++------ 2 files changed, 19 insertions(+), 7 deletions(-)

1 0

[bridgedb/master] Merge branch 'fix/11231-blank-captcha-field' into develop
by isis＠torproject.org 26 Mar '14

26 Mar '14

commit 7fdde318ce5446ff55646ccde44f7586d2d411b6 Merge: 18d8d30 a4dd886 Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Mar 21 22:27:09 2014 +0000 Merge branch 'fix/11231-blank-captcha-field' into develop lib/bridgedb/HTTPServer.py | 8 +- lib/bridgedb/test/test_HTTPServer.py | 597 +++++++++++++++++++++++++++++++++ lib/bridgedb/test/test_txrecaptcha.py | 21 +- 3 files changed, 616 insertions(+), 10 deletions(-)

1 0

[bridgedb/master] Add leekspin dependency; modify test_bridgedb and bridgedb.runner apropos.
by isis＠torproject.org 26 Mar '14

26 Mar '14

commit 6d991ba844be61f944b13ca4a7077c03464406a0 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon Mar 24 03:12:52 2014 +0000 Add leekspin dependency; modify test_bridgedb and bridgedb.runner apropos. --- .travis.yml | 4 ++-- lib/bridgedb/runner.py | 2 +- lib/bridgedb/test/test_bridgedb.py | 12 +++++++++--- setup.py | 8 ++++---- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8d8bfcf..902c3b4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,8 +27,8 @@ python: - "2.7" install: - - pip install -r requirements.txt --use-mirrors - - pip install coverage==3.7 coveralls==0.3 sure==1.2.2 --use-mirrors + - pip install -r requirements.txt + - pip install --no-use-wheel leekspin==0.1.1 coverage==3.7 coveralls==0.3 sure==1.2.2 - make install script: diff --git a/lib/bridgedb/runner.py b/lib/bridgedb/runner.py index 8c641f6..d067bd9 100644 --- a/lib/bridgedb/runner.py +++ b/lib/bridgedb/runner.py @@ -45,7 +45,7 @@ def generateDescriptors(count=None, rundir=None): proc = None statuscode = 0 - script = 'gen_bridge_descriptors' + script = 'leekspin' rundir = rundir if os.path.isdir(rundir) else None count = count if count else 3 try: diff --git a/lib/bridgedb/test/test_bridgedb.py b/lib/bridgedb/test/test_bridgedb.py index 3d21080..5dc890d 100644 --- a/lib/bridgedb/test/test_bridgedb.py +++ b/lib/bridgedb/test/test_bridgedb.py @@ -75,12 +75,18 @@ class BridgeDBCliTest(unittest.TestCase): bridgedbScript = bridgedbScript[0] print("Running bridgedb script %r..." % bridgedbScript) + os.chdir(runDir) # we have to do this to get files to end up there print("Running `bridgedb mock' to generate mock bridge descriptors...") - mockProc = Popen([bridgedbScript, 'mock', - '-n', '50', - '-r', runDir]) + mockProc = Popen([bridgedbScript, 'mock', '-n', '50']) mockProcCode = mockProc.wait() print("`bridgedb mock' exited with status code %d" % int(mockProcCode)) + os.chdir(here) + + # See ticket #11216, cached-extrainfo* files should not be parsed + # cumulatively. + eidesc = pjoin(runDir, 'cached-extrainfo') + eindesc = pjoin(runDir, 'cached-extrainfo.new') + self.doCopyFile(eindesc, eidesc, 'duplicated cached-extrainfo(.new)') print("Running `bridgedb' to test server startups...") bridgedbProc = Popen([bridgedbScript, '-r', runDir]) diff --git a/setup.py b/setup.py index 5ec2d79..d859f87 100644 --- a/setup.py +++ b/setup.py @@ -274,10 +274,10 @@ setuptools.setup( packages=['bridgedb', 'bridgedb.parse', 'bridgedb.test'], - scripts=['scripts/bridgedb', - 'scripts/gen_bridge_descriptors'], - extras_require={'test': ["sure==0.4.5", - "coverage==3.6"]}, + scripts=['scripts/bridgedb'], + extras_require={'test': ["sure==1.2.2", + "coverage==3.7", + "leekspin==0.1.1"]}, zip_safe=False, cmdclass=get_cmdclass(), include_package_data=True,

1 0