- tor-commits - lists.torproject.org

[torbrowser/maint-2.3] bump versions of Firefox, libpng, NoScript, https-everywhere, and torbutton for pending release
by erinn＠torproject.org 08 Oct '12

08 Oct '12

commit 8831b1a96b1e46fd4a86fb39076604c5f76af66f Author: Erinn Clark <erinn(a)torproject.org> Date: Mon Oct 8 07:41:40 2012 +0100 bump versions of Firefox, libpng, NoScript, https-everywhere, and torbutton for pending release --- build-scripts/versions.mk | 10 +++++----- 1 files changed, 5 insertions(+), 5 deletions(-) diff --git a/build-scripts/versions.mk b/build-scripts/versions.mk index 4640acc..52fa772 100644 --- a/build-scripts/versions.mk +++ b/build-scripts/versions.mk @@ -4,18 +4,18 @@ RELEASE_VER=2.2.39 ZLIB_VER=1.2.7 OPENSSL_VER=1.0.1c -LIBPNG_VER=1.5.12 +LIBPNG_VER=1.5.13 QT_VER=4.8.1 VIDALIA_VER=0.2.20 LIBEVENT_VER=2.0.20-stable TOR_VER=0.2.2.39 PIDGIN_VER=2.6.4 -FIREFOX_VER=10.0.7esr +FIREFOX_VER=10.0.8esr MOZBUILD_VER=1.5.1 PYMAKE_VER=87d436cd8974 -TORBUTTON_VER=1.4.6 -NOSCRIPT_VER=2.5.4 -HTTPSEVERYWHERE_VER=2.2.1 +TORBUTTON_VER=1.4.6.1 +NOSCRIPT_VER=2.5.6 +HTTPSEVERYWHERE_VER=2.2.2 OTR_VER=3.2.0 OBFSPROXY_VER=0.1.4

1 0

[torbrowser/master] bump versions of Firefox, libpng, NoScript, https-everywhere, and torbutton for pending release
by erinn＠torproject.org 08 Oct '12

08 Oct '12

commit 8831b1a96b1e46fd4a86fb39076604c5f76af66f Author: Erinn Clark <erinn(a)torproject.org> Date: Mon Oct 8 07:41:40 2012 +0100 bump versions of Firefox, libpng, NoScript, https-everywhere, and torbutton for pending release --- build-scripts/versions.mk | 10 +++++----- 1 files changed, 5 insertions(+), 5 deletions(-) diff --git a/build-scripts/versions.mk b/build-scripts/versions.mk index 4640acc..52fa772 100644 --- a/build-scripts/versions.mk +++ b/build-scripts/versions.mk @@ -4,18 +4,18 @@ RELEASE_VER=2.2.39 ZLIB_VER=1.2.7 OPENSSL_VER=1.0.1c -LIBPNG_VER=1.5.12 +LIBPNG_VER=1.5.13 QT_VER=4.8.1 VIDALIA_VER=0.2.20 LIBEVENT_VER=2.0.20-stable TOR_VER=0.2.2.39 PIDGIN_VER=2.6.4 -FIREFOX_VER=10.0.7esr +FIREFOX_VER=10.0.8esr MOZBUILD_VER=1.5.1 PYMAKE_VER=87d436cd8974 -TORBUTTON_VER=1.4.6 -NOSCRIPT_VER=2.5.4 -HTTPSEVERYWHERE_VER=2.2.1 +TORBUTTON_VER=1.4.6.1 +NOSCRIPT_VER=2.5.6 +HTTPSEVERYWHERE_VER=2.2.2 OTR_VER=3.2.0 OBFSPROXY_VER=0.1.4

1 0

[torbrowser/master] Merge branch 'maint-2.2' into maint-2.3
by erinn＠torproject.org 08 Oct '12

08 Oct '12

commit 6f2a09b585034c93419e192b6418518eeb68dea8 Merge: 31f5977 8831b1a Author: Erinn Clark <erinn(a)torproject.org> Date: Mon Oct 8 07:42:10 2012 +0100 Merge branch 'maint-2.2' into maint-2.3 build-scripts/versions.mk | 10 +- ...d-mozIThirdPartyUtil.getFirstPartyURI-API.patch | 148 ++++++++++++++++++++ 2 files changed, 153 insertions(+), 5 deletions(-)

1 0

[torbrowser/maint-2.3] Add mozIThirdPartyUtil.getFirstPartyURI API patch.
by erinn＠torproject.org 08 Oct '12

08 Oct '12

commit 7a1abc347144e4e24bef26c80a2c7b2ac441317b Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Tue Sep 25 14:16:49 2012 -0700 Add mozIThirdPartyUtil.getFirstPartyURI API patch. --- ...d-mozIThirdPartyUtil.getFirstPartyURI-API.patch | 148 ++++++++++++++++++++ 1 files changed, 148 insertions(+), 0 deletions(-) diff --git a/src/current-patches/firefox/0020-Add-mozIThirdPartyUtil.getFirstPartyURI-API.patch b/src/current-patches/firefox/0020-Add-mozIThirdPartyUtil.getFirstPartyURI-API.patch new file mode 100644 index 0000000..56d9848 --- /dev/null +++ b/src/current-patches/firefox/0020-Add-mozIThirdPartyUtil.getFirstPartyURI-API.patch @@ -0,0 +1,148 @@ +From 73e548ceee36b99f06e33010163ed8b8cc86b3dd Mon Sep 17 00:00:00 2001 +From: Mike Perry <mikeperry-git(a)torproject.org> +Date: Tue, 28 Aug 2012 18:35:33 -0700 +Subject: [PATCH 20/20] Add mozIThirdPartyUtil.getFirstPartyURI API + +API allows you to get the url bar URI for a channel or nsIDocument. +--- + content/base/src/ThirdPartyUtil.cpp | 52 ++++++++++++++++++++++++++++ + content/base/src/ThirdPartyUtil.h | 2 + + netwerk/base/public/mozIThirdPartyUtil.idl | 21 +++++++++++ + 3 files changed, 75 insertions(+), 0 deletions(-) + +diff --git a/content/base/src/ThirdPartyUtil.cpp b/content/base/src/ThirdPartyUtil.cpp +index 6a415e9..62333f3 100644 +--- a/content/base/src/ThirdPartyUtil.cpp ++++ b/content/base/src/ThirdPartyUtil.cpp +@@ -40,6 +40,9 @@ + #include "nsIServiceManager.h" + #include "nsIHttpChannelInternal.h" + #include "nsIDOMWindow.h" ++#include "nsICookiePermission.h" ++#include "nsIDOMDocument.h" ++#include "nsIDocument.h" + #include "nsILoadContext.h" + #include "nsIPrincipal.h" + #include "nsIScriptObjectPrincipal.h" +@@ -54,6 +57,7 @@ ThirdPartyUtil::Init() + + nsresult rv; + mTLDService = do_GetService(NS_EFFECTIVETLDSERVICE_CONTRACTID, &rv); ++ mCookiePermissions = do_GetService(NS_COOKIEPERMISSION_CONTRACTID); + return rv; + } + +@@ -315,3 +319,51 @@ ThirdPartyUtil::GetBaseDomain(nsIURI* aHostURI, + + return NS_OK; + } ++ ++NS_IMETHODIMP ++ThirdPartyUtil::GetFirstPartyURI(nsIChannel *aChannel, ++ nsIDocument *aDoc, ++ nsIURI **aOutput) ++{ ++ nsresult rv = NS_ERROR_NULL_POINTER; ++ ++ if (!aChannel && aDoc) { ++ aChannel = aDoc->GetChannel(); ++ } ++ ++ // If aChannel is specified or available, use the official route ++ // for sure ++ if (aChannel) { ++ rv = mCookiePermissions->GetOriginatingURI(aChannel, aOutput); ++ } ++ ++ // If the channel was missing, closed or broken, try the ++ // window hierarchy directly. ++ // ++ // This might fail to work for first-party loads themselves, but ++ // we don't need this codepath for that case. ++ if (NS_FAILED(rv) && aDoc) { ++ nsCOMPtr<nsIDOMWindow> top; ++ nsCOMPtr<nsIDOMDocument> topDDoc; ++ ++ aDoc->GetWindow()->GetTop(getter_AddRefs(top)); ++ top->GetDocument(getter_AddRefs(topDDoc)); ++ ++ nsCOMPtr<nsIDocument> topDoc(do_QueryInterface(topDDoc)); ++ *aOutput = topDoc->GetOriginalURI(); ++ ++ if (*aOutput) ++ rv = NS_OK; ++ } ++ ++ // TODO: We could provide a route through the loadgroup + notification ++ // callbacks too, but either channel or document was always available ++ // in the cases where this function was originally needed (the image cache). ++ // The notification callbacks also appear to suffers from the same limitation ++ // as the document path. See nsICookiePermissions.GetOriginatingURI() for ++ // details. ++ ++ return rv; ++} ++ ++ +diff --git a/content/base/src/ThirdPartyUtil.h b/content/base/src/ThirdPartyUtil.h +index 58ddb15..c8eab11 100644 +--- a/content/base/src/ThirdPartyUtil.h ++++ b/content/base/src/ThirdPartyUtil.h +@@ -42,6 +42,7 @@ + #include "nsString.h" + #include "mozIThirdPartyUtil.h" + #include "nsIEffectiveTLDService.h" ++#include "nsICookiePermission.h" + + class nsIURI; + class nsIChannel; +@@ -61,6 +62,7 @@ private: + static already_AddRefed<nsIURI> GetURIFromWindow(nsIDOMWindow* aWin); + + nsCOMPtr<nsIEffectiveTLDService> mTLDService; ++ nsCOMPtr<nsICookiePermission> mCookiePermissions; + }; + + #endif +diff --git a/netwerk/base/public/mozIThirdPartyUtil.idl b/netwerk/base/public/mozIThirdPartyUtil.idl +index ad41985..fd2cb38 100644 +--- a/netwerk/base/public/mozIThirdPartyUtil.idl ++++ b/netwerk/base/public/mozIThirdPartyUtil.idl +@@ -40,6 +40,7 @@ + interface nsIURI; + interface nsIDOMWindow; + interface nsIChannel; ++interface nsIDocument; + + /** + * Utility functions for determining whether a given URI, channel, or window +@@ -173,6 +174,26 @@ interface mozIThirdPartyUtil : nsISupports + * @return the base domain. + */ + AUTF8String getBaseDomain(in nsIURI aHostURI); ++ ++ ++ /** ++ * getFirstPartyURI ++ * ++ * Obtain the top-level url bar URI for either a channel or a document. ++ * Either parameter may be null (but not both). ++ * ++ * @param aChannel ++ * An arbitrary channel for some content element of a first party ++ * load. Can be null. ++ * ++ * @param aDoc ++ * An arbitrary third party document. Can be null. ++ * ++ * @return the first party url bar URI for the load. ++ */ ++ nsIURI getFirstPartyURI(in nsIChannel aChannel, ++ in nsIDocument aDoc); ++ + }; + + %{ C++ +-- +1.7.5.4 +

1 0

[torbrowser/master] Add mozIThirdPartyUtil.getFirstPartyURI API patch.
by erinn＠torproject.org 08 Oct '12

08 Oct '12

commit 7a1abc347144e4e24bef26c80a2c7b2ac441317b Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Tue Sep 25 14:16:49 2012 -0700 Add mozIThirdPartyUtil.getFirstPartyURI API patch. --- ...d-mozIThirdPartyUtil.getFirstPartyURI-API.patch | 148 ++++++++++++++++++++ 1 files changed, 148 insertions(+), 0 deletions(-) diff --git a/src/current-patches/firefox/0020-Add-mozIThirdPartyUtil.getFirstPartyURI-API.patch b/src/current-patches/firefox/0020-Add-mozIThirdPartyUtil.getFirstPartyURI-API.patch new file mode 100644 index 0000000..56d9848 --- /dev/null +++ b/src/current-patches/firefox/0020-Add-mozIThirdPartyUtil.getFirstPartyURI-API.patch @@ -0,0 +1,148 @@ +From 73e548ceee36b99f06e33010163ed8b8cc86b3dd Mon Sep 17 00:00:00 2001 +From: Mike Perry <mikeperry-git(a)torproject.org> +Date: Tue, 28 Aug 2012 18:35:33 -0700 +Subject: [PATCH 20/20] Add mozIThirdPartyUtil.getFirstPartyURI API + +API allows you to get the url bar URI for a channel or nsIDocument. +--- + content/base/src/ThirdPartyUtil.cpp | 52 ++++++++++++++++++++++++++++ + content/base/src/ThirdPartyUtil.h | 2 + + netwerk/base/public/mozIThirdPartyUtil.idl | 21 +++++++++++ + 3 files changed, 75 insertions(+), 0 deletions(-) + +diff --git a/content/base/src/ThirdPartyUtil.cpp b/content/base/src/ThirdPartyUtil.cpp +index 6a415e9..62333f3 100644 +--- a/content/base/src/ThirdPartyUtil.cpp ++++ b/content/base/src/ThirdPartyUtil.cpp +@@ -40,6 +40,9 @@ + #include "nsIServiceManager.h" + #include "nsIHttpChannelInternal.h" + #include "nsIDOMWindow.h" ++#include "nsICookiePermission.h" ++#include "nsIDOMDocument.h" ++#include "nsIDocument.h" + #include "nsILoadContext.h" + #include "nsIPrincipal.h" + #include "nsIScriptObjectPrincipal.h" +@@ -54,6 +57,7 @@ ThirdPartyUtil::Init() + + nsresult rv; + mTLDService = do_GetService(NS_EFFECTIVETLDSERVICE_CONTRACTID, &rv); ++ mCookiePermissions = do_GetService(NS_COOKIEPERMISSION_CONTRACTID); + return rv; + } + +@@ -315,3 +319,51 @@ ThirdPartyUtil::GetBaseDomain(nsIURI* aHostURI, + + return NS_OK; + } ++ ++NS_IMETHODIMP ++ThirdPartyUtil::GetFirstPartyURI(nsIChannel *aChannel, ++ nsIDocument *aDoc, ++ nsIURI **aOutput) ++{ ++ nsresult rv = NS_ERROR_NULL_POINTER; ++ ++ if (!aChannel && aDoc) { ++ aChannel = aDoc->GetChannel(); ++ } ++ ++ // If aChannel is specified or available, use the official route ++ // for sure ++ if (aChannel) { ++ rv = mCookiePermissions->GetOriginatingURI(aChannel, aOutput); ++ } ++ ++ // If the channel was missing, closed or broken, try the ++ // window hierarchy directly. ++ // ++ // This might fail to work for first-party loads themselves, but ++ // we don't need this codepath for that case. ++ if (NS_FAILED(rv) && aDoc) { ++ nsCOMPtr<nsIDOMWindow> top; ++ nsCOMPtr<nsIDOMDocument> topDDoc; ++ ++ aDoc->GetWindow()->GetTop(getter_AddRefs(top)); ++ top->GetDocument(getter_AddRefs(topDDoc)); ++ ++ nsCOMPtr<nsIDocument> topDoc(do_QueryInterface(topDDoc)); ++ *aOutput = topDoc->GetOriginalURI(); ++ ++ if (*aOutput) ++ rv = NS_OK; ++ } ++ ++ // TODO: We could provide a route through the loadgroup + notification ++ // callbacks too, but either channel or document was always available ++ // in the cases where this function was originally needed (the image cache). ++ // The notification callbacks also appear to suffers from the same limitation ++ // as the document path. See nsICookiePermissions.GetOriginatingURI() for ++ // details. ++ ++ return rv; ++} ++ ++ +diff --git a/content/base/src/ThirdPartyUtil.h b/content/base/src/ThirdPartyUtil.h +index 58ddb15..c8eab11 100644 +--- a/content/base/src/ThirdPartyUtil.h ++++ b/content/base/src/ThirdPartyUtil.h +@@ -42,6 +42,7 @@ + #include "nsString.h" + #include "mozIThirdPartyUtil.h" + #include "nsIEffectiveTLDService.h" ++#include "nsICookiePermission.h" + + class nsIURI; + class nsIChannel; +@@ -61,6 +62,7 @@ private: + static already_AddRefed<nsIURI> GetURIFromWindow(nsIDOMWindow* aWin); + + nsCOMPtr<nsIEffectiveTLDService> mTLDService; ++ nsCOMPtr<nsICookiePermission> mCookiePermissions; + }; + + #endif +diff --git a/netwerk/base/public/mozIThirdPartyUtil.idl b/netwerk/base/public/mozIThirdPartyUtil.idl +index ad41985..fd2cb38 100644 +--- a/netwerk/base/public/mozIThirdPartyUtil.idl ++++ b/netwerk/base/public/mozIThirdPartyUtil.idl +@@ -40,6 +40,7 @@ + interface nsIURI; + interface nsIDOMWindow; + interface nsIChannel; ++interface nsIDocument; + + /** + * Utility functions for determining whether a given URI, channel, or window +@@ -173,6 +174,26 @@ interface mozIThirdPartyUtil : nsISupports + * @return the base domain. + */ + AUTF8String getBaseDomain(in nsIURI aHostURI); ++ ++ ++ /** ++ * getFirstPartyURI ++ * ++ * Obtain the top-level url bar URI for either a channel or a document. ++ * Either parameter may be null (but not both). ++ * ++ * @param aChannel ++ * An arbitrary channel for some content element of a first party ++ * load. Can be null. ++ * ++ * @param aDoc ++ * An arbitrary third party document. Can be null. ++ * ++ * @return the first party url bar URI for the load. ++ */ ++ nsIURI getFirstPartyURI(in nsIChannel aChannel, ++ in nsIDocument aDoc); ++ + }; + + %{ C++ +-- +1.7.5.4 +

1 0

[ooni-probe/master] Fine tune DNS Tamper to work properly.
by art＠torproject.org 08 Oct '12

08 Oct '12

commit 87c8296bbd9fcac5335e2d43e98fb254710c67ef Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Oct 8 04:56:22 2012 +0000 Fine tune DNS Tamper to work properly. * Is now able to detect censorship in Italy --- nettests/core/dnstamper.py | 41 ++++++++++++++++++++++++++--------------- 1 files changed, 26 insertions(+), 15 deletions(-) diff --git a/nettests/core/dnstamper.py b/nettests/core/dnstamper.py index 58a8203..58299da 100644 --- a/nettests/core/dnstamper.py +++ b/nettests/core/dnstamper.py @@ -26,12 +26,12 @@ class DNSTamperTest(nettest.TestCase): name = "DNS tamper" description = "DNS censorship detection test" - + version = '0.2' requirements = None inputFile = ['file', 'f', None, 'Input file of list of hostnames to attempt to resolve'] - optParameters = [['controlresolver', 'c', '4.4.4.4', + optParameters = [['controlresolver', 'c', '8.8.8.8', 'Known good DNS server'], ['testresolvers', 't', None, 'file containing list of DNS resolvers to test against'] @@ -45,6 +45,8 @@ class DNSTamperTest(nettest.TestCase): self.report['a_lookups'] = {} + self.report['tampering'] = {} + self.test_a_lookups = {} self.control_a_lookups = [] @@ -83,9 +85,9 @@ class DNSTamperTest(nettest.TestCase): self.test_a_lookups[resolver] = a_a self.report['test_lookups'][resolver] = all_a print "Done" - print self.report def process_ptr_answers(self, answers, resolver): + print "Processing PTR answers for %s" % resolver name = None for answer in answers[0]: if answer.type is 12: @@ -100,14 +102,15 @@ class DNSTamperTest(nettest.TestCase): self.report['test_reverse'][resolver] = name def ptr_lookup_error(self, failure, resolver): - log.err("There was an error in PTR lookup %s" % resolver) + #def ptr_lookup_error(self, *arg, **kw): + print "There was an error in PTR lookup %s" % resolver if resolver == 'control': self.report['control_reverse'] = None else: self.report['test_reverse'][resolver] = None def a_lookup_error(self, failure, resolver): - log.err("There was an error in A lookup %s" % resolver) + print "There was an error in A lookup %s" % resolver if resolver == 'control': self.report['control_lookup'] = None else: @@ -125,12 +128,10 @@ class DNSTamperTest(nettest.TestCase): resolver = [(self.localOptions['controlresolver'], 53)] res = client.createResolver(servers=resolver, resolvconf='') - control_r = res.lookupAllRecords(hostname) + control_r = res.lookupAddress(hostname) control_r.addCallback(self.process_a_answers, 'control') control_r.addErrback(self.a_lookup_error, 'control') - list_of_ds.append(control_r) - for test_resolver in self.test_resolvers: print "Going for %s" % test_resolver resolver = [(test_resolver, 53)] @@ -142,9 +143,10 @@ class DNSTamperTest(nettest.TestCase): d.addErrback(self.a_lookup_error, test_resolver) list_of_ds.append(d) + list_of_ds.append(control_r) dl = defer.DeferredList(list_of_ds) dl.addCallback(self.do_reverse_lookups) - dl.addCallback(self.compare_results) + dl.addBoth(self.compare_results) return dl def reverse_lookup(self, address, resolver): @@ -153,7 +155,7 @@ class DNSTamperTest(nettest.TestCase): return r def do_reverse_lookups(self, result): - print "Doing the reverse lookups" + print "Doing the reverse lookups %s" % self.input list_of_ds = [] resolver = [(self.localOptions['controlresolver'], 53)] @@ -166,7 +168,12 @@ class DNSTamperTest(nettest.TestCase): list_of_ds.append(test_reverse) for test_resolver in self.test_resolvers: - d = self.reverse_lookup(self.test_a_lookups[test_resolver][0], res) + try: + ip = self.test_a_lookups[test_resolver][0] + except: + break + + d = self.reverse_lookup(ip, res) d.addCallback(self.process_ptr_answers, test_resolver) d.addErrback(self.ptr_lookup_error, test_resolver) list_of_ds.append(d) @@ -175,17 +182,21 @@ class DNSTamperTest(nettest.TestCase): return dl def compare_results(self, *arg, **kw): + print "Comparing results for %s" % self.input + print self.test_a_lookups for test, test_a_lookups in self.test_a_lookups.items(): - self.report['tampering'] = {} self.report['tampering'][test] = 'unknown' - - if len(set(test_a_lookups) & set(self.control_a_lookups)) > 0: + if not test_a_lookups: + self.report['tampering'][test] = 'no-result' + continue + if set(test_a_lookups) & set(self.control_a_lookups): # Address has not tampered with on DNS server self.report['tampering'][test] = False - elif len(set(self.control_reverse) & set(self.test_reverse[test])) > 0: + elif set([self.control_reverse]) & set([self.report['test_reverse'][test]]): # Further testing has eliminated false positives self.report['tampering'][test] = 'reverse-match' + else: # Reverse DNS on the results returned by returned # which does not match the expected domainname

1 0

[ooni-probe/master] Fix a nasty bug in inputunit
by art＠torproject.org 08 Oct '12

08 Oct '12

commit 794aef395d2e8210cde6b09ca1e2d8548ecc8ed1 Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Oct 8 03:52:44 2012 +0000 Fix a nasty bug in inputunit * Write input unit to avoid that bug from appearing again * Do some progress in debugging some issues with DNSClient --- nettests/core/dnstamper.py | 31 +++++++++++++++++++++---------- ooni/inputunit.py | 12 +++++++----- tests/test_inputunit.py | 18 ++++++++++++++++++ 3 files changed, 46 insertions(+), 15 deletions(-) diff --git a/nettests/core/dnstamper.py b/nettests/core/dnstamper.py index 92948cd..58a8203 100644 --- a/nettests/core/dnstamper.py +++ b/nettests/core/dnstamper.py @@ -17,6 +17,7 @@ # :licence: see LICENSE from ooni import nettest +from ooni.utils import log from twisted.internet import defer from twisted.names import client @@ -30,7 +31,7 @@ class DNSTamperTest(nettest.TestCase): inputFile = ['file', 'f', None, 'Input file of list of hostnames to attempt to resolve'] - optParameters = [['controlresolver', 'c', '8.8.8.8', + optParameters = [['controlresolver', 'c', '4.4.4.4', 'Known good DNS server'], ['testresolvers', 't', None, 'file containing list of DNS resolvers to test against'] @@ -52,15 +53,17 @@ class DNSTamperTest(nettest.TestCase): if not self.localOptions['testresolvers']: self.test_resolvers = ['8.8.8.8'] return + try: fp = open(self.localOptions['testresolvers']) except: raise usage.UsageError("Invalid test resolvers file") - self.test_resolvers = [x.strip() for x in fp.readlines] + self.test_resolvers = [x.strip() for x in fp.readlines()] fp.close() def process_a_answers(self, answers, resolver): + print "Processing A answers for %s" % resolver all_a = [] a_a = [] for answer in answers[0]: @@ -79,9 +82,10 @@ class DNSTamperTest(nettest.TestCase): else: self.test_a_lookups[resolver] = a_a self.report['test_lookups'][resolver] = all_a + print "Done" + print self.report def process_ptr_answers(self, answers, resolver): - print "Processing PTR ANSWER for %s" % resolver name = None for answer in answers[0]: if answer.type is 12: @@ -95,25 +99,31 @@ class DNSTamperTest(nettest.TestCase): self.test_reverse[resolver] = name self.report['test_reverse'][resolver] = name - def ptr_lookup_error(self, failure, resolver): + log.err("There was an error in PTR lookup %s" % resolver) if resolver == 'control': self.report['control_reverse'] = None else: self.report['test_reverse'][resolver] = None def a_lookup_error(self, failure, resolver): + log.err("There was an error in A lookup %s" % resolver) if resolver == 'control': self.report['control_lookup'] = None else: self.report['test_lookups'][resolver] = None + def createResolver(self, servers): + print "Creating resolver %s" % servers + return client.createResolver(servers=servers, resolvconf='') + def test_lookup(self): + print "Doing the test lookups on %s" % self.input list_of_ds = [] hostname = self.input resolver = [(self.localOptions['controlresolver'], 53)] - res = client.createResolver(servers=resolver) + res = client.createResolver(servers=resolver, resolvconf='') control_r = res.lookupAllRecords(hostname) control_r.addCallback(self.process_a_answers, 'control') @@ -122,8 +132,10 @@ class DNSTamperTest(nettest.TestCase): list_of_ds.append(control_r) for test_resolver in self.test_resolvers: - resolver = [(test_resolver,53)] - res = client.createResolver(servers=resolver) + print "Going for %s" % test_resolver + resolver = [(test_resolver, 53)] + res = client.createResolver(servers=resolver, resolvconf='') + #res = self.createResolver(servers=resolver) d = res.lookupAddress(hostname) d.addCallback(self.process_a_answers, test_resolver) @@ -141,12 +153,11 @@ class DNSTamperTest(nettest.TestCase): return r def do_reverse_lookups(self, result): - - + print "Doing the reverse lookups" list_of_ds = [] resolver = [(self.localOptions['controlresolver'], 53)] - res = client.createResolver(servers=resolver) + res = self.createResolver(servers=resolver) test_reverse = self.reverse_lookup(self.control_a_lookups[0], res) test_reverse.addCallback(self.process_ptr_answers, 'control') diff --git a/ooni/inputunit.py b/ooni/inputunit.py index 157f038..e5b6187 100644 --- a/ooni/inputunit.py +++ b/ooni/inputunit.py @@ -20,7 +20,7 @@ class InputUnitFactory(object): This object is a python iterable, this means that it does not need to keep all the elements in memory to be able to produce InputUnits. """ - inputUnitSize = 3 + inputUnitSize = 10 def __init__(self, inputs=[]): self._inputs = iter(inputs) self._idx = 0 @@ -30,20 +30,22 @@ class InputUnitFactory(object): return self def next(self): + input_unit_elements = [] + if self._ended: raise StopIteration - last_element_idx = self._idx + self.inputUnitSize - input_unit_elements = [] - for i in xrange(last_element_idx): + for i in xrange(self._idx, self._idx + self.inputUnitSize): try: input_unit_elements.append(self._inputs.next()) except: self._ended = True break - self._idx += self.inputUnitSize + if not input_unit_elements: + raise StopIteration + return InputUnit(input_unit_elements) diff --git a/tests/test_inputunit.py b/tests/test_inputunit.py new file mode 100644 index 0000000..f591e23 --- /dev/null +++ b/tests/test_inputunit.py @@ -0,0 +1,18 @@ +import unittest +from ooni.inputunit import InputUnit, InputUnitFactory + +class TestInputUnit(unittest.TestCase): + def test_input_unit_factory(self): + inputs = range(100) + inputUnit = InputUnitFactory(inputs) + for i in inputUnit: + self.assertEqual(len(list(i)), inputUnit.inputUnitSize) + + def test_input_unit(self): + inputs = range(100) + inputUnit = InputUnit(inputs) + idx = 0 + for i in inputUnit: + idx += 1 + + self.assertEqual(idx, 100)

1 0

[ooni-probe/master] Rewrite DNS Tamper test to properly use Twisted DNS client.
by art＠torproject.org 08 Oct '12

08 Oct '12

commit c904e684ac60f9c25c09d52251cb27966aaab065 Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Oct 8 02:25:13 2012 +0000 Rewrite DNS Tamper test to properly use Twisted DNS client. * Should be ready for usage --- nettests/core/dnstamper.py | 182 ++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 182 insertions(+), 0 deletions(-) diff --git a/nettests/core/dnstamper.py b/nettests/core/dnstamper.py new file mode 100644 index 0000000..92948cd --- /dev/null +++ b/nettests/core/dnstamper.py @@ -0,0 +1,182 @@ +# -*- encoding: utf-8 -*- +# +# +# dnstamper +# ********* +# +# The test reports censorship if the cardinality of the intersection of +# the query result set from the control server and the query result set +# from the experimental server is zero, which is to say, if the two sets +# have no matching results whatsoever. +# +# NOTE: This test frequently results in false positives due to GeoIP-based +# load balancing on major global sites such as google, facebook, and +# youtube, etc. +# +# :authors: Arturo Filastò, Isis Lovecruft +# :licence: see LICENSE + +from ooni import nettest +from twisted.internet import defer +from twisted.names import client + +class DNSTamperTest(nettest.TestCase): + + name = "DNS tamper" + + description = "DNS censorship detection test" + + requirements = None + inputFile = ['file', 'f', None, + 'Input file of list of hostnames to attempt to resolve'] + + optParameters = [['controlresolver', 'c', '8.8.8.8', + 'Known good DNS server'], + ['testresolvers', 't', None, + 'file containing list of DNS resolvers to test against'] + ] + + def setUp(self): + self.report['test_lookups'] = {} + self.report['test_reverse'] = {} + + self.report['control_lookup'] = [] + + self.report['a_lookups'] = {} + + self.test_a_lookups = {} + self.control_a_lookups = [] + + self.test_reverse = {} + + if not self.localOptions['testresolvers']: + self.test_resolvers = ['8.8.8.8'] + return + try: + fp = open(self.localOptions['testresolvers']) + except: + raise usage.UsageError("Invalid test resolvers file") + + self.test_resolvers = [x.strip() for x in fp.readlines] + fp.close() + + def process_a_answers(self, answers, resolver): + all_a = [] + a_a = [] + for answer in answers[0]: + if answer.type is 1: + # A type query + r = answer.payload.dottedQuad() + self.report['a_lookups'][resolver] = r + a_a.append(r) + lookup = str(answer.payload) + all_a.append(lookup) + + if resolver == 'control': + self.report['control_server'] = self.localOptions['controlresolver'] + self.report['control_lookup'] = all_a + self.control_a_lookups = a_a + else: + self.test_a_lookups[resolver] = a_a + self.report['test_lookups'][resolver] = all_a + + def process_ptr_answers(self, answers, resolver): + print "Processing PTR ANSWER for %s" % resolver + name = None + for answer in answers[0]: + if answer.type is 12: + # PTR type + name = str(answer.payload.name) + + if resolver == 'control': + self.control_reverse = name + self.report['control_reverse'] = name + else: + self.test_reverse[resolver] = name + self.report['test_reverse'][resolver] = name + + + def ptr_lookup_error(self, failure, resolver): + if resolver == 'control': + self.report['control_reverse'] = None + else: + self.report['test_reverse'][resolver] = None + + def a_lookup_error(self, failure, resolver): + if resolver == 'control': + self.report['control_lookup'] = None + else: + self.report['test_lookups'][resolver] = None + + def test_lookup(self): + list_of_ds = [] + hostname = self.input + + resolver = [(self.localOptions['controlresolver'], 53)] + res = client.createResolver(servers=resolver) + + control_r = res.lookupAllRecords(hostname) + control_r.addCallback(self.process_a_answers, 'control') + control_r.addErrback(self.a_lookup_error, 'control') + + list_of_ds.append(control_r) + + for test_resolver in self.test_resolvers: + resolver = [(test_resolver,53)] + res = client.createResolver(servers=resolver) + + d = res.lookupAddress(hostname) + d.addCallback(self.process_a_answers, test_resolver) + d.addErrback(self.a_lookup_error, test_resolver) + list_of_ds.append(d) + + dl = defer.DeferredList(list_of_ds) + dl.addCallback(self.do_reverse_lookups) + dl.addCallback(self.compare_results) + return dl + + def reverse_lookup(self, address, resolver): + ptr = '.'.join(address.split('.')[::-1]) + '.in-addr.arpa' + r = resolver.lookupPointer(ptr) + return r + + def do_reverse_lookups(self, result): + + + list_of_ds = [] + + resolver = [(self.localOptions['controlresolver'], 53)] + res = client.createResolver(servers=resolver) + + test_reverse = self.reverse_lookup(self.control_a_lookups[0], res) + test_reverse.addCallback(self.process_ptr_answers, 'control') + test_reverse.addErrback(self.ptr_lookup_error, 'control') + + list_of_ds.append(test_reverse) + + for test_resolver in self.test_resolvers: + d = self.reverse_lookup(self.test_a_lookups[test_resolver][0], res) + d.addCallback(self.process_ptr_answers, test_resolver) + d.addErrback(self.ptr_lookup_error, test_resolver) + list_of_ds.append(d) + + dl = defer.DeferredList(list_of_ds) + return dl + + def compare_results(self, *arg, **kw): + for test, test_a_lookups in self.test_a_lookups.items(): + self.report['tampering'] = {} + self.report['tampering'][test] = 'unknown' + + if len(set(test_a_lookups) & set(self.control_a_lookups)) > 0: + # Address has not tampered with on DNS server + self.report['tampering'][test] = False + + elif len(set(self.control_reverse) & set(self.test_reverse[test])) > 0: + # Further testing has eliminated false positives + self.report['tampering'][test] = 'reverse-match' + else: + # Reverse DNS on the results returned by returned + # which does not match the expected domainname + self.report['tampering'][test] = True +

1 0

[ooni-probe/master] Work on input file handling
by art＠torproject.org 07 Oct '12

07 Oct '12

commit 09b3483dd2577e1a78eee47db8d04a3fcb5b1db8 Author: Arturo Filastò <arturo(a)filasto.net> Date: Sun Oct 7 23:35:43 2012 +0000 Work on input file handling --- ooni/runner.py | 12 ++++++------ 1 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ooni/runner.py b/ooni/runner.py index 1a03814..bb63428 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -94,18 +94,18 @@ def adaptLegacyTest(obj, config): return LegacyOONITest def processTest(obj, config): - if obj.optParameters or obj.inputFile: + inputFile = obj.inputFile + + if obj.optParameters or inputFile: if not obj.optParameters: obj.optParameters = [] + if inputFile: + obj.optParameters.append(inputFile) + class Options(usage.Options): optParameters = obj.optParameters - inputFile = obj.inputFile - if inputFile: - Options.optParameters.append(inputFile) - - print Options.optParameters options = Options() options.parseOptions(config['subArgs'])

1 0

[ooni-probe/master] Work on scapy related functionality
by art＠torproject.org 07 Oct '12

07 Oct '12

commit 0f78e3f66aeeb3f830f2ff2b0cd5a465a764ec9e Author: Arturo Filastò <arturo(a)filasto.net> Date: Sun Oct 7 21:57:54 2012 +0000 Work on scapy related functionality * Discover the existence of bugs in scapy when using Arch Linux. (probably has to do with the non existence of ifconfig, somebody should make a ifconfig to ip wrapper that suggests the user with the equivalent ip command. This would assist sysadmins in migrating away from a deprecated tool and persist backward comaptibility) * Fix bugs in runner and nettest --- nettests/examples/example_scapyt.py | 16 ++++++++++++++++ ooni/nettest.py | 2 +- ooni/oonicli.py | 9 +++++++++ ooni/runner.py | 4 +++- ooni/templates/scapyt.py | 22 +++++++--------------- 5 files changed, 36 insertions(+), 17 deletions(-) diff --git a/nettests/examples/example_scapyt.py b/nettests/examples/example_scapyt.py index 1cc8054..403ddc5 100644 --- a/nettests/examples/example_scapyt.py +++ b/nettests/examples/example_scapyt.py @@ -13,3 +13,19 @@ class ExampleScapy(scapyt.ScapyTest): inputs = [IP(dst="8.8.8.8")/TCP(dport=31337), IP(dst="ooni.nu")/TCP(dport=31337)] + + def test_sendReceive(self): + log.msg("Running send receive") + if self.receive: + log.msg("Sending and receiving packets.") + d = self.sendReceivePackets(self.buildPackets()) + else: + log.msg("Sending packets.") + d = self.sendPackets(self.buildPackets()) + + def finished(data): + log.msg("Finished sending") + return data + + d.addCallback(finished) + return diff --git a/ooni/nettest.py b/ooni/nettest.py index 7f9562a..e7ae7d5 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -103,7 +103,7 @@ class TestCase(unittest.TestCase): self.inputs = self.inputProcessor(fp) elif not self.inputs[0]: pass - else: + elif self.inputFile: raise usage.UsageError("No input file specified!") # XXX perhaps we may want to name and version to be inside of a # different object that is not called options. diff --git a/ooni/oonicli.py b/ooni/oonicli.py index af93af9..40c01fd 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -43,6 +43,8 @@ class Options(usage.Options, app.ReactorSelectionMixin): "files listed on the command line") optFlags = [["help", "h"], + ['debug-stacktraces', 'B', + 'Report deferred creation and callback stack traces'], ] optParameters = [ @@ -96,6 +98,13 @@ def run(): except usage.error, ue: raise SystemExit, "%s: %s" % (sys.argv[0], ue) + if config['debug-stacktraces']: + defer.setDebugging(True) + + #logFile = open(config['logfile'], 'w') + #logFileObserver = log.FileLogObserver(logFile) + #log.startLoggingWithObserver(logFileObserver.emit, 0) + classes = runner.findTestClassesFromConfig(config) casesList, options = runner.loadTestsAndOptions(classes) diff --git a/ooni/runner.py b/ooni/runner.py index 0b9503a..1a03814 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -7,6 +7,8 @@ import inspect from twisted.internet import defer, reactor from twisted.python import reflect, failure, usage +from twisted.python import log as tlog + from twisted.trial import unittest from twisted.trial.runner import TrialRunner, TestLoader from twisted.trial.runner import isPackage, isTestCase, ErrorHolder @@ -183,7 +185,7 @@ class ORunner(object): them in input units. I also create all the report instances required to run the tests. """ - def __init__(self, cases, options=None, config=None): + def __init__(self, cases, options=None, config=None, *arg, **kw): self.baseSuite = InputTestSuite self.cases = cases self.options = options diff --git a/ooni/templates/scapyt.py b/ooni/templates/scapyt.py index ac32ad8..b2ac148 100644 --- a/ooni/templates/scapyt.py +++ b/ooni/templates/scapyt.py @@ -31,7 +31,7 @@ class ScapyTest(TestCase): receive = True timeout = 1 - pcapfile = '/home/x/code/networking/ooni-probe/scapytest.pcap' + pcapfile = None input = IP()/TCP() reactor = None def setUp(self): @@ -44,23 +44,15 @@ class ScapyTest(TestCase): def tearDown(self): self.reactor.stop() - def test_sendReceive(self): - log.msg("Running send receive") - if self.receive: - log.msg("Sending and receiving packets.") - d = txsr(self.buildPackets(), pcapfile=self.pcapfile, - timeout=self.timeout, reactor=self.reactor) - else: - log.msg("Sending packets.") - d = txsend(self.buildPackets(), reactor=self.reactor) + def sendReceivePackets(self, packets): + d = txsr(packets, pcapfile=self.pcapfile, + timeout=10, reactor=self.reactor) - def finished(data): - log.msg("Finished sending") - return data - - d.addCallback(finished) return d + def sendPackets(self, packets): + return txsend(self.buildPackets(), reactor=self.reactor) + def buildPackets(self): """ Override this method to build scapy packets.

1 0