August 2019 - tor-commits - lists.torproject.org

[bridgedb/develop] Merge branch 'master' into develop
by phw＠torproject.org 20 Aug '19

20 Aug '19

commit d67abc72951e4448275778da58e6d445cb6cd0ef Merge: 8291822 bbf1bdc Author: Philipp Winter <phw(a)nymity.ch> Date: Tue Aug 20 09:51:44 2019 -0700 Merge branch 'master' into develop CHANGELOG | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

1 0

[bridgedb/develop] Bump version number to 0.8.0.
by phw＠torproject.org 20 Aug '19

20 Aug '19

commit 7df8ccc7d07d503dd46df0dd8808f7ab555c715a Author: Philipp Winter <phw(a)nymity.ch> Date: Tue Aug 20 09:48:17 2019 -0700 Bump version number to 0.8.0. --- CHANGELOG | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index 03390d6..aba6d19 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,4 +1,4 @@ -Changes in version 0.8.0 - YYYY-MM-DD +Changes in version 0.8.0 - 2019-08-20 * FIXES https://bugs.torproject.org/9316 Make BridgeDB export usage metrics every 24 hours. At the end of each

1 0

[bridgedb/master] Add debug message.
by phw＠torproject.org 20 Aug '19

20 Aug '19

commit 4302c6792b07ae1fef4886ec9765d13425864037 Author: Philipp Winter <phw(a)nymity.ch> Date: Wed Jun 26 10:49:45 2019 -0700 Add debug message. This helps with debugging <https://bugs.torproject.org/30441>. --- bridgedb/Bridges.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bridgedb/Bridges.py b/bridgedb/Bridges.py index 3b1e5b5..3c94fdf 100644 --- a/bridgedb/Bridges.py +++ b/bridgedb/Bridges.py @@ -344,6 +344,8 @@ class BridgeRing(object): bridges = [self.bridges[k] for k in keys] bridges = bridges[:N] + logging.debug("Caller asked for N=%d, filterBySubnet=%s bridges. " + "Returning %d bridges." % (N, filterBySubnet, len(bridges))) return bridges

1 0

[bridgedb/master] Merge branch 'master' into develop
by phw＠torproject.org 20 Aug '19

20 Aug '19

commit db2aaadcf22edc3485deeb91fbb5931e9c53c593 Merge: 589d6f7 2d86ae0 Author: Philipp Winter <phw(a)nymity.ch> Date: Fri Jun 7 10:38:10 2019 -0700 Merge branch 'master' into develop CHANGELOG | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

1 0

[bridgedb/develop] Merge branch 'release-0.8.0'
by phw＠torproject.org 20 Aug '19

20 Aug '19

1 0

[bridgedb/master] Remove outdated documentation.
by phw＠torproject.org 20 Aug '19

20 Aug '19

commit 21807e6e45cb99821c56d5c19755dd5813bce57b Author: Philipp Winter <phw(a)nymity.ch> Date: Wed Jul 24 10:40:16 2019 -0700 Remove outdated documentation. BridgeDB no longer has a single handler for SIGUSR1. It was removed in commit f4d55919. As a result, if you now send a SIGUSR1 to BridgeDB, it reacts with the default action, which is termination. --- README.rst | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/README.rst b/README.rst index df90903..ee1c979 100644 --- a/README.rst +++ b/README.rst @@ -362,16 +362,6 @@ Or just give it a SIGHUP:: kill -s SIGHUP `cat .../run/bridgedb.pid` ----------------------------------- -To extract all bridge assignments: ----------------------------------- - -To dump all bridge assignments to files, send BridgeDB a ``SIGUSR1`` -signal by doing:: - - kill -s SIGUSR1 `cat .../run/bridgedb.pid` - - ========================= Using a BridgeDB Instance =========================

1 0

[bridgedb/master] Add version filters for IPv6 vanilla bridges.
by phw＠torproject.org 20 Aug '19

20 Aug '19

commit 594589ec2e7e1893c5e4dc48385f573c09de6f0d Author: Philipp Winter <phw(a)nymity.ch> Date: Tue Aug 6 11:19:41 2019 -0700 Add version filters for IPv6 vanilla bridges. So far, BridgeDB's distribution of vanilla IPv6 bridges was broken. When a user would request one, BridgeDB would use the following two filters to select bridges: * byTransportNotBlockedIn(None,us,6) * byProbingResistance(vanilla,6) Neither filter (correctly) filtered for IPv6. It is not enough to check bridge.address because it's the bridge's IPv4 address. We need to check bridge.allVanillaAddresses because a bridge's IPv6 address is advertised in its "a" consensus line. Note that this patch only fixes IPv6 distribution for *vanilla* bridges. Pluggable transports bridges are currently unable to have dual-stack support for both IPv4 and IPv6. See the following ticket for more details: <https://bugs.torproject.org/11211> This fixes <https://bugs.torproject.org/26542>. --- CHANGELOG | 5 +++++ bridgedb/bridgerequest.py | 2 +- bridgedb/filters.py | 10 +++++++++- bridgedb/test/test_distributors_moat_request.py | 6 +++--- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 1ebd47e..1229578 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,8 @@ +Changes in version 0.7.2 - YYYY-MM-DD + + * FIXES #26542 https://bugs.torproject.org/26542 + Make BridgeDB distribute vanilla IPv6 bridges again. + Changes in version 0.7.1 - 2019-06-07 * FIXES #28496 https://bugs.torproject.org/28496 diff --git a/bridgedb/bridgerequest.py b/bridgedb/bridgerequest.py index bf64069..7bd7b64 100644 --- a/bridgedb/bridgerequest.py +++ b/bridgedb/bridgerequest.py @@ -249,7 +249,7 @@ class BridgeRequestBase(object): for country in self.notBlockedIn: logging.info("%s %s bridges not blocked in %s..." % (msg, pt or "vanilla", country)) - self.addFilter(byNotBlockedIn(country, pt, self.ipVersion)) + self.addFilter(byNotBlockedIn(country, pt or "vanilla", self.ipVersion)) elif pt: logging.info("%s %s bridges..." % (msg, pt)) self.addFilter(byTransport(pt, self.ipVersion)) diff --git a/bridgedb/filters.py b/bridgedb/filters.py index f268c83..a02661e 100644 --- a/bridgedb/filters.py +++ b/bridgedb/filters.py @@ -151,6 +151,13 @@ def byProbingResistance(methodname=None, ipVersion=None): return _cache[name] except KeyError: def _byProbingResistance(bridge): + # If we're dealing with a vanilla bridge, make sure that the bridge + # has the correct IP version. + if methodname == "vanilla": + validVersion = byIPv(ipVersion) + if not validVersion(bridge): + return False + if bridge.hasProbingResistantPT(): return methodname in ('scramblesuit', 'obfs4') return True @@ -262,7 +269,8 @@ def byNotBlockedIn(countryCode=None, methodname=None, ipVersion=4): if not methodname: return not bridge.isBlockedIn(countryCode) elif methodname == "vanilla": - if bridge.address.version == ipVersion: + validVersion = byIPv(ipVersion) + if validVersion(bridge): if not bridge.addressIsBlockedIn(countryCode, bridge.address, bridge.orPort): diff --git a/bridgedb/test/test_distributors_moat_request.py b/bridgedb/test/test_distributors_moat_request.py index 3c6ff51..555b3eb 100644 --- a/bridgedb/test/test_distributors_moat_request.py +++ b/bridgedb/test/test_distributors_moat_request.py @@ -30,9 +30,9 @@ class MoatBridgeRequest(unittest.TestCase): self.bridgeRequest.withoutBlockInCountry(data) self.bridgeRequest.generateFilters() - self.assertItemsEqual(['byTransportNotBlockedIn(None,us,4)', - 'byTransportNotBlockedIn(None,ir,4)', - 'byTransportNotBlockedIn(None,sy,4)', + self.assertItemsEqual(['byTransportNotBlockedIn(vanilla,us,4)', + 'byTransportNotBlockedIn(vanilla,ir,4)', + 'byTransportNotBlockedIn(vanilla,sy,4)', 'byProbingResistance(vanilla,4)'], [x.__name__ for x in self.bridgeRequest.filters])

1 0

[bridgedb/master] Merge branch 'fix/26542' into develop
by phw＠torproject.org 20 Aug '19

20 Aug '19

commit 081ff367458db9d9081035e63a9858df27932041 Merge: 21807e6 8894939 Author: Philipp Winter <phw(a)nymity.ch> Date: Sun Aug 11 19:30:13 2019 -0700 Merge branch 'fix/26542' into develop CHANGELOG | 5 +++++ bridgedb/bridgerequest.py | 2 +- bridgedb/filters.py | 10 +++++++++- bridgedb/test/test_distributors_moat_request.py | 6 +++--- scripts/install-dependencies | 11 +---------- 5 files changed, 19 insertions(+), 15 deletions(-)

1 0

[bridgedb/master] Update PIP_FLAGS to fix Travis CI update.
by phw＠torproject.org 20 Aug '19

20 Aug '19

commit 8894939d5da6a07850ff238d1453601d41f178f4 Author: Philipp Winter <phw(a)nymity.ch> Date: Tue Aug 6 11:40:38 2019 -0700 Update PIP_FLAGS to fix Travis CI update. --- scripts/install-dependencies | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/scripts/install-dependencies b/scripts/install-dependencies index 189cf53..cffef18 100755 --- a/scripts/install-dependencies +++ b/scripts/install-dependencies @@ -6,16 +6,7 @@ SUDO= APT_GET=$(which apt-get) APT_FLAGS='-q --no-install-suggests --no-install-recommends' PIP=$(which pip) -PIP_FLAGS= - -# If this doesn't work, then the version of pip is very old and we need to use --no-use-wheel -$PIP --no-binaries :all: - -if test "$?" -eq 2 ; then - PIP_FLAGS=$PIP_FLAGS' --no-use-wheel' -else - PIP_FLAGS=$PIP_FLAGS' --no-binaries :all:' -fi +PIP_FLAGS='--no-binary :all' DEPENDS="build-essential openssl sqlite3 python-dev python-setuptools" DEPENDS="${DEPENDS} libgeoip-dev geoip-database libjpeg-dev"

1 0

[bridgedb/master] Fix broken download of Tor exit relays.
by phw＠torproject.org 20 Aug '19

20 Aug '19

commit 0d5ed52e5906260e142ef9cfa12752810fd1ffa2 Author: Philipp Winter <phw(a)nymity.ch> Date: Mon Aug 12 13:52:10 2019 -0700 Fix broken download of Tor exit relays. The periodic download of Tor exit relays was broken for a number of reasons. Here's what we're doing to fix this issue: 1. We're moving the proxies variable outside of state. The problem is that the state object is written to disk and reloaded every 30 minutes (a cron job is triggering this reload by running ~/bridgedb-admin/reload-bridgedb). The reload causes state.proxies to be at a different memory address than before the reload, which breaks the looping call that fetches new exit relays every three hours. This looping call expects to write exit relays to the same memory address each time, but after BridgeDB's first reload, the memory address changed, so exit relays are no longer updated. There's no need to keep our proxies in BridgeDB's state. We fetch them continuously anyway, and also right after BridgeDB starts. 2. We're adding the method replaceExitRelays(). Once we have a new batch of exit relay addresses, this method allows us to completely overwrite the past batch. 3. We're adding the argument "setStdout=False" to the call to startLogging() because otherwise we're missing the download script's output. --- bridgedb/main.py | 7 +++---- bridgedb/proxy.py | 15 +++++++++++++-- scripts/get-tor-exits | 2 +- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/bridgedb/main.py b/bridgedb/main.py index c1e6250..1a617b2 100644 --- a/bridgedb/main.py +++ b/bridgedb/main.py @@ -359,7 +359,6 @@ def run(options, reactor=reactor): moatDistributor = None # Save our state - state.proxies = proxies state.key = key state.save() @@ -411,13 +410,13 @@ def run(options, reactor=reactor): logging.info("Reloading the list of open proxies...") for proxyfile in cfg.PROXY_LIST_FILES: logging.info("Loading proxies from: %s" % proxyfile) - proxy.loadProxiesFromFile(proxyfile, state.proxies, removeStale=True) + proxy.loadProxiesFromFile(proxyfile, proxies, removeStale=True) logging.info("Reparsing bridge descriptors...") (hashring, emailDistributorTmp, ipDistributorTmp, - moatDistributorTmp) = createBridgeRings(cfg, state.proxies, key) + moatDistributorTmp) = createBridgeRings(cfg, proxies, key) logging.info("Bridges loaded: %d" % len(hashring)) # Initialize our DB. @@ -483,7 +482,7 @@ def run(options, reactor=reactor): if config.TASKS['GET_TOR_EXIT_LIST']: tasks['GET_TOR_EXIT_LIST'] = task.LoopingCall( proxy.downloadTorExits, - state.proxies, + proxies, config.SERVER_PUBLIC_EXTERNAL_IP) if config.TASKS.get('DELETE_UNPARSEABLE_DESCRIPTORS'): diff --git a/bridgedb/proxy.py b/bridgedb/proxy.py index 39cd109..6d93c48 100644 --- a/bridgedb/proxy.py +++ b/bridgedb/proxy.py @@ -51,7 +51,7 @@ def downloadTorExits(proxyList, ipaddress, port=443, protocol=None): """ proto = ExitListProtocol() if protocol is None else protocol() args = [proto.script, '--stdout', '-a', ipaddress, '-p', str(port)] - proto.deferred.addCallback(proxyList.addExitRelays) + proto.deferred.addCallback(proxyList.replaceExitRelays) proto.deferred.addErrback(logging.exception) transport = reactor.spawnProcess(proto, proto.script, args=args, env={}) return proto.deferred @@ -76,7 +76,7 @@ def loadProxiesFromFile(filename, proxySet=None, removeStale=False): :returns: A list of all the proxies listed in the **files* (regardless of whether they were added or removed). """ - logging.info("Reloading proxy lists...") + logging.info("Reloading proxy lists from file %s" % filename) addresses = [] @@ -256,6 +256,17 @@ class ProxySet(MutableSet): logging.info("Loading exit relays into proxy list...") [self.add(x, self._exitTag) for x in relays] + def replaceExitRelays(self, relays): + existingExitRelays = self.getAllWithTag(self._exitTag) + logging.debug("Replacing %d existing with %d new exit relays." % + (len(existingExitRelays), len(relays))) + + for relay in existingExitRelays: + self.discard(relay) + + self.addExitRelays(relays) + + def getTag(self, ip): """Get the tag for an **ip** in this ``ProxySet``, if available. diff --git a/scripts/get-tor-exits b/scripts/get-tor-exits index fcd9fff..6ab2201 100755 --- a/scripts/get-tor-exits +++ b/scripts/get-tor-exits @@ -36,7 +36,7 @@ from twisted.internet.error import DNSLookupError from twisted.internet.error import TimeoutError -log.startLogging(sys.stderr) +log.startLogging(sys.stderr, setStdout=False) def backupFile(filename):

1 0