September 2016 - tor-commits - lists.torproject.org

[bridgedb/develop] Update stem to latest master.
by isis＠torproject.org 22 Sep '16

22 Sep '16

commit 73c9d3f4bea98245701bb951316881b45a39a874 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Sep 6 18:12:58 2016 +0000 Update stem to latest master. This fixes an issue with stem not correctly parsing IPv6 addresses in extrainfo transport lines, due to the surrounding brackets. * FIXES #20087: https://bugs.torproject.org/20087 --- bridgedb.conf | 4 ++ bridgedb/Main.py | 173 +++++++++++++++++++++++++++----------------------- bridgedb/configure.py | 4 +- requirements.txt | 2 +- 4 files changed, 100 insertions(+), 83 deletions(-) diff --git a/bridgedb.conf b/bridgedb.conf index 52a8ca7..2d73a2b 100644 --- a/bridgedb.conf +++ b/bridgedb.conf @@ -120,6 +120,10 @@ # can read it. #------------------------------------------------------------------------------ +# List of directories which contain versions of the files specified in +# BRIDGE_FILES, EXTRA_INFO_FILES, and STATUS_FILE. +BRIDGE_AUTHORITY_DIRECTORIES = ["from-authority", "from-bifroest"] + # List of filenames from which we read ``@type bridge-server-descriptor``s, on # startup and on SIGHUP. BRIDGE_FILES = ["bridge-descriptors"] diff --git a/bridgedb/Main.py b/bridgedb/Main.py index bf4c213..674b684 100644 --- a/bridgedb/Main.py +++ b/bridgedb/Main.py @@ -41,6 +41,12 @@ from bridgedb import Bridges from bridgedb.Stability import updateBridgeHistory +def expandBridgeAuthDir(state, authdir, filename): + """Expands a descriptor ``filename`` relative to which of the + BRIDGE_AUTHORITY_DIRECTORIES, ``authdir`` it resides within. + """ + return os.path.abspath(os.path.expanduser(os.sep.join([authdir, filename]))) + def load(state, hashring, clear=False): """Read and parse all descriptors, and load into a bridge hashring. @@ -69,95 +75,102 @@ def load(state, hashring, clear=False): if ignoreNetworkstatus: logging.info("Ignoring BridgeAuthority networkstatus documents.") - bridges = {} - timestamps = {} - - logging.info("Opening networkstatus file: %s" % state.STATUS_FILE) - networkstatuses = descriptors.parseNetworkStatusFile(state.STATUS_FILE) - logging.debug("Closing networkstatus file: %s" % state.STATUS_FILE) + for authdir in state.BRIDGE_AUTHORITY_DIRECTORIES: + logging.info("Processing descriptors in %s directory..." % authdir) - logging.info("Processing networkstatus descriptors...") - for router in networkstatuses: - bridge = Bridge() - bridge.updateFromNetworkStatus(router, ignoreNetworkstatus) - try: - bridge.assertOK() - except MalformedBridgeInfo as error: - logging.warn(str(error)) - else: - bridges[bridge.fingerprint] = bridge + bridges = {} + timestamps = {} - for filename in state.BRIDGE_FILES: - logging.info("Opening bridge-server-descriptor file: '%s'" % filename) - serverdescriptors = descriptors.parseServerDescriptorsFile(filename) - logging.debug("Closing bridge-server-descriptor file: '%s'" % filename) + fn = expandBridgeAuthDir(state, authdir, state.STATUS_FILE) + logging.info("Opening networkstatus file: %s" % fn) + networkstatuses = descriptors.parseNetworkStatusFile(fn) + logging.debug("Closing networkstatus file: %s" % fn) - for router in serverdescriptors: + logging.info("Processing networkstatus descriptors...") + for router in networkstatuses: + bridge = Bridge() + bridge.updateFromNetworkStatus(router, ignoreNetworkstatus) try: - bridge = bridges[router.fingerprint] - except KeyError: - logging.warn( - ("Received server descriptor for bridge '%s' which wasn't " - "in the networkstatus!") % router.fingerprint) - if ignoreNetworkstatus: - bridge = Bridge() - else: + bridge.assertOK() + except MalformedBridgeInfo as error: + logging.warn(str(error)) + else: + bridges[bridge.fingerprint] = bridge + + for filename in state.BRIDGE_FILES: + fn = expandBridgeAuthDir(state, authdir, filename) + logging.info("Opening bridge-server-descriptor file: '%s'" % fn) + serverdescriptors = descriptors.parseServerDescriptorsFile(fn) + logging.debug("Closing bridge-server-descriptor file: '%s'" % fn) + + for router in serverdescriptors: + try: + bridge = bridges[router.fingerprint] + except KeyError: + logging.warn( + ("Received server descriptor for bridge '%s' which wasn't " + "in the networkstatus!") % router.fingerprint) + if ignoreNetworkstatus: + bridge = Bridge() + else: + continue + + try: + bridge.updateFromServerDescriptor(router, ignoreNetworkstatus) + except (ServerDescriptorWithoutNetworkstatus, + MissingServerDescriptorDigest, + ServerDescriptorDigestMismatch) as error: + logging.warn(str(error)) + # Reject any routers whose server descriptors didn't pass + # :meth:`~bridges.Bridge._checkServerDescriptor`, i.e. those + # bridges who don't have corresponding networkstatus + # documents, or whose server descriptor digests don't check + # out: + bridges.pop(router.fingerprint) continue + if state.COLLECT_TIMESTAMPS: + # Update timestamps from server descriptors, not from network + # status descriptors (because networkstatus documents and + # descriptors aren't authenticated in any way): + if bridge.fingerprint in timestamps.keys(): + timestamps[bridge.fingerprint].append(router.published) + else: + timestamps[bridge.fingerprint] = [router.published] + + eifiles = [expandBridgeAuthDir(state, authdir, fn) + for fn in state.EXTRA_INFO_FILES] + extrainfos = descriptors.parseExtraInfoFiles(*eifiles) + for fingerprint, router in extrainfos.items(): try: - bridge.updateFromServerDescriptor(router, ignoreNetworkstatus) - except (ServerDescriptorWithoutNetworkstatus, - MissingServerDescriptorDigest, - ServerDescriptorDigestMismatch) as error: + bridges[fingerprint].updateFromExtraInfoDescriptor(router) + except MalformedBridgeInfo as error: logging.warn(str(error)) - # Reject any routers whose server descriptors didn't pass - # :meth:`~bridges.Bridge._checkServerDescriptor`, i.e. those - # bridges who don't have corresponding networkstatus - # documents, or whose server descriptor digests don't check - # out: - bridges.pop(router.fingerprint) - continue - - if state.COLLECT_TIMESTAMPS: - # Update timestamps from server descriptors, not from network - # status descriptors (because networkstatus documents and - # descriptors aren't authenticated in any way): - if bridge.fingerprint in timestamps.keys(): - timestamps[bridge.fingerprint].append(router.published) - else: - timestamps[bridge.fingerprint] = [router.published] - - extrainfos = descriptors.parseExtraInfoFiles(*state.EXTRA_INFO_FILES) - for fingerprint, router in extrainfos.items(): - try: - bridges[fingerprint].updateFromExtraInfoDescriptor(router) - except MalformedBridgeInfo as error: - logging.warn(str(error)) - except KeyError as error: - logging.warn(("Received extrainfo descriptor for bridge '%s', " - "but could not find bridge with that fingerprint.") - % router.fingerprint) - - inserted = 0 - logging.info("Inserting %d bridges into hashring..." % len(bridges)) - for fingerprint, bridge in bridges.items(): - # Skip insertion of bridges which are geolocated to be in one of the - # NO_DISTRIBUTION_COUNTRIES, a.k.a. the countries we don't distribute - # bridges from: - if bridge.country in state.NO_DISTRIBUTION_COUNTRIES: - logging.warn("Not distributing Bridge %s %s:%s in country %s!" % - (bridge, bridge.address, bridge.orPort, bridge.country)) - else: - # If the bridge is not running, then it is skipped during the - # insertion process. - hashring.insert(bridge) - inserted += 1 - logging.info("Done inserting %d bridges into hashring." % inserted) - - if state.COLLECT_TIMESTAMPS: - reactor.callInThread(updateBridgeHistory, bridges, timestamps) + except KeyError as error: + logging.warn(("Received extrainfo descriptor for bridge '%s', " + "but could not find bridge with that fingerprint.") + % router.fingerprint) + + inserted = 0 + logging.info("Inserting %d bridges into hashring..." % len(bridges)) + for fingerprint, bridge in bridges.items(): + # Skip insertion of bridges which are geolocated to be in one of the + # NO_DISTRIBUTION_COUNTRIES, a.k.a. the countries we don't distribute + # bridges from: + if bridge.country in state.NO_DISTRIBUTION_COUNTRIES: + logging.warn("Not distributing Bridge %s %s:%s in country %s!" % + (bridge, bridge.address, bridge.orPort, bridge.country)) + else: + # If the bridge is not running, then it is skipped during the + # insertion process. + hashring.insert(bridge) + inserted += 1 + logging.info("Done inserting %d bridges into hashring." % inserted) + + if state.COLLECT_TIMESTAMPS: + reactor.callInThread(updateBridgeHistory, bridges, timestamps) - state.save() + state.save() def _reloadFn(*args): """Placeholder callback function for :func:`_handleSIGHUP`.""" diff --git a/bridgedb/configure.py b/bridgedb/configure.py index e107a57..6c13595 100644 --- a/bridgedb/configure.py +++ b/bridgedb/configure.py @@ -100,7 +100,7 @@ def loadConfig(configFile=None, configCls=None): # # See :meth:`bridgedb.persistent.State.useUpdatedSettings`. - for attr in ["PROXY_LIST_FILES", "BRIDGE_FILES", "EXTRA_INFO_FILES"]: + for attr in ["PROXY_LIST_FILES"]: setting = getattr(config, attr, None) if setting is None: # pragma: no cover setattr(config, attr, []) # If they weren't set, make them lists @@ -110,7 +110,7 @@ def loadConfig(configFile=None, configCls=None): for attr in ["DB_FILE", "DB_LOG_FILE", "MASTER_KEY_FILE", "PIDFILE", "ASSIGNMENTS_FILE", "HTTPS_CERT_FILE", "HTTPS_KEY_FILE", - "LOG_FILE", "STATUS_FILE", "COUNTRY_BLOCK_FILE", + "LOG_FILE", "COUNTRY_BLOCK_FILE", "GIMP_CAPTCHA_DIR", "GIMP_CAPTCHA_HMAC_KEYFILE", "GIMP_CAPTCHA_RSA_KEYFILE", "EMAIL_GPG_HOMEDIR", "EMAIL_GPG_PASSPHRASE_FILE"]: diff --git a/requirements.txt b/requirements.txt index bd25ae4..26c2c1a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,5 +10,5 @@ gnupg==2.0.1 pillow>=2.6.1 qrcode==5.0.1 service_identity==14.0.0 -git+https://git.torproject.org/stem.git@ba8cee36a0348c1509ad3562051723b7948e19ce#egg=stem-1.4.1.dev0-py2.7 +git+https://git.torproject.org/stem.git@152fa89c33114619913d2be13e67adb4e55ce7ab#egg=stem-1.4.1.dev1-py2.7 zope.interface==3.6.1

1 0

[bridgedb/develop] Add CHANGELOG entries for 0.3.8.
by isis＠torproject.org 22 Sep '16

22 Sep '16

commit 78e352ec18bc55bbb519747a1b1d9e909e3640d7 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Sep 22 12:06:44 2016 +0000 Add CHANGELOG entries for 0.3.8. --- CHANGELOG | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index d2fe18e..c0eb021 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,15 @@ +Changes in version 0.3.8 - 2016-09-22 + + * FIXES #20088 https://bugs.torproject.org/20088 + BridgeDB now supports receiving descriptors from multiple Bridge + Authorities. See also #19690. + + * FIXES #20087 https://bugs.torproject.org/20087 + BridgeDB's version of Stem now supports parsing transport lines in + bridge extrainfo descriptors which contain IPv6 addresses + contained within square brackets. + + Changes in version 0.3.7 - 2016-08-04 * FIXES #19691 https://bugs.torproject.org

1 0

[bridgedb/develop] Merge branch 'fix/20088' into develop
by isis＠torproject.org 22 Sep '16

22 Sep '16

commit 09a62d48ce29e49235ca021dfb03838953eae0af Merge: b98a543 be7550e Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Sep 8 22:27:14 2016 +0000 Merge branch 'fix/20088' into develop bridgedb.conf | 4 ++ bridgedb/Main.py | 172 +++++++++++++++++++++++++++----------------------- bridgedb/configure.py | 4 +- requirements.txt | 2 +- scripts/setup-tests | 6 +- 5 files changed, 103 insertions(+), 85 deletions(-)

1 0

[bridgedb/develop] Merge branch 'release/0.3.8'
by isis＠torproject.org 22 Sep '16

22 Sep '16

commit 9f753b3ccbd159bc3b906875b18a2b4c53aa8e91 Merge: f2fe8fb 78e352e Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Sep 22 12:07:28 2016 +0000 Merge branch 'release/0.3.8' .coveragerc | 1 - .test.requirements.txt | 2 +- .travis.requirements.txt | 4 +- CHANGELOG | 12 +++ Makefile | 15 ++++ bridgedb.conf | 4 + bridgedb/Main.py | 172 +++++++++++++++++++++++-------------------- bridgedb/configure.py | 4 +- requirements.txt | 4 +- scripts/install-dependencies | 12 ++- scripts/setup-tests | 6 +- setup.py | 2 +- 12 files changed, 146 insertions(+), 92 deletions(-)

1 0

[bridgedb/develop] Merge branch 'fix/20087' into develop
by isis＠torproject.org 22 Sep '16

22 Sep '16

commit 121aaca2cb1b6db93fc5512d806cdc27fe303bd4 Merge: 09a62d4 73c9d3f Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Sep 8 23:49:39 2016 +0000 Merge branch 'fix/20087' into develop

1 0

[bridgedb/master] Upgrade ipaddr to version 2.1.11.
by isis＠torproject.org 22 Sep '16

22 Sep '16

commit 835f25099ef2b761bf8b9fdfdc9e1c44de5585ee Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Sep 8 16:45:26 2016 +0000 Upgrade ipaddr to version 2.1.11. --- .travis.requirements.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.requirements.txt b/.travis.requirements.txt index c4f6966..0c1dc95 100644 --- a/.travis.requirements.txt +++ b/.travis.requirements.txt @@ -21,7 +21,7 @@ sure==1.2.2 Babel==0.9.6 BeautifulSoup==3.2.1 Mako==0.8.1 -https://ipaddr-py.googlecode.com/files/ipaddr-2.1.10.tar.gz#sha1=c608450b077b19773d4f1b5f1ef88b26f6650ce0#egg=ipaddr-2.1.10-py2.7 +ipaddr==2.1.11 pycrypto==2.6.1 pygeoip==0.2.7 gnupg==2.0.1 diff --git a/requirements.txt b/requirements.txt index 4591511..bd25ae4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ Babel==0.9.6 BeautifulSoup==3.2.1 Mako==0.8.1 Twisted==14.0.2 -https://ipaddr-py.googlecode.com/files/ipaddr-2.1.10.tar.gz#sha1=c608450b077b19773d4f1b5f1ef88b26f6650ce0#egg=ipaddr-2.1.10-py2.7 +ipaddr==2.1.11 pycrypto==2.6.1 pyOpenSSL==0.14 pygeoip==0.2.7

1 0

[bridgedb/develop] Don't attempt to run coverage with PyPy.
by isis＠torproject.org 22 Sep '16

22 Sep '16

commit b98a543938ce552045a719adf130c93804ceb354 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Sep 8 17:45:53 2016 +0000 Don't attempt to run coverage with PyPy. --- Makefile | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Makefile b/Makefile index 6c20fee..8aa92a3 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,16 @@ TRIAL:=$(shell which trial) VERSION:=$(shell git describe) +define PYTHON_WHICH +import platform +import sys +sys.stdout.write(platform.python_implementation()) +endef + +PYTHON_IMPLEMENTATION:=$(shell python -c '$(PYTHON_WHICH)') +PYTHON_PYTHON=CPython +PYTHON_PYPY=PyPy + all: uninstall clean install coverage-test test: @@ -59,8 +69,13 @@ clean: clean-docs clean-coverage-html -rm -rf _trial_temp coverage-test: +ifeq ($(PYTHON_IMPLEMENTATION),PyPy) + @echo "Detected PyPy... not running coverage." + python setup.py test +else coverage run --rcfile=".coveragerc" $(TRIAL) ./test/test_*.py coverage report --rcfile=".coveragerc" +endif coverage-html: coverage html --rcfile=".coveragerc"

1 0

[bridgedb/develop] Merge branch 'master' into develop
by isis＠torproject.org 22 Sep '16

22 Sep '16

commit a333183218e4239f2d7bebeb7a3a988bc4990a6c Merge: 78e352e 9f753b3 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Sep 22 12:08:16 2016 +0000 Merge branch 'master' into develop

1 0

[bridgedb/develop] Create run/from-{authority, bifroest} directories during CI test runs.
by isis＠torproject.org 22 Sep '16

22 Sep '16

commit be7550e31fb2f76bca659319abc3b560ef472581 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Sep 8 17:52:15 2016 +0000 Create run/from-{authority,bifroest} directories during CI test runs. --- scripts/setup-tests | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/setup-tests b/scripts/setup-tests index ec915d8..8e42762 100755 --- a/scripts/setup-tests +++ b/scripts/setup-tests @@ -11,7 +11,8 @@ THERE="$HERE" echo "Found top-level repo directory: $THERE" cd $THERE -mkdir run +mkdir -p run/from-authority +mkdir -p run/from-bifroest cp -R -t run bridgedb.conf captchas .gnupg # Add '127.0.0.1' to EMAIL_DOMAINS in bridgedb.conf. This should ONLY be # done on testing servers, never on production servers. @@ -25,6 +26,7 @@ sed -r -i -e "s/(HTTP_UNENCRYPTED_BIND_IP = )(None)/\1'127.0.0.1'/" run/bridgedb sed -r -i -e "s/(HTTP_UNENCRYPTED_PORT = )(None)/\16788/" run/bridgedb.conf sed -r -i -e "s/(SERVER_PUBLIC_FQDN = )(.*)/\1'127.0.0.1:6788'/" run/bridgedb.conf leekspin -n 100 -cp -t run networkstatus-bridges cached-extrainfo* bridge-descriptors +cp -t run/from-authority networkstatus-bridges cached-extrainfo* bridge-descriptors +cp -t run/from-bifroest networkstatus-bridges cached-extrainfo* bridge-descriptors ./scripts/make-ssl-cert cp -t run privkey.pem cert

1 0

[bridgedb/master] Merge branch 'master' into develop
by isis＠torproject.org 22 Sep '16

22 Sep '16

commit 926a9899c1a1f18649838c857291d4e6daaff746 Merge: f1267fc f2fe8fb Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon Aug 8 15:23:04 2016 +0000 Merge branch 'master' into develop

1 0