September 2016 - tor-commits - lists.torproject.org

[bridgedb/master] Upgrade coverage to version 4.2.
by isis＠torproject.org 22 Sep '16

22 Sep '16

commit 9634930606dbf1c07a5d9a370bf80b62b3631029 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Sep 8 16:51:56 2016 +0000 Upgrade coverage to version 4.2. --- .test.requirements.txt | 2 +- .travis.requirements.txt | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.test.requirements.txt b/.test.requirements.txt index bd4e741..f48bba3 100644 --- a/.test.requirements.txt +++ b/.test.requirements.txt @@ -5,7 +5,7 @@ # $ pip install -r .test.requirements.txt # $ make coverage # -coverage==3.7.1 +coverage==4.2 leekspin==1.1.4 mechanize==0.2.5 pep8==1.5.7 diff --git a/.travis.requirements.txt b/.travis.requirements.txt index ebda8e0..c4f6966 100644 --- a/.travis.requirements.txt +++ b/.travis.requirements.txt @@ -13,7 +13,7 @@ # $ make coverage # #------------------------------------------------------------------------------ -coverage==3.7.1 +coverage==4.2 coveralls==0.4.2 leekspin==1.1.4 mechanize==0.2.5 diff --git a/setup.py b/setup.py index 189d5b5..97dee01 100644 --- a/setup.py +++ b/setup.py @@ -381,7 +381,7 @@ setuptools.setup( scripts=['scripts/bridgedb', 'scripts/get-tor-exits'], extras_require={'test': ["sure==1.2.2", - "coverage==3.7.1", + "coverage==4.2", "leekspin==1.1.4"]}, zip_safe=False, cmdclass=get_cmdclass(),

1 0

[bridgedb/master] Create run/from-{authority, bifroest} directories during CI test runs.
by isis＠torproject.org 22 Sep '16

22 Sep '16

commit be7550e31fb2f76bca659319abc3b560ef472581 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Sep 8 17:52:15 2016 +0000 Create run/from-{authority,bifroest} directories during CI test runs. --- scripts/setup-tests | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/setup-tests b/scripts/setup-tests index ec915d8..8e42762 100755 --- a/scripts/setup-tests +++ b/scripts/setup-tests @@ -11,7 +11,8 @@ THERE="$HERE" echo "Found top-level repo directory: $THERE" cd $THERE -mkdir run +mkdir -p run/from-authority +mkdir -p run/from-bifroest cp -R -t run bridgedb.conf captchas .gnupg # Add '127.0.0.1' to EMAIL_DOMAINS in bridgedb.conf. This should ONLY be # done on testing servers, never on production servers. @@ -25,6 +26,7 @@ sed -r -i -e "s/(HTTP_UNENCRYPTED_BIND_IP = )(None)/\1'127.0.0.1'/" run/bridgedb sed -r -i -e "s/(HTTP_UNENCRYPTED_PORT = )(None)/\16788/" run/bridgedb.conf sed -r -i -e "s/(SERVER_PUBLIC_FQDN = )(.*)/\1'127.0.0.1:6788'/" run/bridgedb.conf leekspin -n 100 -cp -t run networkstatus-bridges cached-extrainfo* bridge-descriptors +cp -t run/from-authority networkstatus-bridges cached-extrainfo* bridge-descriptors +cp -t run/from-bifroest networkstatus-bridges cached-extrainfo* bridge-descriptors ./scripts/make-ssl-cert cp -t run privkey.pem cert

1 0

[bridgedb/master] Use pip --no-binaries :all: for newer versions of pip.
by isis＠torproject.org 22 Sep '16

22 Sep '16

commit 09e6cb3577a06385f5350ba0d5187cb3914e01f3 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Sep 8 16:56:27 2016 +0000 Use pip --no-binaries :all: for newer versions of pip. --- scripts/install-dependencies | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/scripts/install-dependencies b/scripts/install-dependencies index f9a3a15..189cf53 100755 --- a/scripts/install-dependencies +++ b/scripts/install-dependencies @@ -6,7 +6,17 @@ SUDO= APT_GET=$(which apt-get) APT_FLAGS='-q --no-install-suggests --no-install-recommends' PIP=$(which pip) -PIP_FLAGS='--no-use-wheel' +PIP_FLAGS= + +# If this doesn't work, then the version of pip is very old and we need to use --no-use-wheel +$PIP --no-binaries :all: + +if test "$?" -eq 2 ; then + PIP_FLAGS=$PIP_FLAGS' --no-use-wheel' +else + PIP_FLAGS=$PIP_FLAGS' --no-binaries :all:' +fi + DEPENDS="build-essential openssl sqlite3 python-dev python-setuptools" DEPENDS="${DEPENDS} libgeoip-dev geoip-database libjpeg-dev" HERE=$(dirname $0)

1 0

[bridgedb/master] Merge branch 'release/0.3.8'
by isis＠torproject.org 22 Sep '16

22 Sep '16

commit 9f753b3ccbd159bc3b906875b18a2b4c53aa8e91 Merge: f2fe8fb 78e352e Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Sep 22 12:07:28 2016 +0000 Merge branch 'release/0.3.8' .coveragerc | 1 - .test.requirements.txt | 2 +- .travis.requirements.txt | 4 +- CHANGELOG | 12 +++ Makefile | 15 ++++ bridgedb.conf | 4 + bridgedb/Main.py | 172 +++++++++++++++++++++++-------------------- bridgedb/configure.py | 4 +- requirements.txt | 4 +- scripts/install-dependencies | 12 ++- scripts/setup-tests | 6 +- setup.py | 2 +- 12 files changed, 146 insertions(+), 92 deletions(-)

1 0

[bridgedb/master] Merge branch 'fix/20088' into develop
by isis＠torproject.org 22 Sep '16

22 Sep '16

commit 09a62d48ce29e49235ca021dfb03838953eae0af Merge: b98a543 be7550e Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Sep 8 22:27:14 2016 +0000 Merge branch 'fix/20088' into develop bridgedb.conf | 4 ++ bridgedb/Main.py | 172 +++++++++++++++++++++++++++----------------------- bridgedb/configure.py | 4 +- requirements.txt | 2 +- scripts/setup-tests | 6 +- 5 files changed, 103 insertions(+), 85 deletions(-)

1 0

[bridgedb/master] Merge branch 'fix/20087' into develop
by isis＠torproject.org 22 Sep '16

22 Sep '16

commit 121aaca2cb1b6db93fc5512d806cdc27fe303bd4 Merge: 09a62d4 73c9d3f Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Sep 8 23:49:39 2016 +0000 Merge branch 'fix/20087' into develop

1 0

[bridgedb/master] Don't attempt to run coverage with PyPy.
by isis＠torproject.org 22 Sep '16

22 Sep '16

commit b98a543938ce552045a719adf130c93804ceb354 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Sep 8 17:45:53 2016 +0000 Don't attempt to run coverage with PyPy. --- Makefile | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Makefile b/Makefile index 6c20fee..8aa92a3 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,16 @@ TRIAL:=$(shell which trial) VERSION:=$(shell git describe) +define PYTHON_WHICH +import platform +import sys +sys.stdout.write(platform.python_implementation()) +endef + +PYTHON_IMPLEMENTATION:=$(shell python -c '$(PYTHON_WHICH)') +PYTHON_PYTHON=CPython +PYTHON_PYPY=PyPy + all: uninstall clean install coverage-test test: @@ -59,8 +69,13 @@ clean: clean-docs clean-coverage-html -rm -rf _trial_temp coverage-test: +ifeq ($(PYTHON_IMPLEMENTATION),PyPy) + @echo "Detected PyPy... not running coverage." + python setup.py test +else coverage run --rcfile=".coveragerc" $(TRIAL) ./test/test_*.py coverage report --rcfile=".coveragerc" +endif coverage-html: coverage html --rcfile=".coveragerc"

1 0

[bridgedb/master] Add CHANGELOG entries for 0.3.8.
by isis＠torproject.org 22 Sep '16

22 Sep '16

commit 78e352ec18bc55bbb519747a1b1d9e909e3640d7 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Sep 22 12:06:44 2016 +0000 Add CHANGELOG entries for 0.3.8. --- CHANGELOG | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index d2fe18e..c0eb021 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,15 @@ +Changes in version 0.3.8 - 2016-09-22 + + * FIXES #20088 https://bugs.torproject.org/20088 + BridgeDB now supports receiving descriptors from multiple Bridge + Authorities. See also #19690. + + * FIXES #20087 https://bugs.torproject.org/20087 + BridgeDB's version of Stem now supports parsing transport lines in + bridge extrainfo descriptors which contain IPv6 addresses + contained within square brackets. + + Changes in version 0.3.7 - 2016-08-04 * FIXES #19691 https://bugs.torproject.org

1 0

[bridgedb/master] Update stem to latest master.
by isis＠torproject.org 22 Sep '16

22 Sep '16

commit 73c9d3f4bea98245701bb951316881b45a39a874 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Sep 6 18:12:58 2016 +0000 Update stem to latest master. This fixes an issue with stem not correctly parsing IPv6 addresses in extrainfo transport lines, due to the surrounding brackets. * FIXES #20087: https://bugs.torproject.org/20087 --- bridgedb.conf | 4 ++ bridgedb/Main.py | 173 +++++++++++++++++++++++++++----------------------- bridgedb/configure.py | 4 +- requirements.txt | 2 +- 4 files changed, 100 insertions(+), 83 deletions(-) diff --git a/bridgedb.conf b/bridgedb.conf index 52a8ca7..2d73a2b 100644 --- a/bridgedb.conf +++ b/bridgedb.conf @@ -120,6 +120,10 @@ # can read it. #------------------------------------------------------------------------------ +# List of directories which contain versions of the files specified in +# BRIDGE_FILES, EXTRA_INFO_FILES, and STATUS_FILE. +BRIDGE_AUTHORITY_DIRECTORIES = ["from-authority", "from-bifroest"] + # List of filenames from which we read ``@type bridge-server-descriptor``s, on # startup and on SIGHUP. BRIDGE_FILES = ["bridge-descriptors"] diff --git a/bridgedb/Main.py b/bridgedb/Main.py index bf4c213..674b684 100644 --- a/bridgedb/Main.py +++ b/bridgedb/Main.py @@ -41,6 +41,12 @@ from bridgedb import Bridges from bridgedb.Stability import updateBridgeHistory +def expandBridgeAuthDir(state, authdir, filename): + """Expands a descriptor ``filename`` relative to which of the + BRIDGE_AUTHORITY_DIRECTORIES, ``authdir`` it resides within. + """ + return os.path.abspath(os.path.expanduser(os.sep.join([authdir, filename]))) + def load(state, hashring, clear=False): """Read and parse all descriptors, and load into a bridge hashring. @@ -69,95 +75,102 @@ def load(state, hashring, clear=False): if ignoreNetworkstatus: logging.info("Ignoring BridgeAuthority networkstatus documents.") - bridges = {} - timestamps = {} - - logging.info("Opening networkstatus file: %s" % state.STATUS_FILE) - networkstatuses = descriptors.parseNetworkStatusFile(state.STATUS_FILE) - logging.debug("Closing networkstatus file: %s" % state.STATUS_FILE) + for authdir in state.BRIDGE_AUTHORITY_DIRECTORIES: + logging.info("Processing descriptors in %s directory..." % authdir) - logging.info("Processing networkstatus descriptors...") - for router in networkstatuses: - bridge = Bridge() - bridge.updateFromNetworkStatus(router, ignoreNetworkstatus) - try: - bridge.assertOK() - except MalformedBridgeInfo as error: - logging.warn(str(error)) - else: - bridges[bridge.fingerprint] = bridge + bridges = {} + timestamps = {} - for filename in state.BRIDGE_FILES: - logging.info("Opening bridge-server-descriptor file: '%s'" % filename) - serverdescriptors = descriptors.parseServerDescriptorsFile(filename) - logging.debug("Closing bridge-server-descriptor file: '%s'" % filename) + fn = expandBridgeAuthDir(state, authdir, state.STATUS_FILE) + logging.info("Opening networkstatus file: %s" % fn) + networkstatuses = descriptors.parseNetworkStatusFile(fn) + logging.debug("Closing networkstatus file: %s" % fn) - for router in serverdescriptors: + logging.info("Processing networkstatus descriptors...") + for router in networkstatuses: + bridge = Bridge() + bridge.updateFromNetworkStatus(router, ignoreNetworkstatus) try: - bridge = bridges[router.fingerprint] - except KeyError: - logging.warn( - ("Received server descriptor for bridge '%s' which wasn't " - "in the networkstatus!") % router.fingerprint) - if ignoreNetworkstatus: - bridge = Bridge() - else: + bridge.assertOK() + except MalformedBridgeInfo as error: + logging.warn(str(error)) + else: + bridges[bridge.fingerprint] = bridge + + for filename in state.BRIDGE_FILES: + fn = expandBridgeAuthDir(state, authdir, filename) + logging.info("Opening bridge-server-descriptor file: '%s'" % fn) + serverdescriptors = descriptors.parseServerDescriptorsFile(fn) + logging.debug("Closing bridge-server-descriptor file: '%s'" % fn) + + for router in serverdescriptors: + try: + bridge = bridges[router.fingerprint] + except KeyError: + logging.warn( + ("Received server descriptor for bridge '%s' which wasn't " + "in the networkstatus!") % router.fingerprint) + if ignoreNetworkstatus: + bridge = Bridge() + else: + continue + + try: + bridge.updateFromServerDescriptor(router, ignoreNetworkstatus) + except (ServerDescriptorWithoutNetworkstatus, + MissingServerDescriptorDigest, + ServerDescriptorDigestMismatch) as error: + logging.warn(str(error)) + # Reject any routers whose server descriptors didn't pass + # :meth:`~bridges.Bridge._checkServerDescriptor`, i.e. those + # bridges who don't have corresponding networkstatus + # documents, or whose server descriptor digests don't check + # out: + bridges.pop(router.fingerprint) continue + if state.COLLECT_TIMESTAMPS: + # Update timestamps from server descriptors, not from network + # status descriptors (because networkstatus documents and + # descriptors aren't authenticated in any way): + if bridge.fingerprint in timestamps.keys(): + timestamps[bridge.fingerprint].append(router.published) + else: + timestamps[bridge.fingerprint] = [router.published] + + eifiles = [expandBridgeAuthDir(state, authdir, fn) + for fn in state.EXTRA_INFO_FILES] + extrainfos = descriptors.parseExtraInfoFiles(*eifiles) + for fingerprint, router in extrainfos.items(): try: - bridge.updateFromServerDescriptor(router, ignoreNetworkstatus) - except (ServerDescriptorWithoutNetworkstatus, - MissingServerDescriptorDigest, - ServerDescriptorDigestMismatch) as error: + bridges[fingerprint].updateFromExtraInfoDescriptor(router) + except MalformedBridgeInfo as error: logging.warn(str(error)) - # Reject any routers whose server descriptors didn't pass - # :meth:`~bridges.Bridge._checkServerDescriptor`, i.e. those - # bridges who don't have corresponding networkstatus - # documents, or whose server descriptor digests don't check - # out: - bridges.pop(router.fingerprint) - continue - - if state.COLLECT_TIMESTAMPS: - # Update timestamps from server descriptors, not from network - # status descriptors (because networkstatus documents and - # descriptors aren't authenticated in any way): - if bridge.fingerprint in timestamps.keys(): - timestamps[bridge.fingerprint].append(router.published) - else: - timestamps[bridge.fingerprint] = [router.published] - - extrainfos = descriptors.parseExtraInfoFiles(*state.EXTRA_INFO_FILES) - for fingerprint, router in extrainfos.items(): - try: - bridges[fingerprint].updateFromExtraInfoDescriptor(router) - except MalformedBridgeInfo as error: - logging.warn(str(error)) - except KeyError as error: - logging.warn(("Received extrainfo descriptor for bridge '%s', " - "but could not find bridge with that fingerprint.") - % router.fingerprint) - - inserted = 0 - logging.info("Inserting %d bridges into hashring..." % len(bridges)) - for fingerprint, bridge in bridges.items(): - # Skip insertion of bridges which are geolocated to be in one of the - # NO_DISTRIBUTION_COUNTRIES, a.k.a. the countries we don't distribute - # bridges from: - if bridge.country in state.NO_DISTRIBUTION_COUNTRIES: - logging.warn("Not distributing Bridge %s %s:%s in country %s!" % - (bridge, bridge.address, bridge.orPort, bridge.country)) - else: - # If the bridge is not running, then it is skipped during the - # insertion process. - hashring.insert(bridge) - inserted += 1 - logging.info("Done inserting %d bridges into hashring." % inserted) - - if state.COLLECT_TIMESTAMPS: - reactor.callInThread(updateBridgeHistory, bridges, timestamps) + except KeyError as error: + logging.warn(("Received extrainfo descriptor for bridge '%s', " + "but could not find bridge with that fingerprint.") + % router.fingerprint) + + inserted = 0 + logging.info("Inserting %d bridges into hashring..." % len(bridges)) + for fingerprint, bridge in bridges.items(): + # Skip insertion of bridges which are geolocated to be in one of the + # NO_DISTRIBUTION_COUNTRIES, a.k.a. the countries we don't distribute + # bridges from: + if bridge.country in state.NO_DISTRIBUTION_COUNTRIES: + logging.warn("Not distributing Bridge %s %s:%s in country %s!" % + (bridge, bridge.address, bridge.orPort, bridge.country)) + else: + # If the bridge is not running, then it is skipped during the + # insertion process. + hashring.insert(bridge) + inserted += 1 + logging.info("Done inserting %d bridges into hashring." % inserted) + + if state.COLLECT_TIMESTAMPS: + reactor.callInThread(updateBridgeHistory, bridges, timestamps) - state.save() + state.save() def _reloadFn(*args): """Placeholder callback function for :func:`_handleSIGHUP`.""" diff --git a/bridgedb/configure.py b/bridgedb/configure.py index e107a57..6c13595 100644 --- a/bridgedb/configure.py +++ b/bridgedb/configure.py @@ -100,7 +100,7 @@ def loadConfig(configFile=None, configCls=None): # # See :meth:`bridgedb.persistent.State.useUpdatedSettings`. - for attr in ["PROXY_LIST_FILES", "BRIDGE_FILES", "EXTRA_INFO_FILES"]: + for attr in ["PROXY_LIST_FILES"]: setting = getattr(config, attr, None) if setting is None: # pragma: no cover setattr(config, attr, []) # If they weren't set, make them lists @@ -110,7 +110,7 @@ def loadConfig(configFile=None, configCls=None): for attr in ["DB_FILE", "DB_LOG_FILE", "MASTER_KEY_FILE", "PIDFILE", "ASSIGNMENTS_FILE", "HTTPS_CERT_FILE", "HTTPS_KEY_FILE", - "LOG_FILE", "STATUS_FILE", "COUNTRY_BLOCK_FILE", + "LOG_FILE", "COUNTRY_BLOCK_FILE", "GIMP_CAPTCHA_DIR", "GIMP_CAPTCHA_HMAC_KEYFILE", "GIMP_CAPTCHA_RSA_KEYFILE", "EMAIL_GPG_HOMEDIR", "EMAIL_GPG_PASSPHRASE_FILE"]: diff --git a/requirements.txt b/requirements.txt index bd25ae4..26c2c1a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,5 +10,5 @@ gnupg==2.0.1 pillow>=2.6.1 qrcode==5.0.1 service_identity==14.0.0 -git+https://git.torproject.org/stem.git@ba8cee36a0348c1509ad3562051723b7948e19ce#egg=stem-1.4.1.dev0-py2.7 +git+https://git.torproject.org/stem.git@152fa89c33114619913d2be13e67adb4e55ce7ab#egg=stem-1.4.1.dev1-py2.7 zope.interface==3.6.1

1 0

[bridgedb/master] Add support for bridge descriptors from multiple authorities.
by isis＠torproject.org 22 Sep '16

22 Sep '16

commit ae45f903f260fcc98997793c52919b2d9b289295 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Sep 6 18:33:33 2016 +0000 Add support for bridge descriptors from multiple authorities. * FIXES #20088: https://bugs.torproject.org/20088 --- bridgedb.conf | 4 ++ bridgedb/Main.py | 172 +++++++++++++++++++++++++++----------------------- bridgedb/configure.py | 4 +- requirements.txt | 2 +- 4 files changed, 99 insertions(+), 83 deletions(-) diff --git a/bridgedb.conf b/bridgedb.conf index 52a8ca7..2d73a2b 100644 --- a/bridgedb.conf +++ b/bridgedb.conf @@ -120,6 +120,10 @@ # can read it. #------------------------------------------------------------------------------ +# List of directories which contain versions of the files specified in +# BRIDGE_FILES, EXTRA_INFO_FILES, and STATUS_FILE. +BRIDGE_AUTHORITY_DIRECTORIES = ["from-authority", "from-bifroest"] + # List of filenames from which we read ``@type bridge-server-descriptor``s, on # startup and on SIGHUP. BRIDGE_FILES = ["bridge-descriptors"] diff --git a/bridgedb/Main.py b/bridgedb/Main.py index bf4c213..dc618cb 100644 --- a/bridgedb/Main.py +++ b/bridgedb/Main.py @@ -41,6 +41,12 @@ from bridgedb import Bridges from bridgedb.Stability import updateBridgeHistory +def expandBridgeAuthDir(authdir, filename): + """Expands a descriptor ``filename`` relative to which of the + BRIDGE_AUTHORITY_DIRECTORIES, ``authdir`` it resides within. + """ + return os.path.abspath(os.path.expanduser(os.sep.join([authdir, filename]))) + def load(state, hashring, clear=False): """Read and parse all descriptors, and load into a bridge hashring. @@ -69,95 +75,101 @@ def load(state, hashring, clear=False): if ignoreNetworkstatus: logging.info("Ignoring BridgeAuthority networkstatus documents.") - bridges = {} - timestamps = {} - - logging.info("Opening networkstatus file: %s" % state.STATUS_FILE) - networkstatuses = descriptors.parseNetworkStatusFile(state.STATUS_FILE) - logging.debug("Closing networkstatus file: %s" % state.STATUS_FILE) + for auth in state.BRIDGE_AUTHORITY_DIRECTORIES: + logging.info("Processing descriptors in %s directory..." % auth) - logging.info("Processing networkstatus descriptors...") - for router in networkstatuses: - bridge = Bridge() - bridge.updateFromNetworkStatus(router, ignoreNetworkstatus) - try: - bridge.assertOK() - except MalformedBridgeInfo as error: - logging.warn(str(error)) - else: - bridges[bridge.fingerprint] = bridge + bridges = {} + timestamps = {} - for filename in state.BRIDGE_FILES: - logging.info("Opening bridge-server-descriptor file: '%s'" % filename) - serverdescriptors = descriptors.parseServerDescriptorsFile(filename) - logging.debug("Closing bridge-server-descriptor file: '%s'" % filename) + fn = expandBridgeAuthDir(auth, state.STATUS_FILE) + logging.info("Opening networkstatus file: %s" % fn) + networkstatuses = descriptors.parseNetworkStatusFile(fn) + logging.debug("Closing networkstatus file: %s" % fn) - for router in serverdescriptors: + logging.info("Processing networkstatus descriptors...") + for router in networkstatuses: + bridge = Bridge() + bridge.updateFromNetworkStatus(router, ignoreNetworkstatus) try: - bridge = bridges[router.fingerprint] - except KeyError: - logging.warn( - ("Received server descriptor for bridge '%s' which wasn't " - "in the networkstatus!") % router.fingerprint) - if ignoreNetworkstatus: - bridge = Bridge() - else: + bridge.assertOK() + except MalformedBridgeInfo as error: + logging.warn(str(error)) + else: + bridges[bridge.fingerprint] = bridge + + for filename in state.BRIDGE_FILES: + fn = expandBridgeAuthDir(auth, filename) + logging.info("Opening bridge-server-descriptor file: '%s'" % fn) + serverdescriptors = descriptors.parseServerDescriptorsFile(fn) + logging.debug("Closing bridge-server-descriptor file: '%s'" % fn) + + for router in serverdescriptors: + try: + bridge = bridges[router.fingerprint] + except KeyError: + logging.warn( + ("Received server descriptor for bridge '%s' which wasn't " + "in the networkstatus!") % router.fingerprint) + if ignoreNetworkstatus: + bridge = Bridge() + else: + continue + + try: + bridge.updateFromServerDescriptor(router, ignoreNetworkstatus) + except (ServerDescriptorWithoutNetworkstatus, + MissingServerDescriptorDigest, + ServerDescriptorDigestMismatch) as error: + logging.warn(str(error)) + # Reject any routers whose server descriptors didn't pass + # :meth:`~bridges.Bridge._checkServerDescriptor`, i.e. those + # bridges who don't have corresponding networkstatus + # documents, or whose server descriptor digests don't check + # out: + bridges.pop(router.fingerprint) continue + if state.COLLECT_TIMESTAMPS: + # Update timestamps from server descriptors, not from network + # status descriptors (because networkstatus documents and + # descriptors aren't authenticated in any way): + if bridge.fingerprint in timestamps.keys(): + timestamps[bridge.fingerprint].append(router.published) + else: + timestamps[bridge.fingerprint] = [router.published] + + eifiles = [expandBridgeAuthDir(auth, fn) for fn in state.EXTRA_INFO_FILES] + extrainfos = descriptors.parseExtraInfoFiles(*eifiles) + for fingerprint, router in extrainfos.items(): try: - bridge.updateFromServerDescriptor(router, ignoreNetworkstatus) - except (ServerDescriptorWithoutNetworkstatus, - MissingServerDescriptorDigest, - ServerDescriptorDigestMismatch) as error: + bridges[fingerprint].updateFromExtraInfoDescriptor(router) + except MalformedBridgeInfo as error: logging.warn(str(error)) - # Reject any routers whose server descriptors didn't pass - # :meth:`~bridges.Bridge._checkServerDescriptor`, i.e. those - # bridges who don't have corresponding networkstatus - # documents, or whose server descriptor digests don't check - # out: - bridges.pop(router.fingerprint) - continue - - if state.COLLECT_TIMESTAMPS: - # Update timestamps from server descriptors, not from network - # status descriptors (because networkstatus documents and - # descriptors aren't authenticated in any way): - if bridge.fingerprint in timestamps.keys(): - timestamps[bridge.fingerprint].append(router.published) - else: - timestamps[bridge.fingerprint] = [router.published] - - extrainfos = descriptors.parseExtraInfoFiles(*state.EXTRA_INFO_FILES) - for fingerprint, router in extrainfos.items(): - try: - bridges[fingerprint].updateFromExtraInfoDescriptor(router) - except MalformedBridgeInfo as error: - logging.warn(str(error)) - except KeyError as error: - logging.warn(("Received extrainfo descriptor for bridge '%s', " - "but could not find bridge with that fingerprint.") - % router.fingerprint) - - inserted = 0 - logging.info("Inserting %d bridges into hashring..." % len(bridges)) - for fingerprint, bridge in bridges.items(): - # Skip insertion of bridges which are geolocated to be in one of the - # NO_DISTRIBUTION_COUNTRIES, a.k.a. the countries we don't distribute - # bridges from: - if bridge.country in state.NO_DISTRIBUTION_COUNTRIES: - logging.warn("Not distributing Bridge %s %s:%s in country %s!" % - (bridge, bridge.address, bridge.orPort, bridge.country)) - else: - # If the bridge is not running, then it is skipped during the - # insertion process. - hashring.insert(bridge) - inserted += 1 - logging.info("Done inserting %d bridges into hashring." % inserted) - - if state.COLLECT_TIMESTAMPS: - reactor.callInThread(updateBridgeHistory, bridges, timestamps) + except KeyError as error: + logging.warn(("Received extrainfo descriptor for bridge '%s', " + "but could not find bridge with that fingerprint.") + % router.fingerprint) + + inserted = 0 + logging.info("Inserting %d bridges into hashring..." % len(bridges)) + for fingerprint, bridge in bridges.items(): + # Skip insertion of bridges which are geolocated to be in one of the + # NO_DISTRIBUTION_COUNTRIES, a.k.a. the countries we don't distribute + # bridges from: + if bridge.country in state.NO_DISTRIBUTION_COUNTRIES: + logging.warn("Not distributing Bridge %s %s:%s in country %s!" % + (bridge, bridge.address, bridge.orPort, bridge.country)) + else: + # If the bridge is not running, then it is skipped during the + # insertion process. + hashring.insert(bridge) + inserted += 1 + logging.info("Done inserting %d bridges into hashring." % inserted) + + if state.COLLECT_TIMESTAMPS: + reactor.callInThread(updateBridgeHistory, bridges, timestamps) - state.save() + state.save() def _reloadFn(*args): """Placeholder callback function for :func:`_handleSIGHUP`.""" diff --git a/bridgedb/configure.py b/bridgedb/configure.py index e107a57..6c13595 100644 --- a/bridgedb/configure.py +++ b/bridgedb/configure.py @@ -100,7 +100,7 @@ def loadConfig(configFile=None, configCls=None): # # See :meth:`bridgedb.persistent.State.useUpdatedSettings`. - for attr in ["PROXY_LIST_FILES", "BRIDGE_FILES", "EXTRA_INFO_FILES"]: + for attr in ["PROXY_LIST_FILES"]: setting = getattr(config, attr, None) if setting is None: # pragma: no cover setattr(config, attr, []) # If they weren't set, make them lists @@ -110,7 +110,7 @@ def loadConfig(configFile=None, configCls=None): for attr in ["DB_FILE", "DB_LOG_FILE", "MASTER_KEY_FILE", "PIDFILE", "ASSIGNMENTS_FILE", "HTTPS_CERT_FILE", "HTTPS_KEY_FILE", - "LOG_FILE", "STATUS_FILE", "COUNTRY_BLOCK_FILE", + "LOG_FILE", "COUNTRY_BLOCK_FILE", "GIMP_CAPTCHA_DIR", "GIMP_CAPTCHA_HMAC_KEYFILE", "GIMP_CAPTCHA_RSA_KEYFILE", "EMAIL_GPG_HOMEDIR", "EMAIL_GPG_PASSPHRASE_FILE"]: diff --git a/requirements.txt b/requirements.txt index bd25ae4..26c2c1a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,5 +10,5 @@ gnupg==2.0.1 pillow>=2.6.1 qrcode==5.0.1 service_identity==14.0.0 -git+https://git.torproject.org/stem.git@ba8cee36a0348c1509ad3562051723b7948e19ce#egg=stem-1.4.1.dev0-py2.7 +git+https://git.torproject.org/stem.git@152fa89c33114619913d2be13e67adb4e55ce7ab#egg=stem-1.4.1.dev1-py2.7 zope.interface==3.6.1

1 0