commit ae45f903f260fcc98997793c52919b2d9b289295 Author: Isis Lovecruft <isis@torproject.org> Date: Tue Sep 6 18:33:33 2016 +0000 Add support for bridge descriptors from multiple authorities. * FIXES #20088: https://bugs.torproject.org/20088 --- bridgedb.conf | 4 ++ bridgedb/Main.py | 172 +++++++++++++++++++++++++++----------------------- bridgedb/configure.py | 4 +- requirements.txt | 2 +- 4 files changed, 99 insertions(+), 83 deletions(-) diff --git a/bridgedb.conf b/bridgedb.conf index 52a8ca7..2d73a2b 100644 --- a/bridgedb.conf +++ b/bridgedb.conf @@ -120,6 +120,10 @@ # can read it. #------------------------------------------------------------------------------ +# List of directories which contain versions of the files specified in +# BRIDGE_FILES, EXTRA_INFO_FILES, and STATUS_FILE. +BRIDGE_AUTHORITY_DIRECTORIES = ["from-authority", "from-bifroest"] + # List of filenames from which we read ``@type bridge-server-descriptor``s, on # startup and on SIGHUP. BRIDGE_FILES = ["bridge-descriptors"] diff --git a/bridgedb/Main.py b/bridgedb/Main.py index bf4c213..dc618cb 100644 --- a/bridgedb/Main.py +++ b/bridgedb/Main.py @@ -41,6 +41,12 @@ from bridgedb import Bridges from bridgedb.Stability import updateBridgeHistory +def expandBridgeAuthDir(authdir, filename): + """Expands a descriptor ``filename`` relative to which of the + BRIDGE_AUTHORITY_DIRECTORIES, ``authdir`` it resides within. + """ + return os.path.abspath(os.path.expanduser(os.sep.join([authdir, filename]))) + def load(state, hashring, clear=False): """Read and parse all descriptors, and load into a bridge hashring. @@ -69,95 +75,101 @@ def load(state, hashring, clear=False): if ignoreNetworkstatus: logging.info("Ignoring BridgeAuthority networkstatus documents.") - bridges = {} - timestamps = {} - - logging.info("Opening networkstatus file: %s" % state.STATUS_FILE) - networkstatuses = descriptors.parseNetworkStatusFile(state.STATUS_FILE) - logging.debug("Closing networkstatus file: %s" % state.STATUS_FILE) + for auth in state.BRIDGE_AUTHORITY_DIRECTORIES: + logging.info("Processing descriptors in %s directory..." % auth) - logging.info("Processing networkstatus descriptors...") - for router in networkstatuses: - bridge = Bridge() - bridge.updateFromNetworkStatus(router, ignoreNetworkstatus) - try: - bridge.assertOK() - except MalformedBridgeInfo as error: - logging.warn(str(error)) - else: - bridges[bridge.fingerprint] = bridge + bridges = {} + timestamps = {} - for filename in state.BRIDGE_FILES: - logging.info("Opening bridge-server-descriptor file: '%s'" % filename) - serverdescriptors = descriptors.parseServerDescriptorsFile(filename) - logging.debug("Closing bridge-server-descriptor file: '%s'" % filename) + fn = expandBridgeAuthDir(auth, state.STATUS_FILE) + logging.info("Opening networkstatus file: %s" % fn) + networkstatuses = descriptors.parseNetworkStatusFile(fn) + logging.debug("Closing networkstatus file: %s" % fn) - for router in serverdescriptors: + logging.info("Processing networkstatus descriptors...") + for router in networkstatuses: + bridge = Bridge() + bridge.updateFromNetworkStatus(router, ignoreNetworkstatus) try: - bridge = bridges[router.fingerprint] - except KeyError: - logging.warn( - ("Received server descriptor for bridge '%s' which wasn't " - "in the networkstatus!") % router.fingerprint) - if ignoreNetworkstatus: - bridge = Bridge() - else: + bridge.assertOK() + except MalformedBridgeInfo as error: + logging.warn(str(error)) + else: + bridges[bridge.fingerprint] = bridge + + for filename in state.BRIDGE_FILES: + fn = expandBridgeAuthDir(auth, filename) + logging.info("Opening bridge-server-descriptor file: '%s'" % fn) + serverdescriptors = descriptors.parseServerDescriptorsFile(fn) + logging.debug("Closing bridge-server-descriptor file: '%s'" % fn) + + for router in serverdescriptors: + try: + bridge = bridges[router.fingerprint] + except KeyError: + logging.warn( + ("Received server descriptor for bridge '%s' which wasn't " + "in the networkstatus!") % router.fingerprint) + if ignoreNetworkstatus: + bridge = Bridge() + else: + continue + + try: + bridge.updateFromServerDescriptor(router, ignoreNetworkstatus) + except (ServerDescriptorWithoutNetworkstatus, + MissingServerDescriptorDigest, + ServerDescriptorDigestMismatch) as error: + logging.warn(str(error)) + # Reject any routers whose server descriptors didn't pass + # :meth:`~bridges.Bridge._checkServerDescriptor`, i.e. those + # bridges who don't have corresponding networkstatus + # documents, or whose server descriptor digests don't check + # out: + bridges.pop(router.fingerprint) continue + if state.COLLECT_TIMESTAMPS: + # Update timestamps from server descriptors, not from network + # status descriptors (because networkstatus documents and + # descriptors aren't authenticated in any way): + if bridge.fingerprint in timestamps.keys(): + timestamps[bridge.fingerprint].append(router.published) + else: + timestamps[bridge.fingerprint] = [router.published] + + eifiles = [expandBridgeAuthDir(auth, fn) for fn in state.EXTRA_INFO_FILES] + extrainfos = descriptors.parseExtraInfoFiles(*eifiles) + for fingerprint, router in extrainfos.items(): try: - bridge.updateFromServerDescriptor(router, ignoreNetworkstatus) - except (ServerDescriptorWithoutNetworkstatus, - MissingServerDescriptorDigest, - ServerDescriptorDigestMismatch) as error: + bridges[fingerprint].updateFromExtraInfoDescriptor(router) + except MalformedBridgeInfo as error: logging.warn(str(error)) - # Reject any routers whose server descriptors didn't pass - # :meth:`~bridges.Bridge._checkServerDescriptor`, i.e. those - # bridges who don't have corresponding networkstatus - # documents, or whose server descriptor digests don't check - # out: - bridges.pop(router.fingerprint) - continue - - if state.COLLECT_TIMESTAMPS: - # Update timestamps from server descriptors, not from network - # status descriptors (because networkstatus documents and - # descriptors aren't authenticated in any way): - if bridge.fingerprint in timestamps.keys(): - timestamps[bridge.fingerprint].append(router.published) - else: - timestamps[bridge.fingerprint] = [router.published] - - extrainfos = descriptors.parseExtraInfoFiles(*state.EXTRA_INFO_FILES) - for fingerprint, router in extrainfos.items(): - try: - bridges[fingerprint].updateFromExtraInfoDescriptor(router) - except MalformedBridgeInfo as error: - logging.warn(str(error)) - except KeyError as error: - logging.warn(("Received extrainfo descriptor for bridge '%s', " - "but could not find bridge with that fingerprint.") - % router.fingerprint) - - inserted = 0 - logging.info("Inserting %d bridges into hashring..." % len(bridges)) - for fingerprint, bridge in bridges.items(): - # Skip insertion of bridges which are geolocated to be in one of the - # NO_DISTRIBUTION_COUNTRIES, a.k.a. the countries we don't distribute - # bridges from: - if bridge.country in state.NO_DISTRIBUTION_COUNTRIES: - logging.warn("Not distributing Bridge %s %s:%s in country %s!" % - (bridge, bridge.address, bridge.orPort, bridge.country)) - else: - # If the bridge is not running, then it is skipped during the - # insertion process. - hashring.insert(bridge) - inserted += 1 - logging.info("Done inserting %d bridges into hashring." % inserted) - - if state.COLLECT_TIMESTAMPS: - reactor.callInThread(updateBridgeHistory, bridges, timestamps) + except KeyError as error: + logging.warn(("Received extrainfo descriptor for bridge '%s', " + "but could not find bridge with that fingerprint.") + % router.fingerprint) + + inserted = 0 + logging.info("Inserting %d bridges into hashring..." % len(bridges)) + for fingerprint, bridge in bridges.items(): + # Skip insertion of bridges which are geolocated to be in one of the + # NO_DISTRIBUTION_COUNTRIES, a.k.a. the countries we don't distribute + # bridges from: + if bridge.country in state.NO_DISTRIBUTION_COUNTRIES: + logging.warn("Not distributing Bridge %s %s:%s in country %s!" % + (bridge, bridge.address, bridge.orPort, bridge.country)) + else: + # If the bridge is not running, then it is skipped during the + # insertion process. + hashring.insert(bridge) + inserted += 1 + logging.info("Done inserting %d bridges into hashring." % inserted) + + if state.COLLECT_TIMESTAMPS: + reactor.callInThread(updateBridgeHistory, bridges, timestamps) - state.save() + state.save() def _reloadFn(*args): """Placeholder callback function for :func:`_handleSIGHUP`.""" diff --git a/bridgedb/configure.py b/bridgedb/configure.py index e107a57..6c13595 100644 --- a/bridgedb/configure.py +++ b/bridgedb/configure.py @@ -100,7 +100,7 @@ def loadConfig(configFile=None, configCls=None): # # See :meth:`bridgedb.persistent.State.useUpdatedSettings`. - for attr in ["PROXY_LIST_FILES", "BRIDGE_FILES", "EXTRA_INFO_FILES"]: + for attr in ["PROXY_LIST_FILES"]: setting = getattr(config, attr, None) if setting is None: # pragma: no cover setattr(config, attr, []) # If they weren't set, make them lists @@ -110,7 +110,7 @@ def loadConfig(configFile=None, configCls=None): for attr in ["DB_FILE", "DB_LOG_FILE", "MASTER_KEY_FILE", "PIDFILE", "ASSIGNMENTS_FILE", "HTTPS_CERT_FILE", "HTTPS_KEY_FILE", - "LOG_FILE", "STATUS_FILE", "COUNTRY_BLOCK_FILE", + "LOG_FILE", "COUNTRY_BLOCK_FILE", "GIMP_CAPTCHA_DIR", "GIMP_CAPTCHA_HMAC_KEYFILE", "GIMP_CAPTCHA_RSA_KEYFILE", "EMAIL_GPG_HOMEDIR", "EMAIL_GPG_PASSPHRASE_FILE"]: diff --git a/requirements.txt b/requirements.txt index bd25ae4..26c2c1a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,5 +10,5 @@ gnupg==2.0.1 pillow>=2.6.1 qrcode==5.0.1 service_identity==14.0.0 -git+https://git.torproject.org/stem.git@ba8cee36a0348c1509ad3562051723b7948e19ce... +git+https://git.torproject.org/stem.git@152fa89c33114619913d2be13e67adb4e55ce7ab... zope.interface==3.6.1