September 2016 - tor-commits - lists.torproject.org

[bridgedb/master] Add support for bridge descriptors from multiple authorities.
by isis＠torproject.org 22 Sep '16

22 Sep '16

commit ae45f903f260fcc98997793c52919b2d9b289295 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Sep 6 18:33:33 2016 +0000 Add support for bridge descriptors from multiple authorities. * FIXES #20088: https://bugs.torproject.org/20088 --- bridgedb.conf | 4 ++ bridgedb/Main.py | 172 +++++++++++++++++++++++++++----------------------- bridgedb/configure.py | 4 +- requirements.txt | 2 +- 4 files changed, 99 insertions(+), 83 deletions(-) diff --git a/bridgedb.conf b/bridgedb.conf index 52a8ca7..2d73a2b 100644 --- a/bridgedb.conf +++ b/bridgedb.conf @@ -120,6 +120,10 @@ # can read it. #------------------------------------------------------------------------------ +# List of directories which contain versions of the files specified in +# BRIDGE_FILES, EXTRA_INFO_FILES, and STATUS_FILE. +BRIDGE_AUTHORITY_DIRECTORIES = ["from-authority", "from-bifroest"] + # List of filenames from which we read ``@type bridge-server-descriptor``s, on # startup and on SIGHUP. BRIDGE_FILES = ["bridge-descriptors"] diff --git a/bridgedb/Main.py b/bridgedb/Main.py index bf4c213..dc618cb 100644 --- a/bridgedb/Main.py +++ b/bridgedb/Main.py @@ -41,6 +41,12 @@ from bridgedb import Bridges from bridgedb.Stability import updateBridgeHistory +def expandBridgeAuthDir(authdir, filename): + """Expands a descriptor ``filename`` relative to which of the + BRIDGE_AUTHORITY_DIRECTORIES, ``authdir`` it resides within. + """ + return os.path.abspath(os.path.expanduser(os.sep.join([authdir, filename]))) + def load(state, hashring, clear=False): """Read and parse all descriptors, and load into a bridge hashring. @@ -69,95 +75,101 @@ def load(state, hashring, clear=False): if ignoreNetworkstatus: logging.info("Ignoring BridgeAuthority networkstatus documents.") - bridges = {} - timestamps = {} - - logging.info("Opening networkstatus file: %s" % state.STATUS_FILE) - networkstatuses = descriptors.parseNetworkStatusFile(state.STATUS_FILE) - logging.debug("Closing networkstatus file: %s" % state.STATUS_FILE) + for auth in state.BRIDGE_AUTHORITY_DIRECTORIES: + logging.info("Processing descriptors in %s directory..." % auth) - logging.info("Processing networkstatus descriptors...") - for router in networkstatuses: - bridge = Bridge() - bridge.updateFromNetworkStatus(router, ignoreNetworkstatus) - try: - bridge.assertOK() - except MalformedBridgeInfo as error: - logging.warn(str(error)) - else: - bridges[bridge.fingerprint] = bridge + bridges = {} + timestamps = {} - for filename in state.BRIDGE_FILES: - logging.info("Opening bridge-server-descriptor file: '%s'" % filename) - serverdescriptors = descriptors.parseServerDescriptorsFile(filename) - logging.debug("Closing bridge-server-descriptor file: '%s'" % filename) + fn = expandBridgeAuthDir(auth, state.STATUS_FILE) + logging.info("Opening networkstatus file: %s" % fn) + networkstatuses = descriptors.parseNetworkStatusFile(fn) + logging.debug("Closing networkstatus file: %s" % fn) - for router in serverdescriptors: + logging.info("Processing networkstatus descriptors...") + for router in networkstatuses: + bridge = Bridge() + bridge.updateFromNetworkStatus(router, ignoreNetworkstatus) try: - bridge = bridges[router.fingerprint] - except KeyError: - logging.warn( - ("Received server descriptor for bridge '%s' which wasn't " - "in the networkstatus!") % router.fingerprint) - if ignoreNetworkstatus: - bridge = Bridge() - else: + bridge.assertOK() + except MalformedBridgeInfo as error: + logging.warn(str(error)) + else: + bridges[bridge.fingerprint] = bridge + + for filename in state.BRIDGE_FILES: + fn = expandBridgeAuthDir(auth, filename) + logging.info("Opening bridge-server-descriptor file: '%s'" % fn) + serverdescriptors = descriptors.parseServerDescriptorsFile(fn) + logging.debug("Closing bridge-server-descriptor file: '%s'" % fn) + + for router in serverdescriptors: + try: + bridge = bridges[router.fingerprint] + except KeyError: + logging.warn( + ("Received server descriptor for bridge '%s' which wasn't " + "in the networkstatus!") % router.fingerprint) + if ignoreNetworkstatus: + bridge = Bridge() + else: + continue + + try: + bridge.updateFromServerDescriptor(router, ignoreNetworkstatus) + except (ServerDescriptorWithoutNetworkstatus, + MissingServerDescriptorDigest, + ServerDescriptorDigestMismatch) as error: + logging.warn(str(error)) + # Reject any routers whose server descriptors didn't pass + # :meth:`~bridges.Bridge._checkServerDescriptor`, i.e. those + # bridges who don't have corresponding networkstatus + # documents, or whose server descriptor digests don't check + # out: + bridges.pop(router.fingerprint) continue + if state.COLLECT_TIMESTAMPS: + # Update timestamps from server descriptors, not from network + # status descriptors (because networkstatus documents and + # descriptors aren't authenticated in any way): + if bridge.fingerprint in timestamps.keys(): + timestamps[bridge.fingerprint].append(router.published) + else: + timestamps[bridge.fingerprint] = [router.published] + + eifiles = [expandBridgeAuthDir(auth, fn) for fn in state.EXTRA_INFO_FILES] + extrainfos = descriptors.parseExtraInfoFiles(*eifiles) + for fingerprint, router in extrainfos.items(): try: - bridge.updateFromServerDescriptor(router, ignoreNetworkstatus) - except (ServerDescriptorWithoutNetworkstatus, - MissingServerDescriptorDigest, - ServerDescriptorDigestMismatch) as error: + bridges[fingerprint].updateFromExtraInfoDescriptor(router) + except MalformedBridgeInfo as error: logging.warn(str(error)) - # Reject any routers whose server descriptors didn't pass - # :meth:`~bridges.Bridge._checkServerDescriptor`, i.e. those - # bridges who don't have corresponding networkstatus - # documents, or whose server descriptor digests don't check - # out: - bridges.pop(router.fingerprint) - continue - - if state.COLLECT_TIMESTAMPS: - # Update timestamps from server descriptors, not from network - # status descriptors (because networkstatus documents and - # descriptors aren't authenticated in any way): - if bridge.fingerprint in timestamps.keys(): - timestamps[bridge.fingerprint].append(router.published) - else: - timestamps[bridge.fingerprint] = [router.published] - - extrainfos = descriptors.parseExtraInfoFiles(*state.EXTRA_INFO_FILES) - for fingerprint, router in extrainfos.items(): - try: - bridges[fingerprint].updateFromExtraInfoDescriptor(router) - except MalformedBridgeInfo as error: - logging.warn(str(error)) - except KeyError as error: - logging.warn(("Received extrainfo descriptor for bridge '%s', " - "but could not find bridge with that fingerprint.") - % router.fingerprint) - - inserted = 0 - logging.info("Inserting %d bridges into hashring..." % len(bridges)) - for fingerprint, bridge in bridges.items(): - # Skip insertion of bridges which are geolocated to be in one of the - # NO_DISTRIBUTION_COUNTRIES, a.k.a. the countries we don't distribute - # bridges from: - if bridge.country in state.NO_DISTRIBUTION_COUNTRIES: - logging.warn("Not distributing Bridge %s %s:%s in country %s!" % - (bridge, bridge.address, bridge.orPort, bridge.country)) - else: - # If the bridge is not running, then it is skipped during the - # insertion process. - hashring.insert(bridge) - inserted += 1 - logging.info("Done inserting %d bridges into hashring." % inserted) - - if state.COLLECT_TIMESTAMPS: - reactor.callInThread(updateBridgeHistory, bridges, timestamps) + except KeyError as error: + logging.warn(("Received extrainfo descriptor for bridge '%s', " + "but could not find bridge with that fingerprint.") + % router.fingerprint) + + inserted = 0 + logging.info("Inserting %d bridges into hashring..." % len(bridges)) + for fingerprint, bridge in bridges.items(): + # Skip insertion of bridges which are geolocated to be in one of the + # NO_DISTRIBUTION_COUNTRIES, a.k.a. the countries we don't distribute + # bridges from: + if bridge.country in state.NO_DISTRIBUTION_COUNTRIES: + logging.warn("Not distributing Bridge %s %s:%s in country %s!" % + (bridge, bridge.address, bridge.orPort, bridge.country)) + else: + # If the bridge is not running, then it is skipped during the + # insertion process. + hashring.insert(bridge) + inserted += 1 + logging.info("Done inserting %d bridges into hashring." % inserted) + + if state.COLLECT_TIMESTAMPS: + reactor.callInThread(updateBridgeHistory, bridges, timestamps) - state.save() + state.save() def _reloadFn(*args): """Placeholder callback function for :func:`_handleSIGHUP`.""" diff --git a/bridgedb/configure.py b/bridgedb/configure.py index e107a57..6c13595 100644 --- a/bridgedb/configure.py +++ b/bridgedb/configure.py @@ -100,7 +100,7 @@ def loadConfig(configFile=None, configCls=None): # # See :meth:`bridgedb.persistent.State.useUpdatedSettings`. - for attr in ["PROXY_LIST_FILES", "BRIDGE_FILES", "EXTRA_INFO_FILES"]: + for attr in ["PROXY_LIST_FILES"]: setting = getattr(config, attr, None) if setting is None: # pragma: no cover setattr(config, attr, []) # If they weren't set, make them lists @@ -110,7 +110,7 @@ def loadConfig(configFile=None, configCls=None): for attr in ["DB_FILE", "DB_LOG_FILE", "MASTER_KEY_FILE", "PIDFILE", "ASSIGNMENTS_FILE", "HTTPS_CERT_FILE", "HTTPS_KEY_FILE", - "LOG_FILE", "STATUS_FILE", "COUNTRY_BLOCK_FILE", + "LOG_FILE", "COUNTRY_BLOCK_FILE", "GIMP_CAPTCHA_DIR", "GIMP_CAPTCHA_HMAC_KEYFILE", "GIMP_CAPTCHA_RSA_KEYFILE", "EMAIL_GPG_HOMEDIR", "EMAIL_GPG_PASSPHRASE_FILE"]: diff --git a/requirements.txt b/requirements.txt index bd25ae4..26c2c1a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,5 +10,5 @@ gnupg==2.0.1 pillow>=2.6.1 qrcode==5.0.1 service_identity==14.0.0 -git+https://git.torproject.org/stem.git@ba8cee36a0348c1509ad3562051723b7948e19ce#egg=stem-1.4.1.dev0-py2.7 +git+https://git.torproject.org/stem.git@152fa89c33114619913d2be13e67adb4e55ce7ab#egg=stem-1.4.1.dev1-py2.7 zope.interface==3.6.1

1 0

[bridgedb/master] Coverage 4.2 doesn't understand [report] modules= config option.
by isis＠torproject.org 22 Sep '16

22 Sep '16

commit 0137fb5b202f3623fcc0a30a51e85230ab85e79a Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Sep 8 17:04:55 2016 +0000 Coverage 4.2 doesn't understand [report] modules= config option. --- .coveragerc | 1 - 1 file changed, 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index 04dc0a6..d16f7e2 100644 --- a/.coveragerc +++ b/.coveragerc @@ -5,7 +5,6 @@ branch = True timid = False [report] -modules = bridgedb omit = */_langs* */_version*

1 0

[tor/release-0.2.8] Merge branch 'maint-0.2.8' into release-0.2.8
by nickm＠torproject.org 22 Sep '16

22 Sep '16

commit caa16bffc75b8e36ae3f834c52e9b107513e263f Merge: a8b607b 1edea87 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Sep 22 09:00:23 2016 -0400 Merge branch 'maint-0.2.8' into release-0.2.8 changes/bug20103 | 2 +- changes/geoip-september2016 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-)

1 0

[tor/maint-0.2.8] Fix lintchanges warnings in 028
by nickm＠torproject.org 22 Sep '16

22 Sep '16

commit 1edea87c2ac66d9d9ba5c92e6b2e4baddb6bd886 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Sep 22 09:00:16 2016 -0400 Fix lintchanges warnings in 028 --- changes/bug20103 | 2 +- changes/geoip-september2016 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/changes/bug20103 b/changes/bug20103 index c2b81d3..bf0aeec 100644 --- a/changes/bug20103 +++ b/changes/bug20103 @@ -1,4 +1,4 @@ - o Major bug fixes (crash): + o Major bugfixes (crash): - Fix a complicated crash bug that could affect Tor clients configured to use bridges when replacing a networkstatus consensus diff --git a/changes/geoip-september2016 b/changes/geoip-september2016 index a14c7c6..1bf5570 100644 --- a/changes/geoip-september2016 +++ b/changes/geoip-september2016 @@ -1,4 +1,4 @@ - o Minor features: + o Minor features (geoip): - Update geoip and geoip6 to the September 6 2016 Maxmind GeoLite2 Country database.

1 0

[tor/master] Fix lintchanges warnings in 028
by nickm＠torproject.org 22 Sep '16

22 Sep '16

commit 1edea87c2ac66d9d9ba5c92e6b2e4baddb6bd886 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Sep 22 09:00:16 2016 -0400 Fix lintchanges warnings in 028 --- changes/bug20103 | 2 +- changes/geoip-september2016 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/changes/bug20103 b/changes/bug20103 index c2b81d3..bf0aeec 100644 --- a/changes/bug20103 +++ b/changes/bug20103 @@ -1,4 +1,4 @@ - o Major bug fixes (crash): + o Major bugfixes (crash): - Fix a complicated crash bug that could affect Tor clients configured to use bridges when replacing a networkstatus consensus diff --git a/changes/geoip-september2016 b/changes/geoip-september2016 index a14c7c6..1bf5570 100644 --- a/changes/geoip-september2016 +++ b/changes/geoip-september2016 @@ -1,4 +1,4 @@ - o Minor features: + o Minor features (geoip): - Update geoip and geoip6 to the September 6 2016 Maxmind GeoLite2 Country database.

1 0

[tor/release-0.2.8] Fix lintchanges warnings in 028
by nickm＠torproject.org 22 Sep '16

22 Sep '16

commit 1edea87c2ac66d9d9ba5c92e6b2e4baddb6bd886 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Sep 22 09:00:16 2016 -0400 Fix lintchanges warnings in 028 --- changes/bug20103 | 2 +- changes/geoip-september2016 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/changes/bug20103 b/changes/bug20103 index c2b81d3..bf0aeec 100644 --- a/changes/bug20103 +++ b/changes/bug20103 @@ -1,4 +1,4 @@ - o Major bug fixes (crash): + o Major bugfixes (crash): - Fix a complicated crash bug that could affect Tor clients configured to use bridges when replacing a networkstatus consensus diff --git a/changes/geoip-september2016 b/changes/geoip-september2016 index a14c7c6..1bf5570 100644 --- a/changes/geoip-september2016 +++ b/changes/geoip-september2016 @@ -1,4 +1,4 @@ - o Minor features: + o Minor features (geoip): - Update geoip and geoip6 to the September 6 2016 Maxmind GeoLite2 Country database.

1 0

[tor/master] Merge branch 'maint-0.2.8'
by nickm＠torproject.org 22 Sep '16

22 Sep '16

commit 6deeedb5e09fc0998ab5d4e2fc53415bc6dc5c39 Merge: 4c69ba5 1edea87 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Sep 22 09:00:37 2016 -0400 Merge branch 'maint-0.2.8' changes/bug20103 | 2 +- changes/geoip-september2016 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-)

1 0

[tor/master] Fix conflicting types errors for aes.c.
by nickm＠torproject.org 22 Sep '16

22 Sep '16

commit 4c69ba5895dd448001f80645d62b0d4497b61ac5 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Sep 22 08:52:42 2016 -0400 Fix conflicting types errors for aes.c. --- src/common/aes.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/aes.c b/src/common/aes.c index cb8fb68..ef94d8a 100644 --- a/src/common/aes.c +++ b/src/common/aes.c @@ -259,9 +259,9 @@ evaluate_ctr_for_aes(void) #define COUNTER(c, n) ((c)->counter ## n) #endif -static void aes_set_key(aes_cnt_cipher_t *cipher, const char *key, +static void aes_set_key(aes_cnt_cipher_t *cipher, const uint8_t *key, int key_bits); -static void aes_set_iv(aes_cnt_cipher_t *cipher, const char *iv); +static void aes_set_iv(aes_cnt_cipher_t *cipher, const uint8_t *iv); /** * Return a newly allocated counter-mode AES128 cipher implementation,

1 0

[tor/master] Fix warnings from lintChanges.py
by nickm＠torproject.org 22 Sep '16

22 Sep '16

commit 20ce9f23dc89847dfc3b70ff1cc9f7ec5bd5d644 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Sep 22 08:48:05 2016 -0400 Fix warnings from lintChanges.py --- changes/19122 | 5 +++-- changes/19977 | 4 ++-- changes/bug19677 | 6 +++--- changes/bug19767 | 4 +++- changes/feature17178 | 14 +++++++------- changes/reject-tap | 4 +++- 6 files changed, 21 insertions(+), 16 deletions(-) diff --git a/changes/19122 b/changes/19122 index cface80..80f38a6 100644 --- a/changes/19122 +++ b/changes/19122 @@ -1,3 +1,4 @@ o Minor bugfixes (documentation): - - man page incorrectly states the User option takes a UID when it really - takes a username. Fixes bug 19122. + - Fix manual for the User option: it takes a username, not a UID. + Fixes bug 19122; bugfix on 0.0.2pre16 (the first version to have + a manpage!). diff --git a/changes/19977 b/changes/19977 index 0ca1af1..0e55fbd 100644 --- a/changes/19977 +++ b/changes/19977 @@ -1,6 +1,6 @@ - o Minor bugfixes (unit test) + o Minor bugfixes (unit tests): - Fix shared random unit test that was failing on big endian architecture due to internal representation of a integer copied to a buffer. The test is changed to take a full 32 bytes of data and use the output of a python script that make the COMMIT and REVEAL calculation according to - the spec. Fixes #19977; bugfix on tor-0.2.9.1-alpha. + the spec. Fixes bug 19977; bugfix on tor-0.2.9.1-alpha. diff --git a/changes/bug19677 b/changes/bug19677 index e8ba3dd..2c84b7e 100644 --- a/changes/bug19677 +++ b/changes/bug19677 @@ -1,6 +1,6 @@ - o Minor bug fixes (option parsing): + o Minor bugfixes (option parsing): - Count unix sockets when counting client listeners (SOCKS, Trans, NATD, and DNS). This has no user-visible behaviour changes: these options are set once, and never read. - Required for correct behaviour in ticket #17178. - Fixes bug #19677, patch by teor. + Required for correct behaviour in ticket 17178. + Fixes bug 19677; bugfix on 0.2.6.3-alpha. Patch by teor. diff --git a/changes/bug19767 b/changes/bug19767 index f0a010b..a534922 100644 --- a/changes/bug19767 +++ b/changes/bug19767 @@ -1,4 +1,6 @@ o Minor bugfixes (compilation): - Always include orconfig.h before including any other C headers. Sometimes, it includes macros that affect the behavior of the - standard headers. Closes bug 19767. + standard headers. Fixes bug 19767; bugfix on 0.2.9.1-alpha (the first + version to use AC_USE_SYSTEM_EXTENSIONS). + diff --git a/changes/feature17178 b/changes/feature17178 index 172097c..d6a8e8d 100644 --- a/changes/feature17178 +++ b/changes/feature17178 @@ -8,23 +8,23 @@ This is compatible with the existing hidden service implementation, and works on the current tor network without any changes to older relays or clients. - Implements proposal #260, completes ticket #17178. Patch by teor & asn. + Implements proposal 260, completes ticket 17178. Patch by teor and asn. o Minor features (Tor2web): - Make Tor2web clients respect ReachableAddresses. This feature was inadvertently enabled in 0.2.8.6, then removed - by bugfix #19973 on 0.2.8.7. - Implements feature #20034. Patch by teor. + by bugfix 19973 on 0.2.8.7. + Implements feature 20034. Patch by teor. o Minor bug fixes (Tor2web): - Prevent Tor2web clients running hidden services, these services are not anonymous due to the one-hop client paths. - Fixes bug #19678. Patch by teor. + Fixes bug 19678. Patch by teor. o Minor bug fixes (circuits): - Use CircuitBuildTimeout whenever LearnCircuitBuildTimeout is disabled. - Fixes bug #19678 in commit 5b0b51ca3 in 0.2.4.12-alpha. Patch by teor. + Fixes bug 19678; bugfix on commit 5b0b51ca3 in 0.2.4.12-alpha. Patch by teor. o Minor bug fixes (options): - Stop changing the configured value of UseEntryGuards on authorities and Tor2web clients. - Fixes bug #20074 in commits 51fc6799 in tor-0.1.1.16-rc and + Fixes bug 20074; bugfix on commits 51fc6799 in tor-0.1.1.16-rc and acda1735 in tor-0.2.4.3-alpha. Patch by teor. - Check the consistency of UseEntryGuards and EntryNodes more reliably. - Fixes bug #20074 in commit 686aaa5c in tor-0.2.4.12-alpha. Patch by teor. + Fixes bug 20074; bugfix on commit 686aaa5c in tor-0.2.4.12-alpha. Patch by teor. diff --git a/changes/reject-tap b/changes/reject-tap index 8e616de..dad554f 100644 --- a/changes/reject-tap +++ b/changes/reject-tap @@ -1,4 +1,4 @@ - o Major bug fixes (circuit building): + o Major features (circuit building, security): - Tor authorities, relays, and clients only use ntor, except for rare cases in the hidden service protocol. - Authorities, relays and clients specifically check that each @@ -10,6 +10,8 @@ mirrors, or as introduction or rendezvous points. - Circuit-building code assumes that all hops can use ntor, except for rare hidden service protocol cases. + + o Major bugfixes (circuit building): - Hidden service client to intro point and service to rendezvous point connections use the TAP key supplied by the protocol. Fixes bug 19163; bugfix on 0.2.4.18-rc.

1 0

[tor/master] fix crash in lintChanges.py
by nickm＠torproject.org 22 Sep '16

22 Sep '16

commit b64c22436216d92e2dc2d3754e121713e0a1f69f Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Sep 22 08:33:09 2016 -0400 fix crash in lintChanges.py --- scripts/maint/lintChanges.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/maint/lintChanges.py b/scripts/maint/lintChanges.py index 0b8c5e5..b63a4eb 100755 --- a/scripts/maint/lintChanges.py +++ b/scripts/maint/lintChanges.py @@ -55,7 +55,10 @@ def lintfile(fname): '(' not in m.group(2)): warn("Missing subcategory on %s"%m.group(1)) - isBug = ("bug" in m.group(1).lower() or "fix" in m.group(1).lower()) + if m: + isBug = ("bug" in m.group(1).lower() or "fix" in m.group(1).lower()) + else: + isBug = False contents = " ".join(contents.split())

1 0