January 2018 - tor-commits - lists.torproject.org

[tor/master] Merge remote-tracking branch 'ffmancera/bug24861'
by nickm＠torproject.org 12 Jan '18

12 Jan '18

commit edc87b263f09d09eb96599fde27cd82d7ea5aa13 Merge: 1f65a2310 06368e531 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Jan 12 13:29:53 2018 -0500 Merge remote-tracking branch 'ffmancera/bug24861' changes/ticket24861 | 5 +++++ src/common/torint.h | 10 ++++++++++ src/or/circuitlist.c | 11 +++++++---- 3 files changed, 22 insertions(+), 4 deletions(-)

1 0

[tor/master] Fix minGW compatibility issue with zu format specifier.
by nickm＠torproject.org 12 Jan '18

12 Jan '18

commit 06368e531097d3b973f1f052e77e7ffdecde1b70 Author: Fernando Fernandez Mancera <ffernandezmancera(a)gmail.com> Date: Fri Jan 12 17:44:30 2018 +0100 Fix minGW compatibility issue with zu format specifier. Define TOR_PRIuSZ as minGW compiler doesn't support zu format specifier for size_t type. Fixes #24861 on ac9eebd. Signed-off-by: Fernando Fernandez Mancera <ffernandezmancera(a)gmail.com> --- changes/ticket24861 | 5 +++++ src/common/torint.h | 10 ++++++++++ src/or/circuitlist.c | 11 +++++++---- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/changes/ticket24861 b/changes/ticket24861 new file mode 100644 index 000000000..beddb770b --- /dev/null +++ b/changes/ticket24861 @@ -0,0 +1,5 @@ + o Minor bugfix (zu format specifier on minGW): + - Fix minGW compatibility issue. Define TOR_PRIuSZ as minGW compiler + doesn't support zu format specifier for size_t type. Fixes bug 24861; + bugfix on ac9eebd6363. + Patch by "ffmancera". diff --git a/src/common/torint.h b/src/common/torint.h index bc81c114f..0b8061d24 100644 --- a/src/common/torint.h +++ b/src/common/torint.h @@ -348,6 +348,16 @@ typedef uint32_t uintptr_t; #endif /* (SIZEOF_SIZE_T == 4) || ... */ #endif /* !defined(SIZE_MAX) */ +#ifdef _WIN32 +# ifdef _WIN64 +# define TOR_PRIuSZ PRIu64 +# else +# define TOR_PRIuSZ PRIu32 +# endif +#else +# define TOR_PRIuSZ "zu" +#endif + #ifndef SSIZE_MAX #if (SIZEOF_SIZE_T == 4) #define SSIZE_MAX INT32_MAX diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index 661438d8f..028fdbfc2 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -51,6 +51,8 @@ * logic, which was originally circuit-focused. **/ #define CIRCUITLIST_PRIVATE +#include "torint.h" /* TOR_PRIuSZ */ + #include "or.h" #include "channel.h" #include "circpathbias.h" @@ -2404,10 +2406,11 @@ circuits_handle_oom(size_t current_allocation) int n_circuits_killed=0; int n_dirconns_killed=0; uint32_t now_ts; - log_notice(LD_GENERAL, "We're low on memory (cell queues total alloc: %zu," - " buffer total alloc: %zu, tor compress total alloc: %zu," - " rendezvous cache total alloc: %zu). Killing circuits with" - " over-long queues. (This behavior is controlled by" + log_notice(LD_GENERAL, "We're low on memory (cell queues total alloc:" + " %"TOR_PRIuSZ" buffer total alloc: %" TOR_PRIuSZ "," + " tor compress total alloc: %" TOR_PRIuSZ + " rendezvous cache total alloc: %" TOR_PRIuSZ "). Killing" + " circuits withover-long queues. (This behavior is controlled by" " MaxMemInQueues.)", cell_queues_get_total_allocation(), buf_get_total_allocation(),

1 0

[tor/master] Remove a needless changefile. (Needless because bug not released.)
by nickm＠torproject.org 12 Jan '18

12 Jan '18

commit a86324d1fa7b8dec8f84d3ab8bd246780b962c92 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Jan 12 13:30:06 2018 -0500 Remove a needless changefile. (Needless because bug not released.) --- changes/ticket24861 | 5 ----- 1 file changed, 5 deletions(-) diff --git a/changes/ticket24861 b/changes/ticket24861 deleted file mode 100644 index beddb770b..000000000 --- a/changes/ticket24861 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfix (zu format specifier on minGW): - - Fix minGW compatibility issue. Define TOR_PRIuSZ as minGW compiler - doesn't support zu format specifier for size_t type. Fixes bug 24861; - bugfix on ac9eebd6363. - Patch by "ffmancera".

1 0

[translation/tor-browser-manual] Update translations for tor-browser-manual
by translation＠torproject.org 12 Jan '18

12 Jan '18

commit 9d6ec6d3a917fbacbab93338024fab070a5ead89 Author: Translation commit bot <translation(a)torproject.org> Date: Fri Jan 12 18:20:32 2018 +0000 Update translations for tor-browser-manual --- sv/sv.po | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/sv/sv.po b/sv/sv.po index 62336740a..64bc1a219 100644 --- a/sv/sv.po +++ b/sv/sv.po @@ -1276,7 +1276,7 @@ msgstr "Konfigurera Tor Browser för säkerhet och användbarhet" #: security-slider.page:10 msgid "Security Slider" -msgstr "" +msgstr "Säkerhetsreglage" #: security-slider.page:11 msgid "" @@ -1289,7 +1289,7 @@ msgstr "" #: security-slider.page:21 msgid "Accessing the Security Slider" -msgstr "" +msgstr "Åtkomst till säkerhetsreglaget" #. This is a reference to an external file such as an image or video. When #. the file changes, the md5 hash will change to let you know you need to @@ -1301,6 +1301,8 @@ msgid "" "external ref='media/security-slider/slider.png' " "md5='3c469cd3ed9f60ebb6bbbc63daa90082'" msgstr "" +"external ref='media/security-slider/slider.png' " +"md5='3c469cd3ed9f60ebb6bbbc63daa90082'" #: security-slider.page:25 msgid "" @@ -1322,6 +1324,8 @@ msgid "" "external ref='media/security-slider/slider_window.png' " "md5='c733bdccd1731ed1a772777b25bae7a1'" msgstr "" +"external ref='media/security-slider/slider_window.png' " +"md5='c733bdccd1731ed1a772777b25bae7a1'" #: security-slider.page:36 msgid ""

1 0

[translation/torbutton-torbuttondtd] Update translations for torbutton-torbuttondtd
by translation＠torproject.org 12 Jan '18

12 Jan '18

commit bb2b4d99fd7fc223c495b38cfa07bd142ad3c357 Author: Translation commit bot <translation(a)torproject.org> Date: Fri Jan 12 18:17:42 2018 +0000 Update translations for torbutton-torbuttondtd --- sv/torbutton.dtd | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/sv/torbutton.dtd b/sv/torbutton.dtd index f407136d9..aac0ea81d 100644 --- a/sv/torbutton.dtd +++ b/sv/torbutton.dtd @@ -34,17 +34,17 @@ <!ENTITY torbutton.prefs.sec_caption "Säkerhetsnivå"> <!ENTITY torbutton.prefs.sec_caption_tooltip "Säkerhetsreglaget låter dig stänga av vissa webbläsarfunktioner som kan göra din webbläsare mer sårbar mot intrångsförsök."> <!ENTITY torbutton.prefs.sec_standard_label "Standard"> -<!ENTITY torbutton.prefs.sec_standard_description "All Tor Browser and website features are enabled."> +<!ENTITY torbutton.prefs.sec_standard_description "Alla funktioner för Tor Browser och webbplatsfunktioner är aktiverade."> <!ENTITY torbutton.prefs.sec_safer_label "Säkrare"> -<!ENTITY torbutton.prefs.sec_safer_description "Disables website features that are often dangerous, causing some sites to lose functionality."> -<!ENTITY torbutton.prefs.sec_safer_list_label "At the safer setting:"> +<!ENTITY torbutton.prefs.sec_safer_description "Inaktiverar webbplatsfunktioner som ofta är farliga, vilket gör att vissa webbplatser förlorar funktionalitet."> +<!ENTITY torbutton.prefs.sec_safer_list_label "I den säkrare inställningen:"> <!ENTITY torbutton.prefs.sec_safest_label "Säkrast"> -<!ENTITY torbutton.prefs.sec_safest_description "Only allows website features required for static sites and basic services. These changes affect images, media, and scripts."> -<!ENTITY torbutton.prefs.sec_safest_list_label "At the safest setting:"> +<!ENTITY torbutton.prefs.sec_safest_description "Tillåt endast webbplatsfunktioner som krävs för statiska webbplatser och grundläggande tjänster. Dessa förändringar påverkar bilder, media och skript."> +<!ENTITY torbutton.prefs.sec_safest_list_label "I den säkraste inställningen:"> <!ENTITY torbutton.prefs.sec_learn_more_label "Läs mer"> -<!ENTITY torbutton.prefs.sec_js_on_https_sites_only "JavaScript is disabled on non-HTTPS sites."> -<!ENTITY torbutton.prefs.sec_js_disabled "JavaScript is disabled by default on all sites."> -<!ENTITY torbutton.prefs.sec_limit_typography "Some fonts and math symbols are disabled."> -<!ENTITY torbutton.prefs.sec_limit_graphics_and_typography "Some fonts, icons, math symbols, and images are disabled."> -<!ENTITY torbutton.prefs.sec_click_to_play_media "Audio and video (HTML5 media) are click-to-play."> +<!ENTITY torbutton.prefs.sec_js_on_https_sites_only "JavaScript är inaktiverat på webbplatser utan HTTPS."> +<!ENTITY torbutton.prefs.sec_js_disabled "JavaScript är inaktiverat som standard på alla webbplatser."> +<!ENTITY torbutton.prefs.sec_limit_typography "Vissa typsnitt och matte symboler är inaktiverade."> +<!ENTITY torbutton.prefs.sec_limit_graphics_and_typography "Vissa typsnitt, ikoner, matte symboler och bilder är inaktiverade."> +<!ENTITY torbutton.prefs.sec_click_to_play_media "Ljud och video (HTML5-media) är klicka-för-att-spel."> <!ENTITY torbutton.circuit_display.title "Tor-krets för den här webbplatsen">

1 0

[translation/torbutton-torbuttonproperties] Update translations for torbutton-torbuttonproperties
by translation＠torproject.org 12 Jan '18

12 Jan '18

commit 6dd8baf260e061894a470ec0251348867b5f1aa7 Author: Translation commit bot <translation(a)torproject.org> Date: Fri Jan 12 18:17:36 2018 +0000 Update translations for torbutton-torbuttonproperties --- sv/torbutton.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sv/torbutton.properties b/sv/torbutton.properties index 2656cc13e..45bd229fd 100644 --- a/sv/torbutton.properties +++ b/sv/torbutton.properties @@ -31,7 +31,7 @@ torbutton.popup.confirm_newnym = Tor Browser kommer att stänga alla fönster oc torbutton.slider_notification = Den gröna onion menyn har nu en säkerhetsinställning. Kolla in den! torbutton.slider_notification_button = Öppna säkerhetsinställningar -torbutton.maximize_warning = Att maximera Tor Browser kan ge webbsajter möjlighet att upptäcka din monitors storlek, vilket kan användas för att spåra dig. Vi rekommenderar att du lämnar Tor Browser fönstret i dess ursprungliga storlek. +torbutton.maximize_warning = Att maximera Tor Browser kan ge webbplatser möjlighet att upptäcka din monitors storlek, vilket kan användas för att spåra dig. Vi rekommenderar att du lämnar Tor Browser fönstret i dess ursprungliga storlek. # Canvas permission prompt. Strings are kept here for ease of translation. canvas.siteprompt=Denna webbplats (%S) försökte komma åt HTML5-bilddata på ett kanvas, som skulle kunna användas för att identifiera just din dator.\n\nSka Tor Browser tillåta den här webbplatsen att använda HTML5-bilddata på kanvas?

1 0

[stem/master] Drop v1 fallback directory parsing
by atagar＠torproject.org 12 Jan '18

12 Jan '18

commit 88800b93911d8041d38f4ea06081fb053b2aab09 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jan 7 11:32:26 2018 -0800 Drop v1 fallback directory parsing We only need parsing for the present format (since the old format isn't available anywhere). This is only used for fetching and caching from github after all. --- stem/descriptor/remote.py | 62 +----------------------------------------- test/unit/descriptor/remote.py | 41 ++-------------------------- 2 files changed, 4 insertions(+), 99 deletions(-) diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index 8e4fa219..13fa2893 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -1049,67 +1049,7 @@ class FallbackDirectory(Directory): exc = sys.exc_info()[1] raise IOError("Unable to download tor's fallback directories from %s: %s" % (GITWEB_FALLBACK_DIR_URL, exc)) - if '/* nickname=' in fallback_dir_page: - return FallbackDirectory._parse_v2(fallback_dir_page) - else: - return FallbackDirectory._parse_v1(fallback_dir_page) - - @staticmethod - def _parse_v1(fallback_dir_page): - # Example of an entry... - # - # "5.175.233.86:80 orport=443 id=5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33" - # " ipv6=[2a03:b0c0:0:1010::a4:b001]:9001" - # " weight=43680", - - # TODO: this method can be removed once gitweb provides a v2 formatted document - - results, attr = {}, {} - - for line in fallback_dir_page.splitlines(): - if line.startswith('"'): - addr_line_match = re.match('"([\d\.]+):(\d+) orport=(\d+) id=([\dA-F]{40}).*', line) - ipv6_line_match = re.match('" ipv6=\[([\da-f:]+)\]:(\d+)"', line) - - if addr_line_match: - address, dir_port, or_port, fingerprint = addr_line_match.groups() - - if not connection.is_valid_ipv4_address(address): - raise IOError('%s has an invalid IPv4 address: %s' % (fingerprint, address)) - elif not connection.is_valid_port(or_port): - raise IOError('%s has an invalid or_port: %s' % (fingerprint, or_port)) - elif not connection.is_valid_port(dir_port): - raise IOError('%s has an invalid dir_port: %s' % (fingerprint, dir_port)) - elif not tor_tools.is_valid_fingerprint(fingerprint): - raise IOError('%s has an invalid fingerprint: %s' % (fingerprint, fingerprint)) - - attr = { - 'address': address, - 'or_port': int(or_port), - 'dir_port': int(dir_port), - 'fingerprint': fingerprint, - } - elif ipv6_line_match: - address, port = ipv6_line_match.groups() - - if not connection.is_valid_ipv6_address(address): - raise IOError('%s has an invalid IPv6 address: %s' % (fingerprint, address)) - elif not connection.is_valid_port(port): - raise IOError('%s has an invalid ORPort for its IPv6 endpoint: %s' % (fingerprint, port)) - - attr['orport_v6'] = (address, int(port)) - elif line.startswith('" weight=') and 'fingerprint' in attr: - results[attr.get('fingerprint')] = FallbackDirectory( - address = attr.get('address'), - or_port = attr.get('or_port'), - dir_port = attr.get('dir_port'), - fingerprint = attr.get('fingerprint'), - orport_v6 = attr.get('orport_v6'), - ) - - attr = {} - - return results + return FallbackDirectory._parse_v2(fallback_dir_page) @staticmethod def _parse_v2(fallback_dir_page): diff --git a/test/unit/descriptor/remote.py b/test/unit/descriptor/remote.py index 69ffd8e3..414be7e8 100644 --- a/test/unit/descriptor/remote.py +++ b/test/unit/descriptor/remote.py @@ -58,19 +58,7 @@ iO3EUE0AEYah2W9gdz8t+i3Dtr0zgqLS841GC/TyDKCm+MKmN8d098qnwK0NGF9q -----END SIGNATURE----- """ -FALLBACK_DIR_CONTENT_V1 = b"""\ -/* Trial fallbacks for 0.2.8.1-alpha with ADDRESS_AND_PORT_STABLE_DAYS = 30 - * This works around an issue where relays post a descriptor without a DirPort - * when restarted. If these relays stay up, they will have been up for 120 days - * by the 0.2.8 stable release -- teor */ -"5.175.233.86:80 orport=443 id=5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33" -" weight=43680", -"62.210.124.124:9130 orport=9101 id=2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E" -" ipv6=[2001:bc8:3f23:100::1]:9101" -" weight=43680", -""" - -FALLBACK_DIR_CONTENT_V2 = b"""\ +FALLBACK_DIR_CONTENT = b"""\ "5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB" " ipv6=[2a01:4f8:162:51e2::2]:9001" /* nickname=rueckgrat */ @@ -185,31 +173,8 @@ class TestDescriptorDownloader(unittest.TestCase): self.assertEqual('5.39.92.199', fallback_directories['0BEA4A88D069753218EAAAD6D22EA87B9A1319D6'].address) @patch(URL_OPEN) - def test_fallback_directories_from_remote_v1(self, urlopen_mock): - urlopen_mock.return_value = io.BytesIO(FALLBACK_DIR_CONTENT_V1) - fallback_directories = stem.descriptor.remote.FallbackDirectory.from_remote() - - expected = { - '5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33': stem.descriptor.remote.FallbackDirectory( - address = '5.175.233.86', - or_port = 443, - dir_port = 80, - fingerprint = '5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33', - ), - '2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E': stem.descriptor.remote.FallbackDirectory( - address = '62.210.124.124', - or_port = 9101, - dir_port = 9130, - fingerprint = '2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E', - orport_v6 = ('2001:bc8:3f23:100::1', 9101), - ), - } - - self.assertEqual(expected, fallback_directories) - - @patch(URL_OPEN) - def test_fallback_directories_from_remote_v2(self, urlopen_mock): - urlopen_mock.return_value = io.BytesIO(FALLBACK_DIR_CONTENT_V2) + def test_fallback_directories_from_remote(self, urlopen_mock): + urlopen_mock.return_value = io.BytesIO(FALLBACK_DIR_CONTENT) fallback_directories = stem.descriptor.remote.FallbackDirectory.from_remote() expected = {

1 0

[stem/master] Support new v2 fallback format
by atagar＠torproject.org 12 Jan '18

12 Jan '18

commit ae6583197cceefcf88b09e5a8c05920ca3ecfe50 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jan 7 19:03:43 2018 -0800 Support new v2 fallback format Iterating on the fallback format has changed it a bit to be easier to parse and include header metadata. Supporting this new format. --- docs/change_log.rst | 1 + stem/descriptor/remote.py | 228 +++++++++++++++++++++++++++-------------- test/unit/descriptor/remote.py | 35 +++++++ 3 files changed, 189 insertions(+), 75 deletions(-) diff --git a/docs/change_log.rst b/docs/change_log.rst index c1af4f58..2d2490b5 100644 --- a/docs/change_log.rst +++ b/docs/change_log.rst @@ -51,6 +51,7 @@ The following are only available within Stem's `git repository * **Descriptors** + * `Fallback directory v2 support <https://lists.torproject.org/pipermail/tor-dev/2017-December/012721.html>`_, which adds *nickname* and *extrainfo* * Reduced maximum descriptors fetched by the remote module to match tor's new limit (:trac:`24743`) * **Website** diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index 13fa2893..1f002e27 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -12,7 +12,7 @@ information... for desc in stem.descriptor.remote.get_server_descriptors(): if desc.exit_policy.is_exiting_allowed(): - print ' %s (%s)' % (desc.nickname, desc.fingerprint) + print(' %s (%s)' % (desc.nickname, desc.fingerprint)) More custom downloading behavior can be done through the :class:`~stem.descriptor.remote.DescriptorDownloader` class, which issues @@ -30,17 +30,17 @@ content. For example... query = downloader.get_server_descriptors() - print 'Exit Relays:' + print('Exit Relays:') try: for desc in query.run(): if desc.exit_policy.is_exiting_allowed(): - print ' %s (%s)' % (desc.nickname, desc.fingerprint) + print(' %s (%s)' % (desc.nickname, desc.fingerprint)) print - print 'Query took %0.2f seconds' % query.runtime + print('Query took %0.2f seconds' % query.runtime) except Exception as exc: - print 'Unable to retrieve the server descriptors: %s' % exc + print('Unable to retrieve the server descriptors: %s' % exc) :: @@ -112,6 +112,14 @@ MAX_MICRODESCRIPTOR_HASHES = 90 GITWEB_FALLBACK_DIR_URL = 'https://gitweb.torproject.org/tor.git/plain/src/or/fallback_dirs.inc' CACHE_PATH = os.path.join(os.path.dirname(__file__), 'fallback_directories.cfg') +FALLBACK_DIV = '/* ===== */' +FALLBACK_MAPPING = re.compile('/\*\s+(\S+)=(\S*)\s+\*/') + +FALLBACK_ADDR = re.compile('"([\d\.]+):(\d+) orport=(\d+) id=([\dA-F]{40}).*') +FALLBACK_NICKNAME = re.compile('/\* nickname=(\S+) \*/') +FALLBACK_EXTRAINFO = re.compile('/\* extrainfo=([0-1]) \*/') +FALLBACK_IPV6 = re.compile('" ipv6=\[([\da-f:]+)\]:(\d+)"') + SINGLETON_DOWNLOADER = None @@ -215,22 +223,22 @@ class Query(object): timeout = 30, ) - print 'Current relays:' + print('Current relays:') if not query.error: for desc in query: - print desc.fingerprint + print(desc.fingerprint) else: - print 'Unable to retrieve the server descriptors: %s' % query.error + print('Unable to retrieve the server descriptors: %s' % query.error) ... while iterating fails silently... :: - print 'Current relays:' + print('Current relays:') for desc in Query('/tor/server/all.z', 'server-descriptor 1.0'): - print desc.fingerprint + print(desc.fingerprint) In either case exceptions are available via our 'error' attribute. @@ -568,7 +576,7 @@ class DescriptorDownloader(object): return self.query(resource, **query_args) - # TODO: drop in python 2.x + # TODO: drop in stem 2.x def get_microdescriptors(self, hashes, **query_args): """ @@ -940,10 +948,8 @@ class FallbackDirectory(Directory): .. versionadded:: 1.5.0 .. versionchanged:: 1.7.0 - Added the nickname and has_extrainfo attributes. - - .. versionchanged:: 1.7.0 - Support for parsing `second version of the fallback directories + Added the nickname and has_extrainfo attributes, which are part of the + `second version of the fallback directories <https://lists.torproject.org/pipermail/tor-dev/2017-December/012721.html>`_. :var str nickname: relay nickname @@ -1044,80 +1050,152 @@ class FallbackDirectory(Directory): """ try: - fallback_dir_page = str_tools._to_unicode(urllib.urlopen(GITWEB_FALLBACK_DIR_URL, timeout = timeout).read()) + lines = str_tools._to_unicode(urllib.urlopen(GITWEB_FALLBACK_DIR_URL, timeout = timeout).read()).splitlines() except: exc = sys.exc_info()[1] raise IOError("Unable to download tor's fallback directories from %s: %s" % (GITWEB_FALLBACK_DIR_URL, exc)) - return FallbackDirectory._parse_v2(fallback_dir_page) + if not lines: + raise IOError('%s did not have any content' % GITWEB_FALLBACK_DIR_URL) + elif lines[0] != '/* type=fallback */': + raise IOError('%s does not have a type field indicating it is fallback directory metadata' % GITWEB_FALLBACK_DIR_URL) + + # header metadata + + header = {} + + for line in FallbackDirectory._pop_section(lines): + mapping = FALLBACK_MAPPING.match(line) + + if mapping: + header[mapping.group(1)] = mapping.group(2) + else: + raise IOError('Malformed fallback directory header line: %s' % line) + + # human readable comments + + FallbackDirectory._pop_section(lines) + + # content, everything remaining are fallback directories + + results = {} + + while lines: + # Example of an entry... + # + # "5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB" + # " ipv6=[2a01:4f8:162:51e2::2]:9001" + # /* nickname=rueckgrat */ + # /* extrainfo=1 */ + + section = FallbackDirectory._pop_section(lines) + + if not section: + continue + + address, or_port, dir_port, fingerprint = FallbackDirectory._parse_addr(section) + + results[fingerprint] = FallbackDirectory( + address = address, + or_port = or_port, + dir_port = dir_port, + fingerprint = fingerprint, + nickname = FallbackDirectory._parse_nickname(section, fingerprint), + has_extrainfo = FallbackDirectory._parse_has_extrainfo(section), + orport_v6 = FallbackDirectory._parse_ipv6(section, fingerprint), + ) + + return results @staticmethod - def _parse_v2(fallback_dir_page): - # Example of an entry... - # - # "5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB" - # " ipv6=[2a01:4f8:162:51e2::2]:9001" - # /* nickname=rueckgrat */ - # /* extrainfo=1 */ - - results, attr = {}, {} - - for line in fallback_dir_page.splitlines(): - addr_line_match = re.match('"([\d\.]+):(\d+) orport=(\d+) id=([\dA-F]{40}).*', line) - nickname_match = re.match('/\* nickname=(\S+) \*/', line) - has_extrainfo_match = re.match('/\* extrainfo=([0-1]) \*/', line) - ipv6_line_match = re.match('" ipv6=\[([\da-f:]+)\]:(\d+)"', line) - - if addr_line_match: - address, dir_port, or_port, fingerprint = addr_line_match.groups() - - if not connection.is_valid_ipv4_address(address): - raise IOError('%s has an invalid IPv4 address: %s' % (fingerprint, address)) - elif not connection.is_valid_port(or_port): - raise IOError('%s has an invalid or_port: %s' % (fingerprint, or_port)) - elif not connection.is_valid_port(dir_port): - raise IOError('%s has an invalid dir_port: %s' % (fingerprint, dir_port)) - elif not tor_tools.is_valid_fingerprint(fingerprint): - raise IOError('%s has an invalid fingerprint: %s' % (fingerprint, fingerprint)) - - attr = { - 'address': address, - 'or_port': int(or_port), - 'dir_port': int(dir_port), - 'fingerprint': fingerprint, - } - elif ipv6_line_match: - address, port = ipv6_line_match.groups() + def _pop_section(lines): + """ + Provides lines up through the next divider. This excludes lines with just a + comma since they're an artifact of these being C strings. + """ - if not connection.is_valid_ipv6_address(address): - raise IOError('%s has an invalid IPv6 address: %s' % (fingerprint, address)) - elif not connection.is_valid_port(port): - raise IOError('%s has an invalid ORPort for its IPv6 endpoint: %s' % (fingerprint, port)) + section_lines = [] + + if lines: + line = lines.pop(0) + + while lines and line != FALLBACK_DIV: + if line.strip() != ',': + section_lines.append(line) - attr['orport_v6'] = (address, int(port)) - elif nickname_match: - nickname = nickname_match.group(1) + line = lines.pop(0) + + return section_lines + + @staticmethod + def _parse_addr(lines): + """ + Provides the mandatory address information of a fallback. + """ + + addr_match = FALLBACK_ADDR.match(lines[0]) + + if not addr_match: + raise IOError('Malformed initial fallback line:\n\n%s' % '\n'.join(lines)) + + address, dir_port, or_port, fingerprint = addr_match.groups() + + if not connection.is_valid_ipv4_address(address): + raise IOError('%s has an invalid IPv4 address: %s' % (fingerprint, address)) + elif not connection.is_valid_port(or_port): + raise IOError('%s has an invalid or_port: %s' % (fingerprint, or_port)) + elif not connection.is_valid_port(dir_port): + raise IOError('%s has an invalid dir_port: %s' % (fingerprint, dir_port)) + elif not tor_tools.is_valid_fingerprint(fingerprint): + raise IOError('%s has an invalid fingerprint: %s' % (fingerprint, fingerprint)) + + return address, int(or_port), int(dir_port), fingerprint + + @staticmethod + def _parse_nickname(lines, fingerprint): + """ + Provides the nickname of the fallback. + """ + + for line in lines: + match = FALLBACK_NICKNAME.match(line) + + if match: + nickname = match.group(1) if not tor_tools.is_valid_nickname(nickname): raise IOError('%s has an invalid nickname: %s' % (fingerprint, nickname)) - attr['nickname'] = nickname - elif has_extrainfo_match: - attr['has_extrainfo'] = has_extrainfo_match.group(1) == '1' - - results[attr.get('fingerprint')] = FallbackDirectory( - address = attr.get('address'), - or_port = attr.get('or_port'), - dir_port = attr.get('dir_port'), - fingerprint = attr.get('fingerprint'), - nickname = attr.get('nickname'), - has_extrainfo = attr.get('has_extrainfo', False), - orport_v6 = attr.get('orport_v6'), - ) + return nickname + + @staticmethod + def _parse_has_extrainfo(lines): + for line in lines: + match = FALLBACK_EXTRAINFO.match(line) - attr = {} + if match: + return match.group(1) == '1' - return results + return False + + @staticmethod + def _parse_ipv6(lines, fingerprint): + """ + Provides the IPv6 ORPort for a fallback. + """ + + for line in lines: + match = FALLBACK_IPV6.match(line) + + if match: + address, port = match.groups() + + if not connection.is_valid_ipv6_address(address): + raise IOError('%s has an invalid IPv6 address: %s' % (fingerprint, address)) + elif not connection.is_valid_port(port): + raise IOError('%s has an invalid ORPort for its IPv6 endpoint: %s' % (fingerprint, port)) + + return (address, int(port)) def __hash__(self): return _hash_attr(self, 'orport_v6', parent = Directory) diff --git a/test/unit/descriptor/remote.py b/test/unit/descriptor/remote.py index 414be7e8..17e17595 100644 --- a/test/unit/descriptor/remote.py +++ b/test/unit/descriptor/remote.py @@ -59,10 +59,36 @@ iO3EUE0AEYah2W9gdz8t+i3Dtr0zgqLS841GC/TyDKCm+MKmN8d098qnwK0NGF9q """ FALLBACK_DIR_CONTENT = b"""\ +/* type=fallback */ +/* version=2.0.0 */ +/* timestamp=20170526090242 */ +/* ===== */ +/* Whitelist & blacklist excluded 1326 of 1513 candidates. */ +/* Checked IPv4 DirPorts served a consensus within 15.0s. */ +/* +Final Count: 151 (Eligible 187, Target 392 (1963 * 0.20), Max 200) +Excluded: 36 (Same Operator 27, Failed/Skipped Download 9, Excess 0) +Bandwidth Range: 1.3 - 40.0 MByte/s +*/ +/* +Onionoo Source: details Date: 2017-05-16 07:00:00 Version: 4.0 +URL: https:onionoo.torproject.orgdetails?fields=fingerprint%2Cnickname%2Ccontact%2Clast_changed_address_or_port%2Cconsensus_weight%2Cadvertised_bandwidth%2Cor_addresses%2Cdir_address%2Crecommended_version%2Cflags%2Ceffective_family%2Cplatform&flag=V2Dir&type=relay&last_seen_days=-0&first_seen_days=30- +*/ +/* +Onionoo Source: uptime Date: 2017-05-16 07:00:00 Version: 4.0 +URL: https:onionoo.torproject.orguptime?first_seen_days=30-&flag=V2Dir&type=relay&last_seen_days=-0 +*/ +/* ===== */ "5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB" " ipv6=[2a01:4f8:162:51e2::2]:9001" /* nickname=rueckgrat */ /* extrainfo=1 */ +/* ===== */ +, +"193.171.202.146:9030 orport=9001 id=01A9258A46E97FF8B2CAC7910577862C14F2C524" +/* nickname= */ +/* extrainfo=0 */ +/* ===== */ """ @@ -187,6 +213,15 @@ class TestDescriptorDownloader(unittest.TestCase): has_extrainfo = True, orport_v6 = ('2a01:4f8:162:51e2::2', 9001), ), + '01A9258A46E97FF8B2CAC7910577862C14F2C524': stem.descriptor.remote.FallbackDirectory( + address = '193.171.202.146', + or_port = 9001, + dir_port = 9030, + fingerprint = '01A9258A46E97FF8B2CAC7910577862C14F2C524', + nickname = None, + has_extrainfo = False, + orport_v6 = None, + ), } self.assertEqual(expected, fallback_directories)

1 0

[stem/master] Raise ValueErrors from FallbackDirectory.from_str
by atagar＠torproject.org 12 Jan '18

12 Jan '18

commit ef5b0ef03c7ab8e5d10ee2d4364e8af540bb6528 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Jan 9 09:32:34 2018 -0800 Raise ValueErrors from FallbackDirectory.from_str More appropriate for this to raise a ValueError, and have from_remote() recast it as an IOError since only the later concerns content on the wire. --- stem/descriptor/remote.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index e8d5e585..b318f4e3 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -1084,8 +1084,11 @@ class FallbackDirectory(Directory): section = FallbackDirectory._pop_section(lines) if section: - fallback = FallbackDirectory.from_str('\n'.join(section)) - results[fallback.fingerprint] = fallback + try: + fallback = FallbackDirectory.from_str('\n'.join(section)) + results[fallback.fingerprint] = fallback + except ValueError as exc: + raise IOError(str(exc)) return results @@ -1107,7 +1110,7 @@ class FallbackDirectory(Directory): :returns: :class:`~stem.descriptor.remote.FallbackDirectory` in the text - :raises: **IOError** if content is malformed + :raises: **ValueError** if content is malformed """ matches = {} @@ -1121,7 +1124,7 @@ class FallbackDirectory(Directory): matches[matcher] = match_groups if len(match_groups) > 1 else match_groups[0] if FALLBACK_ADDR not in matches: - raise IOError('Malformed fallback address line:\n\n%s' % content) + raise ValueError('Malformed fallback address line:\n\n%s' % content) address, dir_port, or_port, fingerprint = matches[FALLBACK_ADDR] nickname = matches.get(FALLBACK_NICKNAME) @@ -1129,19 +1132,19 @@ class FallbackDirectory(Directory): orport_v6 = matches.get(FALLBACK_IPV6) if not connection.is_valid_ipv4_address(address): - raise IOError('%s has an invalid IPv4 address: %s' % (fingerprint, address)) + raise ValueError('%s has an invalid IPv4 address: %s' % (fingerprint, address)) elif not connection.is_valid_port(or_port): - raise IOError('%s has an invalid or_port: %s' % (fingerprint, or_port)) + raise ValueError('%s has an invalid or_port: %s' % (fingerprint, or_port)) elif not connection.is_valid_port(dir_port): - raise IOError('%s has an invalid dir_port: %s' % (fingerprint, dir_port)) + raise ValueError('%s has an invalid dir_port: %s' % (fingerprint, dir_port)) elif not tor_tools.is_valid_fingerprint(fingerprint): - raise IOError('%s has an invalid fingerprint: %s' % (fingerprint, fingerprint)) + raise ValueError('%s has an invalid fingerprint: %s' % (fingerprint, fingerprint)) elif nickname and not tor_tools.is_valid_nickname(nickname): - raise IOError('%s has an invalid nickname: %s' % (fingerprint, nickname)) + raise ValueError('%s has an invalid nickname: %s' % (fingerprint, nickname)) elif orport_v6 and not connection.is_valid_ipv6_address(orport_v6[0]): - raise IOError('%s has an invalid IPv6 address: %s' % (fingerprint, orport_v6[0])) + raise ValueError('%s has an invalid IPv6 address: %s' % (fingerprint, orport_v6[0])) elif orport_v6 and not connection.is_valid_port(orport_v6[1]): - raise IOError('%s has an invalid ORPort for its IPv6 endpoint: %s' % (fingerprint, orport_v6[1])) + raise ValueError('%s has an invalid ORPort for its IPv6 endpoint: %s' % (fingerprint, orport_v6[1])) return FallbackDirectory( address = address,

1 0

[stem/master] Include fallback header metadata
by atagar＠torproject.org 12 Jan '18

12 Jan '18

commit cb883a0cb3d62fd16bad740041ed28516cc27e79 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Jan 11 09:20:48 2018 -0800 Include fallback header metadata Presently only included when parsing. Adding persistence next. --- stem/descriptor/remote.py | 11 +++++++---- test/unit/descriptor/remote.py | 3 +++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index 0a372dce..8f5e5cc3 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -948,8 +948,8 @@ class FallbackDirectory(Directory): .. versionadded:: 1.5.0 .. versionchanged:: 1.7.0 - Added the nickname and has_extrainfo attributes, which are part of the - `second version of the fallback directories + Added the nickname, has_extrainfo, and header attributes which are part of + the `second version of the fallback directories <https://lists.torproject.org/pipermail/tor-dev/2017-December/012721.html>`_. :var str nickname: relay nickname @@ -957,14 +957,16 @@ class FallbackDirectory(Directory): extrainfo descriptors, **False** otherwise. :var str orport_v6: **(address, port)** tuple for the directory's IPv6 ORPort, or **None** if it doesn't have one + :var dict header: metadata about the fallback directory file this originated from """ - def __init__(self, address = None, or_port = None, dir_port = None, fingerprint = None, nickname = None, has_extrainfo = False, orport_v6 = None): + def __init__(self, address = None, or_port = None, dir_port = None, fingerprint = None, nickname = None, has_extrainfo = False, orport_v6 = None, header = None): super(FallbackDirectory, self).__init__(address, or_port, dir_port, fingerprint) self.nickname = nickname self.has_extrainfo = has_extrainfo self.orport_v6 = orport_v6 + self.header = header if header else {} @staticmethod def from_cache(): @@ -1086,6 +1088,7 @@ class FallbackDirectory(Directory): if section: try: fallback = FallbackDirectory.from_str('\n'.join(section)) + fallback.header = header results[fallback.fingerprint] = fallback except ValueError as exc: raise IOError(str(exc)) @@ -1177,7 +1180,7 @@ class FallbackDirectory(Directory): return section_lines def __hash__(self): - return _hash_attr(self, 'address', 'or_port', 'dir_port', 'fingerprint', 'nickname', 'has_extrainfo', 'orport_v6', parent = Directory) + return _hash_attr(self, 'address', 'or_port', 'dir_port', 'fingerprint', 'nickname', 'has_extrainfo', 'orport_v6', 'header', parent = Directory) def __eq__(self, other): return hash(self) == hash(other) if isinstance(other, FallbackDirectory) else False diff --git a/test/unit/descriptor/remote.py b/test/unit/descriptor/remote.py index 7789ed21..cc0d51b1 100644 --- a/test/unit/descriptor/remote.py +++ b/test/unit/descriptor/remote.py @@ -209,6 +209,7 @@ class TestDescriptorDownloader(unittest.TestCase): def test_fallback_directories_from_remote(self, urlopen_mock): urlopen_mock.return_value = io.BytesIO(FALLBACK_DIR_CONTENT) fallback_directories = stem.descriptor.remote.FallbackDirectory.from_remote() + header = {'type': 'fallback', 'version': '2.0.0', 'timestamp': '20170526090242'} expected = { '0756B7CD4DFC8182BE23143FAC0642F515182CEB': stem.descriptor.remote.FallbackDirectory( @@ -219,6 +220,7 @@ class TestDescriptorDownloader(unittest.TestCase): nickname = 'rueckgrat', has_extrainfo = True, orport_v6 = ('2a01:4f8:162:51e2::2', 9001), + header = header, ), '01A9258A46E97FF8B2CAC7910577862C14F2C524': stem.descriptor.remote.FallbackDirectory( address = '193.171.202.146', @@ -228,6 +230,7 @@ class TestDescriptorDownloader(unittest.TestCase): nickname = None, has_extrainfo = False, orport_v6 = None, + header = header, ), }

1 0