commit 8ef2fd199baf954d96dabb362151cc594dd25869 Author: Damian Johnson <atagar@torproject.org> Date: Sat Jul 6 17:37:46 2019 -0700 Convert CollecTor indices to key off filenames Why are CollecTor 'files' and 'directories' attributes lists? This only makes sense to allow duplicate filenames (which doesn't make sense). Otherwise it simply makes directory transversal a pita. Converting index files/directories into hashes keyed off filenames so we can more efficiently (and easily) work with them. --- stem/descriptor/collector.py | 27 +- test/unit/descriptor/collector.py | 594 ++++++++++++++++++-------------------- 2 files changed, 304 insertions(+), 317 deletions(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index d723e97a..2633558e 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -121,6 +121,31 @@ def _download(url, compression, timeout, retries): return stem.util.str_tools._to_unicode(response) +def _convert_index_paths(val): + """ + Key files and directories off their paths so we can work with them more + efficiently. + + :val dict val: index to convert + + :returns: index with files and directories converted to path-keyed hashes + + :raises: **ValueError** if unable to parse the index + """ + + if isinstance(val, dict): + return dict([(k, _convert_index_paths(v)) for (k, v) in val.items()]) + elif isinstance(val, list) and all([isinstance(entry, dict) and 'path' in entry for entry in val]): + contents = {} + + for entry in val: + contents[entry['path']] = dict((k, _convert_index_paths(v)) for (k, v) in entry.items() if k != 'path') + + return contents + else: + return val + + class CollecTor(object): """ Downloader for descriptors from CollecTor. The contents of CollecTor are @@ -167,7 +192,7 @@ class CollecTor(object): if not self._cached_index or time.time() - self._cached_index_at >= REFRESH_INDEX_RATE: response = _download(COLLECTOR_URL + 'index/index.json', self.compression, self.timeout, self.retries) - self._cached_index = json.loads(response) + self._cached_index = _convert_index_paths(json.loads(response)) self._cached_index_at = time.time() return self._cached_index diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index e1ff4bdb..267a31a9 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -31,529 +31,491 @@ EXAMPLE_INDEX = { 'index_created': '2019-07-06 01:54', 'build_revision': 'df98ac79', 'path': 'https://collector.torproject.org', - 'directories': [ - { - 'path': 'archive', - 'directories': [ - { - 'path': 'bridge-descriptors', - 'directories': [ - { - 'path': 'extra-infos', - 'files': [ - { - 'path': 'bridge-extra-infos-2008-05.tar.xz', + 'directories': { + 'archive': { + 'directories': { + 'bridge-descriptors': { + 'directories': { + 'extra-infos': { + 'files': { + 'bridge-extra-infos-2008-05.tar.xz': { 'size': 377644, 'last_modified': '2016-09-04 09:21' - }, { - 'path': 'bridge-extra-infos-2008-06.tar.xz', + }, + 'bridge-extra-infos-2008-06.tar.xz': { 'size': 600484, 'last_modified': '2016-09-04 09:21' - }, { - 'path': 'bridge-extra-infos-2008-07.tar.xz', + }, + 'bridge-extra-infos-2008-07.tar.xz': { 'size': 716320, 'last_modified': '2016-09-04 09:21' } - ] - }, { - 'path': 'server-descriptors', - 'files': [ - { - 'path': 'bridge-server-descriptors-2008-05.tar.xz', + } + }, + 'server-descriptors': { + 'files': { + 'bridge-server-descriptors-2008-05.tar.xz': { 'size': 205348, 'last_modified': '2016-09-09 14:13' - }, { - 'path': 'bridge-server-descriptors-2008-06.tar.xz', + }, + 'bridge-server-descriptors-2008-06.tar.xz': { 'size': 342828, 'last_modified': '2016-09-09 14:13' - }, { - 'path': 'bridge-server-descriptors-2008-07.tar.xz', + }, + 'bridge-server-descriptors-2008-07.tar.xz': { 'size': 374848, 'last_modified': '2016-09-09 14:13' } - ] - }, { - 'path': 'statuses', - 'files': [ - { - 'path': 'bridge-statuses-2008-05.tar.xz', + } + }, + 'statuses': { + 'files': { + 'bridge-statuses-2008-05.tar.xz': { 'size': 74792, 'last_modified': '2016-09-14 21:11' - }, { - 'path': 'bridge-statuses-2008-06.tar.xz', + }, + 'bridge-statuses-2008-06.tar.xz': { 'size': 123488, 'last_modified': '2016-09-14 21:11' - }, { - 'path': 'bridge-statuses-2008-07.tar.xz', + }, + 'bridge-statuses-2008-07.tar.xz': { 'size': 143836, 'last_modified': '2016-09-14 21:11' } - ] + } } - ] - }, { - 'path': 'bridge-pool-assignments', - 'files': [ - { - 'path': 'bridge-pool-assignments-2010-09.tar.xz', + } + }, + 'bridge-pool-assignments': { + 'files': { + 'bridge-pool-assignments-2010-09.tar.xz': { 'size': 32804, 'last_modified': '2012-05-31 10:21' - }, { - 'path': 'bridge-pool-assignments-2010-10.tar.xz', + }, + 'bridge-pool-assignments-2010-10.tar.xz': { 'size': 304684, 'last_modified': '2012-05-31 10:21' - }, { - 'path': 'bridge-pool-assignments-2010-11.tar.xz', + }, + 'bridge-pool-assignments-2010-11.tar.xz': { 'size': 292228, 'last_modified': '2012-05-31 10:21' } - ] - }, { - 'path': 'exit-lists', - 'files': [ - { - 'path': 'exit-list-2010-02.tar.xz', + } + }, + 'exit-lists': { + 'files': { + 'exit-list-2010-02.tar.xz': { 'size': 272008, 'last_modified': '2012-05-31 18:57' - }, { - 'path': 'exit-list-2010-03.tar.xz', + }, + 'exit-list-2010-03.tar.xz': { 'size': 1247484, 'last_modified': '2012-05-31 18:57' - }, { - 'path': 'exit-list-2010-04.tar.xz', + }, + 'exit-list-2010-04.tar.xz': { 'size': 1139896, 'last_modified': '2012-05-31 18:57' } - ] - }, { - 'path': 'relay-descriptors', - 'files': [ - { - 'path': 'certs.tar.xz', + } + }, + 'relay-descriptors': { + 'files': { + 'certs.tar.xz': { 'size': 144696, 'last_modified': '2019-07-03 03:29' } - ], - 'directories': [ - { - 'path': 'bandwidths', - 'files': [ - { - 'path': 'bandwidths-2019-05.tar.xz', + }, + 'directories': { + 'bandwidths': { + 'files': { + 'bandwidths-2019-05.tar.xz': { 'size': 50385816, 'last_modified': '2019-06-07 08:05' - }, { - 'path': 'bandwidths-2019-06.tar.xz', + }, + 'bandwidths-2019-06.tar.xz': { 'size': 105881156, 'last_modified': '2019-07-03 07:30' - }, { - 'path': 'bandwidths-2019-07.tar.xz', + }, + 'bandwidths-2019-07.tar.xz': { 'size': 11374436, 'last_modified': '2019-07-03 07:12' } - ] - }, { - 'path': 'consensuses', - 'files': [ - { - 'path': 'consensuses-2007-10.tar.xz', + } + }, + 'consensuses': { + 'files': { + 'consensuses-2007-10.tar.xz': { 'size': 1061648, 'last_modified': '2012-05-15 14:35' - }, { - 'path': 'consensuses-2007-11.tar.xz', + }, + 'consensuses-2007-11.tar.xz': { 'size': 6810308, 'last_modified': '2012-05-15 14:35' - }, { - 'path': 'consensuses-2007-12.tar.xz', + }, + 'consensuses-2007-12.tar.xz': { 'size': 8106968, 'last_modified': '2012-05-15 14:35' } - ] - }, { - 'path': 'extra-infos', - 'files': [ - { - 'path': 'extra-infos-2007-08.tar.xz', + } + }, + 'extra-infos': { + 'files': { + 'extra-infos-2007-08.tar.xz': { 'size': 3016916, 'last_modified': '2016-06-23 09:53' - }, { - 'path': 'extra-infos-2007-09.tar.xz', + }, + 'extra-infos-2007-09.tar.xz': { 'size': 6459884, 'last_modified': '2016-06-23 09:54' - }, { - 'path': 'extra-infos-2007-10.tar.xz', + }, + 'extra-infos-2007-10.tar.xz': { 'size': 7326564, 'last_modified': '2016-06-23 09:54' } - ] - }, { - 'path': 'microdescs', - 'files': [ - { - 'path': 'microdescs-2014-01.tar.xz', + } + }, + 'microdescs': { + 'files': { + 'microdescs-2014-01.tar.xz': { 'size': 7515396, 'last_modified': '2014-02-07 03:59' - }, { - 'path': 'microdescs-2014-02.tar.xz', + }, + 'microdescs-2014-02.tar.xz': { 'size': 21822944, 'last_modified': '2014-03-07 04:54' - }, { - 'path': 'microdescs-2014-03.tar.xz', + }, + 'microdescs-2014-03.tar.xz': { 'size': 24201436, 'last_modified': '2014-04-07 03:54' } - ] - }, { - 'path': 'server-descriptors', - 'files': [ - { - 'path': 'server-descriptors-2005-12.tar.xz', + } + }, + 'server-descriptors': { + 'files': { + 'server-descriptors-2005-12.tar.xz': { 'size': 1348620, 'last_modified': '2016-06-24 08:12' - }, { - 'path': 'server-descriptors-2006-02.tar.xz', + }, + 'server-descriptors-2006-02.tar.xz': { 'size': 28625408, 'last_modified': '2016-06-24 08:14' - }, { - 'path': 'server-descriptors-2006-03.tar.xz', + }, + 'server-descriptors-2006-03.tar.xz': { 'size': 49548736, 'last_modified': '2016-06-24 08:17' } - ] - }, { - 'path': 'statuses', - 'files': [ - { - 'path': 'statuses-2005-12.tar.xz', + } + }, + 'statuses': { + 'files': { + 'statuses-2005-12.tar.xz': { 'size': 1468844, 'last_modified': '2016-06-25 11:50' - }, { - 'path': 'statuses-2006-01.tar.xz', + }, + 'statuses-2006-01.tar.xz': { 'size': 3344280, 'last_modified': '2016-06-25 11:52' - }, { - 'path': 'statuses-2006-02.tar.xz', + }, + 'statuses-2006-02.tar.xz': { 'size': 4006336, 'last_modified': '2016-06-25 11:54' } - ] - }, { - 'path': 'tor', - 'files': [ - { - 'path': 'tor-2004-05.tar.xz', + } + }, + 'tor': { + 'files': { + 'tor-2004-05.tar.xz': { 'size': 386672, 'last_modified': '2012-05-18 14:26' - }, { - 'path': 'tor-2004-06.tar.xz', + }, + 'tor-2004-06.tar.xz': { 'size': 1087980, 'last_modified': '2012-05-18 14:26' - }, { - 'path': 'tor-2004-07.tar.xz', + }, + 'tor-2004-07.tar.xz': { 'size': 1366568, 'last_modified': '2012-05-18 14:26' } - ] - }, { - 'path': 'votes', - 'files': [ - { - 'path': 'votes-2007-10.tar.xz', + } + }, + 'votes': { + 'files': { + 'votes-2007-10.tar.xz': { 'size': 1356504, 'last_modified': '2012-05-15 14:51' - }, { - 'path': 'votes-2007-11.tar.xz', + }, + 'votes-2007-11.tar.xz': { 'size': 10641492, 'last_modified': '2012-05-15 14:51' - }, { - 'path': 'votes-2007-12.tar.xz', + }, + 'votes-2007-12.tar.xz': { 'size': 14712136, 'last_modified': '2012-05-15 14:52' } - ] + } } - ] - }, { - 'path': 'torperf', - 'files': [ - { - 'path': 'torperf-2009-07.tar.xz', + } + }, + 'torperf': { + 'files': { + 'torperf-2009-07.tar.xz': { 'size': 182712, 'last_modified': '2012-05-30 07:23' - }, { - 'path': 'torperf-2009-08.tar.xz', + }, + 'torperf-2009-08.tar.xz': { 'size': 203236, 'last_modified': '2012-05-30 07:23' - }, { - 'path': 'torperf-2009-09.tar.xz', + }, + 'torperf-2009-09.tar.xz': { 'size': 193832, 'last_modified': '2012-05-30 07:23' } - ] - }, { - 'path': 'webstats', - 'files': [ - { - 'path': 'webstats-2015-01.tar', + } + }, + 'webstats': { + 'files': { + 'webstats-2015-01.tar': { 'size': 30720, 'last_modified': '2018-03-19 16:07' - }, { - 'path': 'webstats-2015-02.tar', + }, + 'webstats-2015-02.tar': { 'size': 20480, 'last_modified': '2018-03-19 16:07' - }, { - 'path': 'webstats-2015-03.tar', + }, + 'webstats-2015-03.tar': { 'size': 20480, 'last_modified': '2018-03-19 16:07' } - ] + } } - ] - }, { - 'path': 'contrib' - }, { - 'path': 'recent', - 'directories': [ - { - 'path': 'bridge-descriptors', - 'directories': [ - { - 'path': 'extra-infos', - 'files': [ - { - 'path': '2019-07-03-02-09-00-extra-infos', + } + }, + 'contrib': {}, + 'recent': { + 'directories': { + 'bridge-descriptors': { + 'directories': { + 'extra-infos': { + 'files': { + '2019-07-03-02-09-00-extra-infos': { 'size': 507816, 'last_modified': '2019-07-03 02:09' - }, { - 'path': '2019-07-03-03-09-00-extra-infos', + }, + '2019-07-03-03-09-00-extra-infos': { 'size': 558790, 'last_modified': '2019-07-03 03:09' - }, { - 'path': '2019-07-03-04-09-05-extra-infos', + }, + '2019-07-03-04-09-05-extra-infos': { 'size': 592279, 'last_modified': '2019-07-03 04:09' } - ] - }, { - 'path': 'server-descriptors', - 'files': [ - { - 'path': '2019-07-03-02-09-00-server-descriptors', + } + }, + 'server-descriptors': { + 'files': { + '2019-07-03-02-09-00-server-descriptors': { 'size': 274355, 'last_modified': '2019-07-03 02:09' - }, { - 'path': '2019-07-03-03-09-00-server-descriptors', + }, + '2019-07-03-03-09-00-server-descriptors': { 'size': 295671, 'last_modified': '2019-07-03 03:09' - }, { - 'path': '2019-07-03-04-09-05-server-descriptors', + }, + '2019-07-03-04-09-05-server-descriptors': { 'size': 312822, 'last_modified': '2019-07-03 04:09' } - ] - }, { - 'path': 'statuses', - 'files': [ - { - 'path': '20190703-011231-BA44A889E64B93FAA2B114E02C2A279A8555C533', + } + }, + 'statuses': { + 'files': { + '20190703-011231-BA44A889E64B93FAA2B114E02C2A279A8555C533': { 'size': 238340, 'last_modified': '2019-07-03 02:09' - }, { - 'path': '20190703-014231-BA44A889E64B93FAA2B114E02C2A279A8555C533', + }, + '20190703-014231-BA44A889E64B93FAA2B114E02C2A279A8555C533': { 'size': 238147, 'last_modified': '2019-07-03 02:09' - }, { - 'path': '20190703-021231-BA44A889E64B93FAA2B114E02C2A279A8555C533', + }, + '20190703-021231-BA44A889E64B93FAA2B114E02C2A279A8555C533': { 'size': 238231, 'last_modified': '2019-07-03 03:09' } - ] + } } - ] - }, { - 'path': 'exit-lists', - 'files': [ - { - 'path': '2019-07-03-02-02-00', + } + }, + 'exit-lists': { + 'files': { + '2019-07-03-02-02-00': { 'size': 158516, 'last_modified': '2019-07-03 02:02' - }, { - 'path': '2019-07-03-03-02-00', + }, + '2019-07-03-03-02-00': { 'size': 158830, 'last_modified': '2019-07-03 03:02' - }, { - 'path': '2019-07-03-04-02-00', + }, + '2019-07-03-04-02-00': { 'size': 158831, 'last_modified': '2019-07-03 04:02' } - ] - }, { - 'path': 'relay-descriptors', - 'directories': [ - { - 'path': 'bandwidths', - 'files': [ - { - 'path': '2019-07-03-00-30-57-bandwidth-616DF2278EE2C90F475E4EA2562E2C00EB9F10E517FB901229F331AEB70B8AD7', + } + }, + 'relay-descriptors': { + 'directories': { + 'bandwidths': { + 'files': { + '2019-07-03-00-30-57-bandwidth-616DF2278EE2C90F475E4EA2562E2C00EB9F10E517FB901229F331AEB70B8AD7': { 'size': 2580044, 'last_modified': '2019-07-03 02:35' - }, { - 'path': '2019-07-03-01-35-02-bandwidth-2A5B679E9AA2D5C903CD16C00E16FEFA3E21E38AB3DE338941AE4AC13FD505DE', + }, + '2019-07-03-01-35-02-bandwidth-2A5B679E9AA2D5C903CD16C00E16FEFA3E21E38AB3DE338941AE4AC13FD505DE': { 'size': 4210146, 'last_modified': '2019-07-03 02:35' - }, { - 'path': '2019-07-03-01-35-03-bandwidth-3B06DAD9EDE8E1D08CC494E856AFD9D49256ECC2D68456799DD1EC7ED3436875', + }, + '2019-07-03-01-35-03-bandwidth-3B06DAD9EDE8E1D08CC494E856AFD9D49256ECC2D68456799DD1EC7ED3436875': { 'size': 4249699, 'last_modified': '2019-07-03 02:35' } - ] - }, { - 'path': 'consensuses', - 'files': [ - { - 'path': '2019-07-03-02-00-00-consensus', + } + }, + 'consensuses': { + 'files': { + '2019-07-03-02-00-00-consensus': { 'size': 2211265, 'last_modified': '2019-07-03 02:05' - }, { - 'path': '2019-07-03-03-00-00-consensus', + }, + '2019-07-03-03-00-00-consensus': { 'size': 2204478, 'last_modified': '2019-07-03 03:05' - }, { - 'path': '2019-07-03-04-00-00-consensus', + }, + '2019-07-03-04-00-00-consensus': { 'size': 2202554, 'last_modified': '2019-07-03 04:05' } - ] - }, { - 'path': 'extra-infos', - 'files': [ - { - 'path': '2019-07-03-02-05-00-extra-infos', + } + }, + 'extra-infos': { + 'files': { + '2019-07-03-02-05-00-extra-infos': { 'size': 1162899, 'last_modified': '2019-07-03 02:05' - }, { - 'path': '2019-07-03-03-05-00-extra-infos', + }, + '2019-07-03-03-05-00-extra-infos': { 'size': 1133425, 'last_modified': '2019-07-03 03:05' - }, { - 'path': '2019-07-03-04-05-00-extra-infos', + }, + '2019-07-03-04-05-00-extra-infos': { 'size': 1153793, 'last_modified': '2019-07-03 04:05' } - ] - }, { - 'path': 'microdescs', - 'directories': [ - { - 'path': 'consensus-microdesc', - 'files': [ - { - 'path': '2019-07-03-02-00-00-consensus-microdesc', + } + }, + 'microdescs': { + 'directories': { + 'consensus-microdesc': { + 'files': { + '2019-07-03-02-00-00-consensus-microdesc': { 'size': 2028994, 'last_modified': '2019-07-03 02:05' - }, { - 'path': '2019-07-03-03-00-00-consensus-microdesc', + }, + '2019-07-03-03-00-00-consensus-microdesc': { 'size': 2022808, 'last_modified': '2019-07-03 03:05' - }, { - 'path': '2019-07-03-04-00-00-consensus-microdesc', + }, + '2019-07-03-04-00-00-consensus-microdesc': { 'size': 2020751, 'last_modified': '2019-07-03 04:05' } - ] - }, { - 'path': 'micro', - 'files': [ - { - 'path': '2019-07-02-23-05-00-micro', + } + }, + 'micro': { + 'files': { + '2019-07-02-23-05-00-micro': { 'size': 19934, 'last_modified': '2019-07-02 23:05' - }, { - 'path': '2019-07-03-00-05-00-micro', + }, + '2019-07-03-00-05-00-micro': { 'size': 12030, 'last_modified': '2019-07-03 00:05' - }, { - 'path': '2019-07-03-01-05-00-micro', + }, + '2019-07-03-01-05-00-micro': { 'size': 14850, 'last_modified': '2019-07-03 01:05' } - ] + } } - ] - }, { - 'path': 'server-descriptors', - 'files': [ - { - 'path': '2019-07-03-02-05-00-server-descriptors', + } + }, + 'server-descriptors': { + 'files': { + '2019-07-03-02-05-00-server-descriptors': { 'size': 1374587, 'last_modified': '2019-07-03 02:05' - }, { - 'path': '2019-07-03-03-05-00-server-descriptors', + }, + '2019-07-03-03-05-00-server-descriptors': { 'size': 1353286, 'last_modified': '2019-07-03 03:05' - }, { - 'path': '2019-07-03-04-05-00-server-descriptors', + }, + '2019-07-03-04-05-00-server-descriptors': { 'size': 1336125, 'last_modified': '2019-07-03 04:05' } - ] - }, { - 'path': 'votes', - 'files': [ - { - 'path': '2019-07-03-02-00-00-vote-0232AF901C31A04EE9848595AF9BB7620D4C5B2E-D9B6923A3AF9FB5DB3CD30191BCBD29A7A936D86', + } + }, + 'votes': { + 'files': { + '2019-07-03-02-00-00-vote-0232AF901C31A04EE9848595AF9BB7620D4C5B2E-D9B6923A3AF9FB5DB3CD30191BCBD29A7A936D86': { 'size': 3449707, 'last_modified': '2019-07-03 02:05' - }, { - 'path': '2019-07-03-02-00-00-vote-14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4-39429F0789A9EC07A2A8754C4C449CCABAB787CC', + }, + '2019-07-03-02-00-00-vote-14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4-39429F0789A9EC07A2A8754C4C449CCABAB787CC': { 'size': 3435482, 'last_modified': '2019-07-03 02:05' - }, { - 'path': '2019-07-03-02-00-00-vote-23D15D965BC35114467363C165C4F724B64B4F66-4337842B697467123B5351E1E8EB64ED497C0EDA', + }, + '2019-07-03-02-00-00-vote-23D15D965BC35114467363C165C4F724B64B4F66-4337842B697467123B5351E1E8EB64ED497C0EDA': { 'size': 3571106, 'last_modified': '2019-07-03 02:05' } - ] + } } - ] - }, { - 'path': 'torperf', - 'files': [ - { - 'path': 'op-ab-1048576-2019-07-03.tpf', + } + }, + 'torperf': { + 'files': { + 'op-ab-1048576-2019-07-03.tpf': { 'size': 39152, 'last_modified': '2019-07-04 00:01' - }, { - 'path': 'op-ab-51200-2019-07-03.tpf', + }, + 'op-ab-51200-2019-07-03.tpf': { 'size': 248173, 'last_modified': '2019-07-04 00:01' - }, { - 'path': 'op-ab-5242880-2019-07-03.tpf', + }, + 'op-ab-5242880-2019-07-03.tpf': { 'size': 15900, 'last_modified': '2019-07-04 00:01' } - ] - }, { - 'path': 'webstats', - 'files': [ - { - 'path': '2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20190620.xz', + } + }, + 'webstats': { + 'files': { + '2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20190620.xz': { 'size': 8872, 'last_modified': '2019-07-03 04:21' - }, { - 'path': '2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20190621.xz', + }, + '2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20190621.xz': { 'size': 9312, 'last_modified': '2019-07-04 04:21' - }, { - 'path': '2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20190622.xz', + }, + '2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20190622.xz': { 'size': 9836, 'last_modified': '2019-07-05 04:21' } - ] + } } - ] + } } - ] + } }