August 2019 - tor-commits - lists.torproject.org

[stem/master] Adjust example of writing to disk
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit 1aa6117bcbda629727e7351b2abb88b7d5f1a697 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jul 5 12:37:37 2019 -0700 Adjust example of writing to disk On reflection I won't add special functions for downloading descriptors to disk. That's uncommon, and takes a couple lines for users to do themselves. Adjusting our example to show how. --- stem/descriptor/collector.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/… [View More]

1 0

[stem/master] Separate CollecTor index into its own file
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit 5b89d7c3157d494a6c3d5220bad35a7cec9dce38 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jul 7 09:29:13 2019 -0700 Separate CollecTor index into its own file There's no reason our EXAMPLE_INDEX constant needs to reside in our unit test. It actually makes more sense if it lives alongside the json file it represents. --- test/unit/descriptor/__init__.py | 1 + test/unit/descriptor/collector.py | 492 +-------------------------- … [View More]test/unit/descriptor/data/__init__.py | 7 + test/unit/descriptor/data/collector_index.py | 492 +++++++++++++++++++++++++++ 4 files changed, 501 insertions(+), 491 deletions(-) diff --git a/test/unit/descriptor/__init__.py b/test/unit/descriptor/__init__.py index c5cf01e1..867ed0a4 100644 --- a/test/unit/descriptor/__init__.py +++ b/test/unit/descriptor/__init__.py @@ -7,6 +7,7 @@ import os __all__ = [ 'bandwidth_file', 'collector', + 'data', 'export', 'extrainfo_descriptor', 'microdescriptor', diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index 267a31a9..56ab19c1 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -10,6 +10,7 @@ import stem.prereq from stem.descriptor import Compression from stem.descriptor.collector import CollecTor from test.unit.descriptor import get_resource +from test.unit.descriptor.data.collector_index import EXAMPLE_INDEX try: # added in python 3.3 @@ -27,497 +28,6 @@ MINIMAL_INDEX = { MINIMAL_INDEX_JSON = b'{"index_created":"2017-12-25 21:06","build_revision":"56a303e","path":"https://collector.torproject.org"}' -EXAMPLE_INDEX = { - 'index_created': '2019-07-06 01:54', - 'build_revision': 'df98ac79', - 'path': 'https://collector.torproject.org', - 'directories': { - 'archive': { - 'directories': { - 'bridge-descriptors': { - 'directories': { - 'extra-infos': { - 'files': { - 'bridge-extra-infos-2008-05.tar.xz': { - 'size': 377644, - 'last_modified': '2016-09-04 09:21' - }, - 'bridge-extra-infos-2008-06.tar.xz': { - 'size': 600484, - 'last_modified': '2016-09-04 09:21' - }, - 'bridge-extra-infos-2008-07.tar.xz': { - 'size': 716320, - 'last_modified': '2016-09-04 09:21' - } - } - }, - 'server-descriptors': { - 'files': { - 'bridge-server-descriptors-2008-05.tar.xz': { - 'size': 205348, - 'last_modified': '2016-09-09 14:13' - }, - 'bridge-server-descriptors-2008-06.tar.xz': { - 'size': 342828, - 'last_modified': '2016-09-09 14:13' - }, - 'bridge-server-descriptors-2008-07.tar.xz': { - 'size': 374848, - 'last_modified': '2016-09-09 14:13' - } - } - }, - 'statuses': { - 'files': { - 'bridge-statuses-2008-05.tar.xz': { - 'size': 74792, - 'last_modified': '2016-09-14 21:11' - }, - 'bridge-statuses-2008-06.tar.xz': { - 'size': 123488, - 'last_modified': '2016-09-14 21:11' - }, - 'bridge-statuses-2008-07.tar.xz': { - 'size': 143836, - 'last_modified': '2016-09-14 21:11' - } - } - } - } - }, - 'bridge-pool-assignments': { - 'files': { - 'bridge-pool-assignments-2010-09.tar.xz': { - 'size': 32804, - 'last_modified': '2012-05-31 10:21' - }, - 'bridge-pool-assignments-2010-10.tar.xz': { - 'size': 304684, - 'last_modified': '2012-05-31 10:21' - }, - 'bridge-pool-assignments-2010-11.tar.xz': { - 'size': 292228, - 'last_modified': '2012-05-31 10:21' - } - } - }, - 'exit-lists': { - 'files': { - 'exit-list-2010-02.tar.xz': { - 'size': 272008, - 'last_modified': '2012-05-31 18:57' - }, - 'exit-list-2010-03.tar.xz': { - 'size': 1247484, - 'last_modified': '2012-05-31 18:57' - }, - 'exit-list-2010-04.tar.xz': { - 'size': 1139896, - 'last_modified': '2012-05-31 18:57' - } - } - }, - 'relay-descriptors': { - 'files': { - 'certs.tar.xz': { - 'size': 144696, - 'last_modified': '2019-07-03 03:29' - } - }, - 'directories': { - 'bandwidths': { - 'files': { - 'bandwidths-2019-05.tar.xz': { - 'size': 50385816, - 'last_modified': '2019-06-07 08:05' - }, - 'bandwidths-2019-06.tar.xz': { - 'size': 105881156, - 'last_modified': '2019-07-03 07:30' - }, - 'bandwidths-2019-07.tar.xz': { - 'size': 11374436, - 'last_modified': '2019-07-03 07:12' - } - } - }, - 'consensuses': { - 'files': { - 'consensuses-2007-10.tar.xz': { - 'size': 1061648, - 'last_modified': '2012-05-15 14:35' - }, - 'consensuses-2007-11.tar.xz': { - 'size': 6810308, - 'last_modified': '2012-05-15 14:35' - }, - 'consensuses-2007-12.tar.xz': { - 'size': 8106968, - 'last_modified': '2012-05-15 14:35' - } - } - }, - 'extra-infos': { - 'files': { - 'extra-infos-2007-08.tar.xz': { - 'size': 3016916, - 'last_modified': '2016-06-23 09:53' - }, - 'extra-infos-2007-09.tar.xz': { - 'size': 6459884, - 'last_modified': '2016-06-23 09:54' - }, - 'extra-infos-2007-10.tar.xz': { - 'size': 7326564, - 'last_modified': '2016-06-23 09:54' - } - } - }, - 'microdescs': { - 'files': { - 'microdescs-2014-01.tar.xz': { - 'size': 7515396, - 'last_modified': '2014-02-07 03:59' - }, - 'microdescs-2014-02.tar.xz': { - 'size': 21822944, - 'last_modified': '2014-03-07 04:54' - }, - 'microdescs-2014-03.tar.xz': { - 'size': 24201436, - 'last_modified': '2014-04-07 03:54' - } - } - }, - 'server-descriptors': { - 'files': { - 'server-descriptors-2005-12.tar.xz': { - 'size': 1348620, - 'last_modified': '2016-06-24 08:12' - }, - 'server-descriptors-2006-02.tar.xz': { - 'size': 28625408, - 'last_modified': '2016-06-24 08:14' - }, - 'server-descriptors-2006-03.tar.xz': { - 'size': 49548736, - 'last_modified': '2016-06-24 08:17' - } - } - }, - 'statuses': { - 'files': { - 'statuses-2005-12.tar.xz': { - 'size': 1468844, - 'last_modified': '2016-06-25 11:50' - }, - 'statuses-2006-01.tar.xz': { - 'size': 3344280, - 'last_modified': '2016-06-25 11:52' - }, - 'statuses-2006-02.tar.xz': { - 'size': 4006336, - 'last_modified': '2016-06-25 11:54' - } - } - }, - 'tor': { - 'files': { - 'tor-2004-05.tar.xz': { - 'size': 386672, - 'last_modified': '2012-05-18 14:26' - }, - 'tor-2004-06.tar.xz': { - 'size': 1087980, - 'last_modified': '2012-05-18 14:26' - }, - 'tor-2004-07.tar.xz': { - 'size': 1366568, - 'last_modified': '2012-05-18 14:26' - } - } - }, - 'votes': { - 'files': { - 'votes-2007-10.tar.xz': { - 'size': 1356504, - 'last_modified': '2012-05-15 14:51' - }, - 'votes-2007-11.tar.xz': { - 'size': 10641492, - 'last_modified': '2012-05-15 14:51' - }, - 'votes-2007-12.tar.xz': { - 'size': 14712136, - 'last_modified': '2012-05-15 14:52' - } - } - } - } - }, - 'torperf': { - 'files': { - 'torperf-2009-07.tar.xz': { - 'size': 182712, - 'last_modified': '2012-05-30 07:23' - }, - 'torperf-2009-08.tar.xz': { - 'size': 203236, - 'last_modified': '2012-05-30 07:23' - }, - 'torperf-2009-09.tar.xz': { - 'size': 193832, - 'last_modified': '2012-05-30 07:23' - } - } - }, - 'webstats': { - 'files': { - 'webstats-2015-01.tar': { - 'size': 30720, - 'last_modified': '2018-03-19 16:07' - }, - 'webstats-2015-02.tar': { - 'size': 20480, - 'last_modified': '2018-03-19 16:07' - }, - 'webstats-2015-03.tar': { - 'size': 20480, - 'last_modified': '2018-03-19 16:07' - } - } - } - } - }, - 'contrib': {}, - 'recent': { - 'directories': { - 'bridge-descriptors': { - 'directories': { - 'extra-infos': { - 'files': { - '2019-07-03-02-09-00-extra-infos': { - 'size': 507816, - 'last_modified': '2019-07-03 02:09' - }, - '2019-07-03-03-09-00-extra-infos': { - 'size': 558790, - 'last_modified': '2019-07-03 03:09' - }, - '2019-07-03-04-09-05-extra-infos': { - 'size': 592279, - 'last_modified': '2019-07-03 04:09' - } - } - }, - 'server-descriptors': { - 'files': { - '2019-07-03-02-09-00-server-descriptors': { - 'size': 274355, - 'last_modified': '2019-07-03 02:09' - }, - '2019-07-03-03-09-00-server-descriptors': { - 'size': 295671, - 'last_modified': '2019-07-03 03:09' - }, - '2019-07-03-04-09-05-server-descriptors': { - 'size': 312822, - 'last_modified': '2019-07-03 04:09' - } - } - }, - 'statuses': { - 'files': { - '20190703-011231-BA44A889E64B93FAA2B114E02C2A279A8555C533': { - 'size': 238340, - 'last_modified': '2019-07-03 02:09' - }, - '20190703-014231-BA44A889E64B93FAA2B114E02C2A279A8555C533': { - 'size': 238147, - 'last_modified': '2019-07-03 02:09' - }, - '20190703-021231-BA44A889E64B93FAA2B114E02C2A279A8555C533': { - 'size': 238231, - 'last_modified': '2019-07-03 03:09' - } - } - } - } - }, - 'exit-lists': { - 'files': { - '2019-07-03-02-02-00': { - 'size': 158516, - 'last_modified': '2019-07-03 02:02' - }, - '2019-07-03-03-02-00': { - 'size': 158830, - 'last_modified': '2019-07-03 03:02' - }, - '2019-07-03-04-02-00': { - 'size': 158831, - 'last_modified': '2019-07-03 04:02' - } - } - }, - 'relay-descriptors': { - 'directories': { - 'bandwidths': { - 'files': { - '2019-07-03-00-30-57-bandwidth-616DF2278EE2C90F475E4EA2562E2C00EB9F10E517FB901229F331AEB70B8AD7': { - 'size': 2580044, - 'last_modified': '2019-07-03 02:35' - }, - '2019-07-03-01-35-02-bandwidth-2A5B679E9AA2D5C903CD16C00E16FEFA3E21E38AB3DE338941AE4AC13FD505DE': { - 'size': 4210146, - 'last_modified': '2019-07-03 02:35' - }, - '2019-07-03-01-35-03-bandwidth-3B06DAD9EDE8E1D08CC494E856AFD9D49256ECC2D68456799DD1EC7ED3436875': { - 'size': 4249699, - 'last_modified': '2019-07-03 02:35' - } - } - }, - 'consensuses': { - 'files': { - '2019-07-03-02-00-00-consensus': { - 'size': 2211265, - 'last_modified': '2019-07-03 02:05' - }, - '2019-07-03-03-00-00-consensus': { - 'size': 2204478, - 'last_modified': '2019-07-03 03:05' - }, - '2019-07-03-04-00-00-consensus': { - 'size': 2202554, - 'last_modified': '2019-07-03 04:05' - } - } - }, - 'extra-infos': { - 'files': { - '2019-07-03-02-05-00-extra-infos': { - 'size': 1162899, - 'last_modified': '2019-07-03 02:05' - }, - '2019-07-03-03-05-00-extra-infos': { - 'size': 1133425, - 'last_modified': '2019-07-03 03:05' - }, - '2019-07-03-04-05-00-extra-infos': { - 'size': 1153793, - 'last_modified': '2019-07-03 04:05' - } - } - }, - 'microdescs': { - 'directories': { - 'consensus-microdesc': { - 'files': { - '2019-07-03-02-00-00-consensus-microdesc': { - 'size': 2028994, - 'last_modified': '2019-07-03 02:05' - }, - '2019-07-03-03-00-00-consensus-microdesc': { - 'size': 2022808, - 'last_modified': '2019-07-03 03:05' - }, - '2019-07-03-04-00-00-consensus-microdesc': { - 'size': 2020751, - 'last_modified': '2019-07-03 04:05' - } - } - }, - 'micro': { - 'files': { - '2019-07-02-23-05-00-micro': { - 'size': 19934, - 'last_modified': '2019-07-02 23:05' - }, - '2019-07-03-00-05-00-micro': { - 'size': 12030, - 'last_modified': '2019-07-03 00:05' - }, - '2019-07-03-01-05-00-micro': { - 'size': 14850, - 'last_modified': '2019-07-03 01:05' - } - } - } - } - }, - 'server-descriptors': { - 'files': { - '2019-07-03-02-05-00-server-descriptors': { - 'size': 1374587, - 'last_modified': '2019-07-03 02:05' - }, - '2019-07-03-03-05-00-server-descriptors': { - 'size': 1353286, - 'last_modified': '2019-07-03 03:05' - }, - '2019-07-03-04-05-00-server-descriptors': { - 'size': 1336125, - 'last_modified': '2019-07-03 04:05' - } - } - }, - 'votes': { - 'files': { - '2019-07-03-02-00-00-vote-0232AF901C31A04EE9848595AF9BB7620D4C5B2E-D9B6923A3AF9FB5DB3CD30191BCBD29A7A936D86': { - 'size': 3449707, - 'last_modified': '2019-07-03 02:05' - }, - '2019-07-03-02-00-00-vote-14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4-39429F0789A9EC07A2A8754C4C449CCABAB787CC': { - 'size': 3435482, - 'last_modified': '2019-07-03 02:05' - }, - '2019-07-03-02-00-00-vote-23D15D965BC35114467363C165C4F724B64B4F66-4337842B697467123B5351E1E8EB64ED497C0EDA': { - 'size': 3571106, - 'last_modified': '2019-07-03 02:05' - } - } - } - } - }, - 'torperf': { - 'files': { - 'op-ab-1048576-2019-07-03.tpf': { - 'size': 39152, - 'last_modified': '2019-07-04 00:01' - }, - 'op-ab-51200-2019-07-03.tpf': { - 'size': 248173, - 'last_modified': '2019-07-04 00:01' - }, - 'op-ab-5242880-2019-07-03.tpf': { - 'size': 15900, - 'last_modified': '2019-07-04 00:01' - } - } - }, - 'webstats': { - 'files': { - '2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20190620.xz': { - 'size': 8872, - 'last_modified': '2019-07-03 04:21' - }, - '2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20190621.xz': { - 'size': 9312, - 'last_modified': '2019-07-04 04:21' - }, - '2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20190622.xz': { - 'size': 9836, - 'last_modified': '2019-07-05 04:21' - } - } - } - } - } - } -} - class TestCollector(unittest.TestCase): @patch(URL_OPEN) diff --git a/test/unit/descriptor/data/__init__.py b/test/unit/descriptor/data/__init__.py new file mode 100644 index 00000000..376864fd --- /dev/null +++ b/test/unit/descriptor/data/__init__.py @@ -0,0 +1,7 @@ +""" +Test data for test.unit.descriptor tests. +""" + +__all__ = [ + 'collector_index', +] diff --git a/test/unit/descriptor/data/collector_index.py b/test/unit/descriptor/data/collector_index.py new file mode 100644 index 00000000..b44417af --- /dev/null +++ b/test/unit/descriptor/data/collector_index.py @@ -0,0 +1,492 @@ +# far more readable python hash of collector_index.json + +EXAMPLE_INDEX = { + 'index_created': '2019-07-06 01:54', + 'build_revision': 'df98ac79', + 'path': 'https://collector.torproject.org', + 'directories': { + 'archive': { + 'directories': { + 'bridge-descriptors': { + 'directories': { + 'extra-infos': { + 'files': { + 'bridge-extra-infos-2008-05.tar.xz': { + 'size': 377644, + 'last_modified': '2016-09-04 09:21' + }, + 'bridge-extra-infos-2008-06.tar.xz': { + 'size': 600484, + 'last_modified': '2016-09-04 09:21' + }, + 'bridge-extra-infos-2008-07.tar.xz': { + 'size': 716320, + 'last_modified': '2016-09-04 09:21' + } + } + }, + 'server-descriptors': { + 'files': { + 'bridge-server-descriptors-2008-05.tar.xz': { + 'size': 205348, + 'last_modified': '2016-09-09 14:13' + }, + 'bridge-server-descriptors-2008-06.tar.xz': { + 'size': 342828, + 'last_modified': '2016-09-09 14:13' + }, + 'bridge-server-descriptors-2008-07.tar.xz': { + 'size': 374848, + 'last_modified': '2016-09-09 14:13' + } + } + }, + 'statuses': { + 'files': { + 'bridge-statuses-2008-05.tar.xz': { + 'size': 74792, + 'last_modified': '2016-09-14 21:11' + }, + 'bridge-statuses-2008-06.tar.xz': { + 'size': 123488, + 'last_modified': '2016-09-14 21:11' + }, + 'bridge-statuses-2008-07.tar.xz': { + 'size': 143836, + 'last_modified': '2016-09-14 21:11' + } + } + } + } + }, + 'bridge-pool-assignments': { + 'files': { + 'bridge-pool-assignments-2010-09.tar.xz': { + 'size': 32804, + 'last_modified': '2012-05-31 10:21' + }, + 'bridge-pool-assignments-2010-10.tar.xz': { + 'size': 304684, + 'last_modified': '2012-05-31 10:21' + }, + 'bridge-pool-assignments-2010-11.tar.xz': { + 'size': 292228, + 'last_modified': '2012-05-31 10:21' + } + } + }, + 'exit-lists': { + 'files': { + 'exit-list-2010-02.tar.xz': { + 'size': 272008, + 'last_modified': '2012-05-31 18:57' + }, + 'exit-list-2010-03.tar.xz': { + 'size': 1247484, + 'last_modified': '2012-05-31 18:57' + }, + 'exit-list-2010-04.tar.xz': { + 'size': 1139896, + 'last_modified': '2012-05-31 18:57' + } + } + }, + 'relay-descriptors': { + 'files': { + 'certs.tar.xz': { + 'size': 144696, + 'last_modified': '2019-07-03 03:29' + } + }, + 'directories': { + 'bandwidths': { + 'files': { + 'bandwidths-2019-05.tar.xz': { + 'size': 50385816, + 'last_modified': '2019-06-07 08:05' + }, + 'bandwidths-2019-06.tar.xz': { + 'size': 105881156, + 'last_modified': '2019-07-03 07:30' + }, + 'bandwidths-2019-07.tar.xz': { + 'size': 11374436, + 'last_modified': '2019-07-03 07:12' + } + } + }, + 'consensuses': { + 'files': { + 'consensuses-2007-10.tar.xz': { + 'size': 1061648, + 'last_modified': '2012-05-15 14:35' + }, + 'consensuses-2007-11.tar.xz': { + 'size': 6810308, + 'last_modified': '2012-05-15 14:35' + }, + 'consensuses-2007-12.tar.xz': { + 'size': 8106968, + 'last_modified': '2012-05-15 14:35' + } + } + }, + 'extra-infos': { + 'files': { + 'extra-infos-2007-08.tar.xz': { + 'size': 3016916, + 'last_modified': '2016-06-23 09:53' + }, + 'extra-infos-2007-09.tar.xz': { + 'size': 6459884, + 'last_modified': '2016-06-23 09:54' + }, + 'extra-infos-2007-10.tar.xz': { + 'size': 7326564, + 'last_modified': '2016-06-23 09:54' + } + } + }, + 'microdescs': { + 'files': { + 'microdescs-2014-01.tar.xz': { + 'size': 7515396, + 'last_modified': '2014-02-07 03:59' + }, + 'microdescs-2014-02.tar.xz': { + 'size': 21822944, + 'last_modified': '2014-03-07 04:54' + }, + 'microdescs-2014-03.tar.xz': { + 'size': 24201436, + 'last_modified': '2014-04-07 03:54' + } + } + }, + 'server-descriptors': { + 'files': { + 'server-descriptors-2005-12.tar.xz': { + 'size': 1348620, + 'last_modified': '2016-06-24 08:12' + }, + 'server-descriptors-2006-02.tar.xz': { + 'size': 28625408, + 'last_modified': '2016-06-24 08:14' + }, + 'server-descriptors-2006-03.tar.xz': { + 'size': 49548736, + 'last_modified': '2016-06-24 08:17' + } + } + }, + 'statuses': { + 'files': { + 'statuses-2005-12.tar.xz': { + 'size': 1468844, + 'last_modified': '2016-06-25 11:50' + }, + 'statuses-2006-01.tar.xz': { + 'size': 3344280, + 'last_modified': '2016-06-25 11:52' + }, + 'statuses-2006-02.tar.xz': { + 'size': 4006336, + 'last_modified': '2016-06-25 11:54' + } + } + }, + 'tor': { + 'files': { + 'tor-2004-05.tar.xz': { + 'size': 386672, + 'last_modified': '2012-05-18 14:26' + }, + 'tor-2004-06.tar.xz': { + 'size': 1087980, + 'last_modified': '2012-05-18 14:26' + }, + 'tor-2004-07.tar.xz': { + 'size': 1366568, + 'last_modified': '2012-05-18 14:26' + } + } + }, + 'votes': { + 'files': { + 'votes-2007-10.tar.xz': { + 'size': 1356504, + 'last_modified': '2012-05-15 14:51' + }, + 'votes-2007-11.tar.xz': { + 'size': 10641492, + 'last_modified': '2012-05-15 14:51' + }, + 'votes-2007-12.tar.xz': { + 'size': 14712136, + 'last_modified': '2012-05-15 14:52' + } + } + } + } + }, + 'torperf': { + 'files': { + 'torperf-2009-07.tar.xz': { + 'size': 182712, + 'last_modified': '2012-05-30 07:23' + }, + 'torperf-2009-08.tar.xz': { + 'size': 203236, + 'last_modified': '2012-05-30 07:23' + }, + 'torperf-2009-09.tar.xz': { + 'size': 193832, + 'last_modified': '2012-05-30 07:23' + } + } + }, + 'webstats': { + 'files': { + 'webstats-2015-01.tar': { + 'size': 30720, + 'last_modified': '2018-03-19 16:07' + }, + 'webstats-2015-02.tar': { + 'size': 20480, + 'last_modified': '2018-03-19 16:07' + }, + 'webstats-2015-03.tar': { + 'size': 20480, + 'last_modified': '2018-03-19 16:07' + } + } + } + } + }, + 'contrib': {}, + 'recent': { + 'directories': { + 'bridge-descriptors': { + 'directories': { + 'extra-infos': { + 'files': { + '2019-07-03-02-09-00-extra-infos': { + 'size': 507816, + 'last_modified': '2019-07-03 02:09' + }, + '2019-07-03-03-09-00-extra-infos': { + 'size': 558790, + 'last_modified': '2019-07-03 03:09' + }, + '2019-07-03-04-09-05-extra-infos': { + 'size': 592279, + 'last_modified': '2019-07-03 04:09' + } + } + }, + 'server-descriptors': { + 'files': { + '2019-07-03-02-09-00-server-descriptors': { + 'size': 274355, + 'last_modified': '2019-07-03 02:09' + }, + '2019-07-03-03-09-00-server-descriptors': { + 'size': 295671, + 'last_modified': '2019-07-03 03:09' + }, + '2019-07-03-04-09-05-server-descriptors': { + 'size': 312822, + 'last_modified': '2019-07-03 04:09' + } + } + }, + 'statuses': { + 'files': { + '20190703-011231-BA44A889E64B93FAA2B114E02C2A279A8555C533': { + 'size': 238340, + 'last_modified': '2019-07-03 02:09' + }, + '20190703-014231-BA44A889E64B93FAA2B114E02C2A279A8555C533': { + 'size': 238147, + 'last_modified': '2019-07-03 02:09' + }, + '20190703-021231-BA44A889E64B93FAA2B114E02C2A279A8555C533': { + 'size': 238231, + 'last_modified': '2019-07-03 03:09' + } + } + } + } + }, + 'exit-lists': { + 'files': { + '2019-07-03-02-02-00': { + 'size': 158516, + 'last_modified': '2019-07-03 02:02' + }, + '2019-07-03-03-02-00': { + 'size': 158830, + 'last_modified': '2019-07-03 03:02' + }, + '2019-07-03-04-02-00': { + 'size': 158831, + 'last_modified': '2019-07-03 04:02' + } + } + }, + 'relay-descriptors': { + 'directories': { + 'bandwidths': { + 'files': { + '2019-07-03-00-30-57-bandwidth-616DF2278EE2C90F475E4EA2562E2C00EB9F10E517FB901229F331AEB70B8AD7': { + 'size': 2580044, + 'last_modified': '2019-07-03 02:35' + }, + '2019-07-03-01-35-02-bandwidth-2A5B679E9AA2D5C903CD16C00E16FEFA3E21E38AB3DE338941AE4AC13FD505DE': { + 'size': 4210146, + 'last_modified': '2019-07-03 02:35' + }, + '2019-07-03-01-35-03-bandwidth-3B06DAD9EDE8E1D08CC494E856AFD9D49256ECC2D68456799DD1EC7ED3436875': { + 'size': 4249699, + 'last_modified': '2019-07-03 02:35' + } + } + }, + 'consensuses': { + 'files': { + '2019-07-03-02-00-00-consensus': { + 'size': 2211265, + 'last_modified': '2019-07-03 02:05' + }, + '2019-07-03-03-00-00-consensus': { + 'size': 2204478, + 'last_modified': '2019-07-03 03:05' + }, + '2019-07-03-04-00-00-consensus': { + 'size': 2202554, + 'last_modified': '2019-07-03 04:05' + } + } + }, + 'extra-infos': { + 'files': { + '2019-07-03-02-05-00-extra-infos': { + 'size': 1162899, + 'last_modified': '2019-07-03 02:05' + }, + '2019-07-03-03-05-00-extra-infos': { + 'size': 1133425, + 'last_modified': '2019-07-03 03:05' + }, + '2019-07-03-04-05-00-extra-infos': { + 'size': 1153793, + 'last_modified': '2019-07-03 04:05' + } + } + }, + 'microdescs': { + 'directories': { + 'consensus-microdesc': { + 'files': { + '2019-07-03-02-00-00-consensus-microdesc': { + 'size': 2028994, + 'last_modified': '2019-07-03 02:05' + }, + '2019-07-03-03-00-00-consensus-microdesc': { + 'size': 2022808, + 'last_modified': '2019-07-03 03:05' + }, + '2019-07-03-04-00-00-consensus-microdesc': { + 'size': 2020751, + 'last_modified': '2019-07-03 04:05' + } + } + }, + 'micro': { + 'files': { + '2019-07-02-23-05-00-micro': { + 'size': 19934, + 'last_modified': '2019-07-02 23:05' + }, + '2019-07-03-00-05-00-micro': { + 'size': 12030, + 'last_modified': '2019-07-03 00:05' + }, + '2019-07-03-01-05-00-micro': { + 'size': 14850, + 'last_modified': '2019-07-03 01:05' + } + } + } + } + }, + 'server-descriptors': { + 'files': { + '2019-07-03-02-05-00-server-descriptors': { + 'size': 1374587, + 'last_modified': '2019-07-03 02:05' + }, + '2019-07-03-03-05-00-server-descriptors': { + 'size': 1353286, + 'last_modified': '2019-07-03 03:05' + }, + '2019-07-03-04-05-00-server-descriptors': { + 'size': 1336125, + 'last_modified': '2019-07-03 04:05' + } + } + }, + 'votes': { + 'files': { + '2019-07-03-02-00-00-vote-0232AF901C31A04EE9848595AF9BB7620D4C5B2E-D9B6923A3AF9FB5DB3CD30191BCBD29A7A936D86': { + 'size': 3449707, + 'last_modified': '2019-07-03 02:05' + }, + '2019-07-03-02-00-00-vote-14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4-39429F0789A9EC07A2A8754C4C449CCABAB787CC': { + 'size': 3435482, + 'last_modified': '2019-07-03 02:05' + }, + '2019-07-03-02-00-00-vote-23D15D965BC35114467363C165C4F724B64B4F66-4337842B697467123B5351E1E8EB64ED497C0EDA': { + 'size': 3571106, + 'last_modified': '2019-07-03 02:05' + } + } + } + } + }, + 'torperf': { + 'files': { + 'op-ab-1048576-2019-07-03.tpf': { + 'size': 39152, + 'last_modified': '2019-07-04 00:01' + }, + 'op-ab-51200-2019-07-03.tpf': { + 'size': 248173, + 'last_modified': '2019-07-04 00:01' + }, + 'op-ab-5242880-2019-07-03.tpf': { + 'size': 15900, + 'last_modified': '2019-07-04 00:01' + } + } + }, + 'webstats': { + 'files': { + '2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20190620.xz': { + 'size': 8872, + 'last_modified': '2019-07-03 04:21' + }, + '2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20190621.xz': { + 'size': 9312, + 'last_modified': '2019-07-04 04:21' + }, + '2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20190622.xz': { + 'size': 9836, + 'last_modified': '2019-07-05 04:21' + } + } + } + } + } + } +} [View Less]

1 0

[stem/master] Convert CollecTor indices to key off filenames
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit 8ef2fd199baf954d96dabb362151cc594dd25869 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Jul 6 17:37:46 2019 -0700 Convert CollecTor indices to key off filenames Why are CollecTor 'files' and 'directories' attributes lists? This only makes sense to allow duplicate filenames (which doesn't make sense). Otherwise it simply makes directory transversal a pita. Converting index files/directories into hashes keyed off filenames so we can … [View More]more efficiently (and easily) work with them. --- stem/descriptor/collector.py | 27 +- test/unit/descriptor/collector.py | 594 ++++++++++++++++++-------------------- 2 files changed, 304 insertions(+), 317 deletions(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index d723e97a..2633558e 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -121,6 +121,31 @@ def _download(url, compression, timeout, retries): return stem.util.str_tools._to_unicode(response) +def _convert_index_paths(val): + """ + Key files and directories off their paths so we can work with them more + efficiently. + + :val dict val: index to convert + + :returns: index with files and directories converted to path-keyed hashes + + :raises: **ValueError** if unable to parse the index + """ + + if isinstance(val, dict): + return dict([(k, _convert_index_paths(v)) for (k, v) in val.items()]) + elif isinstance(val, list) and all([isinstance(entry, dict) and 'path' in entry for entry in val]): + contents = {} + + for entry in val: + contents[entry['path']] = dict((k, _convert_index_paths(v)) for (k, v) in entry.items() if k != 'path') + + return contents + else: + return val + + class CollecTor(object): """ Downloader for descriptors from CollecTor. The contents of CollecTor are @@ -167,7 +192,7 @@ class CollecTor(object): if not self._cached_index or time.time() - self._cached_index_at >= REFRESH_INDEX_RATE: response = _download(COLLECTOR_URL + 'index/index.json', self.compression, self.timeout, self.retries) - self._cached_index = json.loads(response) + self._cached_index = _convert_index_paths(json.loads(response)) self._cached_index_at = time.time() return self._cached_index diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index e1ff4bdb..267a31a9 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -31,529 +31,491 @@ EXAMPLE_INDEX = { 'index_created': '2019-07-06 01:54', 'build_revision': 'df98ac79', 'path': 'https://collector.torproject.org', - 'directories': [ - { - 'path': 'archive', - 'directories': [ - { - 'path': 'bridge-descriptors', - 'directories': [ - { - 'path': 'extra-infos', - 'files': [ - { - 'path': 'bridge-extra-infos-2008-05.tar.xz', + 'directories': { + 'archive': { + 'directories': { + 'bridge-descriptors': { + 'directories': { + 'extra-infos': { + 'files': { + 'bridge-extra-infos-2008-05.tar.xz': { 'size': 377644, 'last_modified': '2016-09-04 09:21' - }, { - 'path': 'bridge-extra-infos-2008-06.tar.xz', + }, + 'bridge-extra-infos-2008-06.tar.xz': { 'size': 600484, 'last_modified': '2016-09-04 09:21' - }, { - 'path': 'bridge-extra-infos-2008-07.tar.xz', + }, + 'bridge-extra-infos-2008-07.tar.xz': { 'size': 716320, 'last_modified': '2016-09-04 09:21' } - ] - }, { - 'path': 'server-descriptors', - 'files': [ - { - 'path': 'bridge-server-descriptors-2008-05.tar.xz', + } + }, + 'server-descriptors': { + 'files': { + 'bridge-server-descriptors-2008-05.tar.xz': { 'size': 205348, 'last_modified': '2016-09-09 14:13' - }, { - 'path': 'bridge-server-descriptors-2008-06.tar.xz', + }, + 'bridge-server-descriptors-2008-06.tar.xz': { 'size': 342828, 'last_modified': '2016-09-09 14:13' - }, { - 'path': 'bridge-server-descriptors-2008-07.tar.xz', + }, + 'bridge-server-descriptors-2008-07.tar.xz': { 'size': 374848, 'last_modified': '2016-09-09 14:13' } - ] - }, { - 'path': 'statuses', - 'files': [ - { - 'path': 'bridge-statuses-2008-05.tar.xz', + } + }, + 'statuses': { + 'files': { + 'bridge-statuses-2008-05.tar.xz': { 'size': 74792, 'last_modified': '2016-09-14 21:11' - }, { - 'path': 'bridge-statuses-2008-06.tar.xz', + }, + 'bridge-statuses-2008-06.tar.xz': { 'size': 123488, 'last_modified': '2016-09-14 21:11' - }, { - 'path': 'bridge-statuses-2008-07.tar.xz', + }, + 'bridge-statuses-2008-07.tar.xz': { 'size': 143836, 'last_modified': '2016-09-14 21:11' } - ] + } } - ] - }, { - 'path': 'bridge-pool-assignments', - 'files': [ - { - 'path': 'bridge-pool-assignments-2010-09.tar.xz', + } + }, + 'bridge-pool-assignments': { + 'files': { + 'bridge-pool-assignments-2010-09.tar.xz': { 'size': 32804, 'last_modified': '2012-05-31 10:21' - }, { - 'path': 'bridge-pool-assignments-2010-10.tar.xz', + }, + 'bridge-pool-assignments-2010-10.tar.xz': { 'size': 304684, 'last_modified': '2012-05-31 10:21' - }, { - 'path': 'bridge-pool-assignments-2010-11.tar.xz', + }, + 'bridge-pool-assignments-2010-11.tar.xz': { 'size': 292228, 'last_modified': '2012-05-31 10:21' } - ] - }, { - 'path': 'exit-lists', - 'files': [ - { - 'path': 'exit-list-2010-02.tar.xz', + } + }, + 'exit-lists': { + 'files': { + 'exit-list-2010-02.tar.xz': { 'size': 272008, 'last_modified': '2012-05-31 18:57' - }, { - 'path': 'exit-list-2010-03.tar.xz', + }, + 'exit-list-2010-03.tar.xz': { 'size': 1247484, 'last_modified': '2012-05-31 18:57' - }, { - 'path': 'exit-list-2010-04.tar.xz', + }, + 'exit-list-2010-04.tar.xz': { 'size': 1139896, 'last_modified': '2012-05-31 18:57' } - ] - }, { - 'path': 'relay-descriptors', - 'files': [ - { - 'path': 'certs.tar.xz', + } + }, + 'relay-descriptors': { + 'files': { + 'certs.tar.xz': { 'size': 144696, 'last_modified': '2019-07-03 03:29' } - ], - 'directories': [ - { - 'path': 'bandwidths', - 'files': [ - { - 'path': 'bandwidths-2019-05.tar.xz', + }, + 'directories': { + 'bandwidths': { + 'files': { + 'bandwidths-2019-05.tar.xz': { 'size': 50385816, 'last_modified': '2019-06-07 08:05' - }, { - 'path': 'bandwidths-2019-06.tar.xz', + }, + 'bandwidths-2019-06.tar.xz': { 'size': 105881156, 'last_modified': '2019-07-03 07:30' - }, { - 'path': 'bandwidths-2019-07.tar.xz', + }, + 'bandwidths-2019-07.tar.xz': { 'size': 11374436, 'last_modified': '2019-07-03 07:12' } - ] - }, { - 'path': 'consensuses', - 'files': [ - { - 'path': 'consensuses-2007-10.tar.xz', + } + }, + 'consensuses': { + 'files': { + 'consensuses-2007-10.tar.xz': { 'size': 1061648, 'last_modified': '2012-05-15 14:35' - }, { - 'path': 'consensuses-2007-11.tar.xz', + }, + 'consensuses-2007-11.tar.xz': { 'size': 6810308, 'last_modified': '2012-05-15 14:35' - }, { - 'path': 'consensuses-2007-12.tar.xz', + }, + 'consensuses-2007-12.tar.xz': { 'size': 8106968, 'last_modified': '2012-05-15 14:35' } - ] - }, { - 'path': 'extra-infos', - 'files': [ - { - 'path': 'extra-infos-2007-08.tar.xz', + } + }, + 'extra-infos': { + 'files': { + 'extra-infos-2007-08.tar.xz': { 'size': 3016916, 'last_modified': '2016-06-23 09:53' - }, { - 'path': 'extra-infos-2007-09.tar.xz', + }, + 'extra-infos-2007-09.tar.xz': { 'size': 6459884, 'last_modified': '2016-06-23 09:54' - }, { - 'path': 'extra-infos-2007-10.tar.xz', + }, + 'extra-infos-2007-10.tar.xz': { 'size': 7326564, 'last_modified': '2016-06-23 09:54' } - ] - }, { - 'path': 'microdescs', - 'files': [ - { - 'path': 'microdescs-2014-01.tar.xz', + } + }, + 'microdescs': { + 'files': { + 'microdescs-2014-01.tar.xz': { 'size': 7515396, 'last_modified': '2014-02-07 03:59' - }, { - 'path': 'microdescs-2014-02.tar.xz', + }, + 'microdescs-2014-02.tar.xz': { 'size': 21822944, 'last_modified': '2014-03-07 04:54' - }, { - 'path': 'microdescs-2014-03.tar.xz', + }, + 'microdescs-2014-03.tar.xz': { 'size': 24201436, 'last_modified': '2014-04-07 03:54' } - ] - }, { - 'path': 'server-descriptors', - 'files': [ - { - 'path': 'server-descriptors-2005-12.tar.xz', + } + }, + 'server-descriptors': { + 'files': { + 'server-descriptors-2005-12.tar.xz': { 'size': 1348620, 'last_modified': '2016-06-24 08:12' - }, { - 'path': 'server-descriptors-2006-02.tar.xz', + }, + 'server-descriptors-2006-02.tar.xz': { 'size': 28625408, 'last_modified': '2016-06-24 08:14' - }, { - 'path': 'server-descriptors-2006-03.tar.xz', + }, + 'server-descriptors-2006-03.tar.xz': { 'size': 49548736, 'last_modified': '2016-06-24 08:17' } - ] - }, { - 'path': 'statuses', - 'files': [ - { - 'path': 'statuses-2005-12.tar.xz', + } + }, + 'statuses': { + 'files': { + 'statuses-2005-12.tar.xz': { 'size': 1468844, 'last_modified': '2016-06-25 11:50' - }, { - 'path': 'statuses-2006-01.tar.xz', + }, + 'statuses-2006-01.tar.xz': { 'size': 3344280, 'last_modified': '2016-06-25 11:52' - }, { - 'path': 'statuses-2006-02.tar.xz', + }, + 'statuses-2006-02.tar.xz': { 'size': 4006336, 'last_modified': '2016-06-25 11:54' } - ] - }, { - 'path': 'tor', - 'files': [ - { - 'path': 'tor-2004-05.tar.xz', + } + }, + 'tor': { + 'files': { + 'tor-2004-05.tar.xz': { 'size': 386672, 'last_modified': '2012-05-18 14:26' - }, { - 'path': 'tor-2004-06.tar.xz', + }, + 'tor-2004-06.tar.xz': { 'size': 1087980, 'last_modified': '2012-05-18 14:26' - }, { - 'path': 'tor-2004-07.tar.xz', + }, + 'tor-2004-07.tar.xz': { 'size': 1366568, 'last_modified': '2012-05-18 14:26' } - ] - }, { - 'path': 'votes', - 'files': [ - { - 'path': 'votes-2007-10.tar.xz', + } + }, + 'votes': { + 'files': { + 'votes-2007-10.tar.xz': { 'size': 1356504, 'last_modified': '2012-05-15 14:51' - }, { - 'path': 'votes-2007-11.tar.xz', + }, + 'votes-2007-11.tar.xz': { 'size': 10641492, 'last_modified': '2012-05-15 14:51' - }, { - 'path': 'votes-2007-12.tar.xz', + }, + 'votes-2007-12.tar.xz': { 'size': 14712136, 'last_modified': '2012-05-15 14:52' } - ] + } } - ] - }, { - 'path': 'torperf', - 'files': [ - { - 'path': 'torperf-2009-07.tar.xz', + } + }, + 'torperf': { + 'files': { + 'torperf-2009-07.tar.xz': { 'size': 182712, 'last_modified': '2012-05-30 07:23' - }, { - 'path': 'torperf-2009-08.tar.xz', + }, + 'torperf-2009-08.tar.xz': { 'size': 203236, 'last_modified': '2012-05-30 07:23' - }, { - 'path': 'torperf-2009-09.tar.xz', + }, + 'torperf-2009-09.tar.xz': { 'size': 193832, 'last_modified': '2012-05-30 07:23' } - ] - }, { - 'path': 'webstats', - 'files': [ - { - 'path': 'webstats-2015-01.tar', + } + }, + 'webstats': { + 'files': { + 'webstats-2015-01.tar': { 'size': 30720, 'last_modified': '2018-03-19 16:07' - }, { - 'path': 'webstats-2015-02.tar', + }, + 'webstats-2015-02.tar': { 'size': 20480, 'last_modified': '2018-03-19 16:07' - }, { - 'path': 'webstats-2015-03.tar', + }, + 'webstats-2015-03.tar': { 'size': 20480, 'last_modified': '2018-03-19 16:07' } - ] + } } - ] - }, { - 'path': 'contrib' - }, { - 'path': 'recent', - 'directories': [ - { - 'path': 'bridge-descriptors', - 'directories': [ - { - 'path': 'extra-infos', - 'files': [ - { - 'path': '2019-07-03-02-09-00-extra-infos', + } + }, + 'contrib': {}, + 'recent': { + 'directories': { + 'bridge-descriptors': { + 'directories': { + 'extra-infos': { + 'files': { + '2019-07-03-02-09-00-extra-infos': { 'size': 507816, 'last_modified': '2019-07-03 02:09' - }, { - 'path': '2019-07-03-03-09-00-extra-infos', + }, + '2019-07-03-03-09-00-extra-infos': { 'size': 558790, 'last_modified': '2019-07-03 03:09' - }, { - 'path': '2019-07-03-04-09-05-extra-infos', + }, + '2019-07-03-04-09-05-extra-infos': { 'size': 592279, 'last_modified': '2019-07-03 04:09' } - ] - }, { - 'path': 'server-descriptors', - 'files': [ - { - 'path': '2019-07-03-02-09-00-server-descriptors', + } + }, + 'server-descriptors': { + 'files': { + '2019-07-03-02-09-00-server-descriptors': { 'size': 274355, 'last_modified': '2019-07-03 02:09' - }, { - 'path': '2019-07-03-03-09-00-server-descriptors', + }, + '2019-07-03-03-09-00-server-descriptors': { 'size': 295671, 'last_modified': '2019-07-03 03:09' - }, { - 'path': '2019-07-03-04-09-05-server-descriptors', + }, + '2019-07-03-04-09-05-server-descriptors': { 'size': 312822, 'last_modified': '2019-07-03 04:09' } - ] - }, { - 'path': 'statuses', - 'files': [ - { - 'path': '20190703-011231-BA44A889E64B93FAA2B114E02C2A279A8555C533', + } + }, + 'statuses': { + 'files': { + '20190703-011231-BA44A889E64B93FAA2B114E02C2A279A8555C533': { 'size': 238340, 'last_modified': '2019-07-03 02:09' - }, { - 'path': '20190703-014231-BA44A889E64B93FAA2B114E02C2A279A8555C533', + }, + '20190703-014231-BA44A889E64B93FAA2B114E02C2A279A8555C533': { 'size': 238147, 'last_modified': '2019-07-03 02:09' - }, { - 'path': '20190703-021231-BA44A889E64B93FAA2B114E02C2A279A8555C533', + }, + '20190703-021231-BA44A889E64B93FAA2B114E02C2A279A8555C533': { 'size': 238231, 'last_modified': '2019-07-03 03:09' } - ] + } } - ] - }, { - 'path': 'exit-lists', - 'files': [ - { - 'path': '2019-07-03-02-02-00', + } + }, + 'exit-lists': { + 'files': { + '2019-07-03-02-02-00': { 'size': 158516, 'last_modified': '2019-07-03 02:02' - }, { - 'path': '2019-07-03-03-02-00', + }, + '2019-07-03-03-02-00': { 'size': 158830, 'last_modified': '2019-07-03 03:02' - }, { - 'path': '2019-07-03-04-02-00', + }, + '2019-07-03-04-02-00': { 'size': 158831, 'last_modified': '2019-07-03 04:02' } - ] - }, { - 'path': 'relay-descriptors', - 'directories': [ - { - 'path': 'bandwidths', - 'files': [ - { - 'path': '2019-07-03-00-30-57-bandwidth-616DF2278EE2C90F475E4EA2562E2C00EB9F10E517FB901229F331AEB70B8AD7', + } + }, + 'relay-descriptors': { + 'directories': { + 'bandwidths': { + 'files': { + '2019-07-03-00-30-57-bandwidth-616DF2278EE2C90F475E4EA2562E2C00EB9F10E517FB901229F331AEB70B8AD7': { 'size': 2580044, 'last_modified': '2019-07-03 02:35' - }, { - 'path': '2019-07-03-01-35-02-bandwidth-2A5B679E9AA2D5C903CD16C00E16FEFA3E21E38AB3DE338941AE4AC13FD505DE', + }, + '2019-07-03-01-35-02-bandwidth-2A5B679E9AA2D5C903CD16C00E16FEFA3E21E38AB3DE338941AE4AC13FD505DE': { 'size': 4210146, 'last_modified': '2019-07-03 02:35' - }, { - 'path': '2019-07-03-01-35-03-bandwidth-3B06DAD9EDE8E1D08CC494E856AFD9D49256ECC2D68456799DD1EC7ED3436875', + }, + '2019-07-03-01-35-03-bandwidth-3B06DAD9EDE8E1D08CC494E856AFD9D49256ECC2D68456799DD1EC7ED3436875': { 'size': 4249699, 'last_modified': '2019-07-03 02:35' } - ] - }, { - 'path': 'consensuses', - 'files': [ - { - 'path': '2019-07-03-02-00-00-consensus', + } + }, + 'consensuses': { + 'files': { + '2019-07-03-02-00-00-consensus': { 'size': 2211265, 'last_modified': '2019-07-03 02:05' - }, { - 'path': '2019-07-03-03-00-00-consensus', + }, + '2019-07-03-03-00-00-consensus': { 'size': 2204478, 'last_modified': '2019-07-03 03:05' - }, { - 'path': '2019-07-03-04-00-00-consensus', + }, + '2019-07-03-04-00-00-consensus': { 'size': 2202554, 'last_modified': '2019-07-03 04:05' } - ] - }, { - 'path': 'extra-infos', - 'files': [ - { - 'path': '2019-07-03-02-05-00-extra-infos', + } + }, + 'extra-infos': { + 'files': { + '2019-07-03-02-05-00-extra-infos': { 'size': 1162899, 'last_modified': '2019-07-03 02:05' - }, { - 'path': '2019-07-03-03-05-00-extra-infos', + }, + '2019-07-03-03-05-00-extra-infos': { 'size': 1133425, 'last_modified': '2019-07-03 03:05' - }, { - 'path': '2019-07-03-04-05-00-extra-infos', + }, + '2019-07-03-04-05-00-extra-infos': { 'size': 1153793, 'last_modified': '2019-07-03 04:05' } - ] - }, { - 'path': 'microdescs', - 'directories': [ - { - 'path': 'consensus-microdesc', - 'files': [ - { - 'path': '2019-07-03-02-00-00-consensus-microdesc', + } + }, + 'microdescs': { + 'directories': { + 'consensus-microdesc': { + 'files': { + '2019-07-03-02-00-00-consensus-microdesc': { 'size': 2028994, 'last_modified': '2019-07-03 02:05' - }, { - 'path': '2019-07-03-03-00-00-consensus-microdesc', + }, + '2019-07-03-03-00-00-consensus-microdesc': { 'size': 2022808, 'last_modified': '2019-07-03 03:05' - }, { - 'path': '2019-07-03-04-00-00-consensus-microdesc', + }, + '2019-07-03-04-00-00-consensus-microdesc': { 'size': 2020751, 'last_modified': '2019-07-03 04:05' } - ] - }, { - 'path': 'micro', - 'files': [ - { - 'path': '2019-07-02-23-05-00-micro', + } + }, + 'micro': { + 'files': { + '2019-07-02-23-05-00-micro': { 'size': 19934, 'last_modified': '2019-07-02 23:05' - }, { - 'path': '2019-07-03-00-05-00-micro', + }, + '2019-07-03-00-05-00-micro': { 'size': 12030, 'last_modified': '2019-07-03 00:05' - }, { - 'path': '2019-07-03-01-05-00-micro', + }, + '2019-07-03-01-05-00-micro': { 'size': 14850, 'last_modified': '2019-07-03 01:05' } - ] + } } - ] - }, { - 'path': 'server-descriptors', - 'files': [ - { - 'path': '2019-07-03-02-05-00-server-descriptors', + } + }, + 'server-descriptors': { + 'files': { + '2019-07-03-02-05-00-server-descriptors': { 'size': 1374587, 'last_modified': '2019-07-03 02:05' - }, { - 'path': '2019-07-03-03-05-00-server-descriptors', + }, + '2019-07-03-03-05-00-server-descriptors': { 'size': 1353286, 'last_modified': '2019-07-03 03:05' - }, { - 'path': '2019-07-03-04-05-00-server-descriptors', + }, + '2019-07-03-04-05-00-server-descriptors': { 'size': 1336125, 'last_modified': '2019-07-03 04:05' } - ] - }, { - 'path': 'votes', - 'files': [ - { - 'path': '2019-07-03-02-00-00-vote-0232AF901C31A04EE9848595AF9BB7620D4C5B2E-D9B6923A3AF9FB5DB3CD30191BCBD29A7A936D86', + } + }, + 'votes': { + 'files': { + '2019-07-03-02-00-00-vote-0232AF901C31A04EE9848595AF9BB7620D4C5B2E-D9B6923A3AF9FB5DB3CD30191BCBD29A7A936D86': { 'size': 3449707, 'last_modified': '2019-07-03 02:05' - }, { - 'path': '2019-07-03-02-00-00-vote-14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4-39429F0789A9EC07A2A8754C4C449CCABAB787CC', + }, + '2019-07-03-02-00-00-vote-14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4-39429F0789A9EC07A2A8754C4C449CCABAB787CC': { 'size': 3435482, 'last_modified': '2019-07-03 02:05' - }, { - 'path': '2019-07-03-02-00-00-vote-23D15D965BC35114467363C165C4F724B64B4F66-4337842B697467123B5351E1E8EB64ED497C0EDA', + }, + '2019-07-03-02-00-00-vote-23D15D965BC35114467363C165C4F724B64B4F66-4337842B697467123B5351E1E8EB64ED497C0EDA': { 'size': 3571106, 'last_modified': '2019-07-03 02:05' } - ] + } } - ] - }, { - 'path': 'torperf', - 'files': [ - { - 'path': 'op-ab-1048576-2019-07-03.tpf', + } + }, + 'torperf': { + 'files': { + 'op-ab-1048576-2019-07-03.tpf': { 'size': 39152, 'last_modified': '2019-07-04 00:01' - }, { - 'path': 'op-ab-51200-2019-07-03.tpf', + }, + 'op-ab-51200-2019-07-03.tpf': { 'size': 248173, 'last_modified': '2019-07-04 00:01' - }, { - 'path': 'op-ab-5242880-2019-07-03.tpf', + }, + 'op-ab-5242880-2019-07-03.tpf': { 'size': 15900, 'last_modified': '2019-07-04 00:01' } - ] - }, { - 'path': 'webstats', - 'files': [ - { - 'path': '2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20190620.xz', + } + }, + 'webstats': { + 'files': { + '2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20190620.xz': { 'size': 8872, 'last_modified': '2019-07-03 04:21' - }, { - 'path': '2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20190621.xz', + }, + '2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20190621.xz': { 'size': 9312, 'last_modified': '2019-07-04 04:21' - }, { - 'path': '2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20190622.xz', + }, + '2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20190622.xz': { 'size': 9836, 'last_modified': '2019-07-05 04:21' } - ] + } } - ] + } } - ] + } } [View Less]

1 0

[stem/master] Drop Index class
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit 1d99faac82d172312c5627618e9d09e72e639f96 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Jul 9 12:53:38 2019 -0700 Drop Index class Nope, on reflection an Index class won't simplify working with it. Everything we need is on a per-file level, so generating a files hash is all we need. --- stem/descriptor/collector.py | 117 +++--- test/unit/descriptor/collector.py | 18 +- test/unit/descriptor/data/collector_index.py | 594 +… [View More]+++++++++++++------------- 3 files changed, 372 insertions(+), 357 deletions(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index 135a5d00..1742c596 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -122,75 +122,6 @@ def _download(url, compression, timeout, retries): return stem.util.str_tools._to_unicode(response) -class Index(object): - """ - Index of CollecTor's content. - - :var hash files: mapping of paths to thier content - """ - - def __init__(self, content): - self._str_content = content - self._hash_content = Index._convert_paths(json.loads(content)) - - self.files = Index._get_files(self._hash_content, []) - - def __str__(self): - return self._str_content - - def __iter__(self): - for k, v in self._hash_content.items(): - yield k, v - - @staticmethod - def _get_files(val, path): - """ - Provies a mapping of paths to files within the index. - - :param dict val: index hash - :param list path: path we've transversed into - - :returns: **dict** mapping paths to files - """ - - files = {} - - if isinstance(val, dict): - for k, v in val.items(): - if k == 'files': - for filename, attr in v.items(): - file_path = '/'.join(path + [filename]) - files[file_path] = File(file_path, attr.get('size'), attr.get('last_modified')) - elif k == 'directories': - for filename, attr in v.items(): - files.update(Index._get_files(attr, path + [filename])) - - return files - - @staticmethod - def _convert_paths(val): - """ - Key files and directories off their paths so we can transverse them more - efficiently. - - :val dict val: index to convert - - :returns: index with files and directories converted to path-keyed hashes - """ - - if isinstance(val, dict): - return dict([(k, Index._convert_paths(v)) for (k, v) in val.items()]) - elif isinstance(val, list) and all([isinstance(entry, dict) and 'path' in entry for entry in val]): - contents = {} - - for entry in val: - contents[entry['path']] = dict((k, Index._convert_paths(v)) for (k, v) in entry.items() if k != 'path') - - return contents - else: - return val - - class File(object): """ File within CollecTor. @@ -225,6 +156,7 @@ class CollecTor(object): self.timeout = timeout self._cached_index = None + self._cached_files = None # {path => file} mappings in the index self._cached_index_at = 0 if compression == 'best': @@ -253,7 +185,52 @@ class CollecTor(object): if not self._cached_index or time.time() - self._cached_index_at >= REFRESH_INDEX_RATE: response = _download(COLLECTOR_URL + 'index/index.json', self.compression, self.timeout, self.retries) - self._cached_index = Index(response) + self._cached_index = json.loads(response) self._cached_index_at = time.time() return self._cached_index + + def files(self): + """ + Provides files CollecTor presently has. + + :returns: **hash** mapping paths to :class:`~stem.descriptor.collector.File` + + :raises: + If unable to retrieve the index this provide... + + * **ValueError** if json is malformed + * **IOError** if unable to decompress + * **socket.timeout** if our request timed out + * **urllib2.URLError** for most request failures + """ + + if not self._cached_files or time.time() - self._cached_index_at >= REFRESH_INDEX_RATE: + self._cached_files = CollecTor._files(self.index(), []) + + return self._cached_files + + @staticmethod + def _files(val, path): + """ + Provies a mapping of paths to files within the index. + + :param dict val: index hash + :param list path: path we've transversed into + + :returns: **dict** mapping paths to files + """ + + files = {} + + if isinstance(val, dict): + for k, v in val.items(): + if k == 'files': + for attr in v: + file_path = '/'.join(path + [attr.get('path')]) + files[file_path] = File(file_path, attr.get('size'), attr.get('last_modified')) + elif k == 'directories': + for attr in v: + files.update(CollecTor._files(attr, path + [attr.get('path')])) + + return files diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index cc12ee00..3403ee50 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -39,7 +39,7 @@ class TestCollector(unittest.TestCase): urlopen_mock.return_value = io.BytesIO(MINIMAL_INDEX_JSON) collector = CollecTor(compression = Compression.PLAINTEXT) - self.assertEqual(MINIMAL_INDEX, dict(collector.index())) + self.assertEqual(MINIMAL_INDEX, collector.index()) urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json', timeout = None) @patch(URL_OPEN) @@ -52,7 +52,7 @@ class TestCollector(unittest.TestCase): urlopen_mock.return_value = io.BytesIO(zlib.compress(MINIMAL_INDEX_JSON)) collector = CollecTor(compression = Compression.GZIP) - self.assertEqual(MINIMAL_INDEX, dict(collector.index())) + self.assertEqual(MINIMAL_INDEX, collector.index()) urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.gz', timeout = None) @patch(URL_OPEN) @@ -65,7 +65,7 @@ class TestCollector(unittest.TestCase): urlopen_mock.return_value = io.BytesIO(bz2.compress(MINIMAL_INDEX_JSON)) collector = CollecTor(compression = Compression.BZ2) - self.assertEqual(MINIMAL_INDEX, dict(collector.index())) + self.assertEqual(MINIMAL_INDEX, collector.index()) urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.bz2', timeout = None) @patch(URL_OPEN) @@ -78,7 +78,7 @@ class TestCollector(unittest.TestCase): urlopen_mock.return_value = io.BytesIO(lzma.compress(MINIMAL_INDEX_JSON)) collector = CollecTor(compression = Compression.LZMA) - self.assertEqual(MINIMAL_INDEX, dict(collector.index())) + self.assertEqual(MINIMAL_INDEX, collector.index()) urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.lzma', timeout = None) @patch(URL_OPEN) @@ -98,7 +98,7 @@ class TestCollector(unittest.TestCase): @patch(URL_OPEN, Mock(return_value = io.BytesIO(MINIMAL_INDEX_JSON))) def test_index(self): collector = CollecTor(compression = Compression.PLAINTEXT) - self.assertEqual(MINIMAL_INDEX, dict(collector.index())) + self.assertEqual(MINIMAL_INDEX, collector.index()) @patch(URL_OPEN, Mock(return_value = io.BytesIO(b'not json'))) def test_index_malformed_json(self): @@ -121,17 +121,17 @@ class TestCollector(unittest.TestCase): @patch(URL_OPEN, Mock(return_value = io.BytesIO(EXAMPLE_INDEX_CONTENT))) def test_real_index(self): collector = CollecTor(compression = Compression.PLAINTEXT) - self.assertEqual(EXAMPLE_INDEX, dict(collector.index())) + self.assertEqual(EXAMPLE_INDEX, collector.index()) @patch(URL_OPEN, Mock(return_value = io.BytesIO(EXAMPLE_INDEX_CONTENT))) def test_contents(self): collector = CollecTor(compression = Compression.PLAINTEXT) - index = collector.index() + files = collector.files() - self.assertEqual(85, len(index.files)) + self.assertEqual(85, len(files)) test_path = 'archive/relay-descriptors/extra-infos/extra-infos-2007-09.tar.xz' - extrainfo_file = index.files[test_path] + extrainfo_file = files[test_path] self.assertEqual(test_path, extrainfo_file.path) self.assertEqual(6459884, extrainfo_file.size) self.assertEqual(datetime.datetime(2016, 6, 23, 9, 54), extrainfo_file.last_modified) diff --git a/test/unit/descriptor/data/collector_index.py b/test/unit/descriptor/data/collector_index.py index b44417af..41ef6cbd 100644 --- a/test/unit/descriptor/data/collector_index.py +++ b/test/unit/descriptor/data/collector_index.py @@ -4,489 +4,527 @@ EXAMPLE_INDEX = { 'index_created': '2019-07-06 01:54', 'build_revision': 'df98ac79', 'path': 'https://collector.torproject.org', - 'directories': { - 'archive': { - 'directories': { - 'bridge-descriptors': { - 'directories': { - 'extra-infos': { - 'files': { - 'bridge-extra-infos-2008-05.tar.xz': { + 'directories': [ + { + 'path': 'archive', + 'directories': [ + { + 'path': 'bridge-descriptors', + 'directories': [ + { + 'path': 'extra-infos', + 'files': [ + { + 'path': 'bridge-extra-infos-2008-05.tar.xz', 'size': 377644, 'last_modified': '2016-09-04 09:21' - }, - 'bridge-extra-infos-2008-06.tar.xz': { + }, { + 'path': 'bridge-extra-infos-2008-06.tar.xz', 'size': 600484, 'last_modified': '2016-09-04 09:21' - }, - 'bridge-extra-infos-2008-07.tar.xz': { + }, { + 'path': 'bridge-extra-infos-2008-07.tar.xz', 'size': 716320, 'last_modified': '2016-09-04 09:21' } - } - }, - 'server-descriptors': { - 'files': { - 'bridge-server-descriptors-2008-05.tar.xz': { + ] + }, { + 'path': 'server-descriptors', + 'files': [ + { + 'path': 'bridge-server-descriptors-2008-05.tar.xz', 'size': 205348, 'last_modified': '2016-09-09 14:13' - }, - 'bridge-server-descriptors-2008-06.tar.xz': { + }, { + 'path': 'bridge-server-descriptors-2008-06.tar.xz', 'size': 342828, 'last_modified': '2016-09-09 14:13' - }, - 'bridge-server-descriptors-2008-07.tar.xz': { + }, { + 'path': 'bridge-server-descriptors-2008-07.tar.xz', 'size': 374848, 'last_modified': '2016-09-09 14:13' } - } - }, - 'statuses': { - 'files': { - 'bridge-statuses-2008-05.tar.xz': { + ] + }, { + 'path': 'statuses', + 'files': [ + { + 'path': 'bridge-statuses-2008-05.tar.xz', 'size': 74792, 'last_modified': '2016-09-14 21:11' - }, - 'bridge-statuses-2008-06.tar.xz': { + }, { + 'path': 'bridge-statuses-2008-06.tar.xz', 'size': 123488, 'last_modified': '2016-09-14 21:11' - }, - 'bridge-statuses-2008-07.tar.xz': { + }, { + 'path': 'bridge-statuses-2008-07.tar.xz', 'size': 143836, 'last_modified': '2016-09-14 21:11' } - } + ] } - } - }, - 'bridge-pool-assignments': { - 'files': { - 'bridge-pool-assignments-2010-09.tar.xz': { + ] + }, { + 'path': 'bridge-pool-assignments', + 'files': [ + { + 'path': 'bridge-pool-assignments-2010-09.tar.xz', 'size': 32804, 'last_modified': '2012-05-31 10:21' - }, - 'bridge-pool-assignments-2010-10.tar.xz': { + }, { + 'path': 'bridge-pool-assignments-2010-10.tar.xz', 'size': 304684, 'last_modified': '2012-05-31 10:21' - }, - 'bridge-pool-assignments-2010-11.tar.xz': { + }, { + 'path': 'bridge-pool-assignments-2010-11.tar.xz', 'size': 292228, 'last_modified': '2012-05-31 10:21' } - } - }, - 'exit-lists': { - 'files': { - 'exit-list-2010-02.tar.xz': { + ] + }, { + 'path': 'exit-lists', + 'files': [ + { + 'path': 'exit-list-2010-02.tar.xz', 'size': 272008, 'last_modified': '2012-05-31 18:57' - }, - 'exit-list-2010-03.tar.xz': { + }, { + 'path': 'exit-list-2010-03.tar.xz', 'size': 1247484, 'last_modified': '2012-05-31 18:57' - }, - 'exit-list-2010-04.tar.xz': { + }, { + 'path': 'exit-list-2010-04.tar.xz', 'size': 1139896, 'last_modified': '2012-05-31 18:57' } - } - }, - 'relay-descriptors': { - 'files': { - 'certs.tar.xz': { + ] + }, { + 'path': 'relay-descriptors', + 'files': [ + { + 'path': 'certs.tar.xz', 'size': 144696, 'last_modified': '2019-07-03 03:29' } - }, - 'directories': { - 'bandwidths': { - 'files': { - 'bandwidths-2019-05.tar.xz': { + ], + 'directories': [ + { + 'path': 'bandwidths', + 'files': [ + { + 'path': 'bandwidths-2019-05.tar.xz', 'size': 50385816, 'last_modified': '2019-06-07 08:05' - }, - 'bandwidths-2019-06.tar.xz': { + }, { + 'path': 'bandwidths-2019-06.tar.xz', 'size': 105881156, 'last_modified': '2019-07-03 07:30' - }, - 'bandwidths-2019-07.tar.xz': { + }, { + 'path': 'bandwidths-2019-07.tar.xz', 'size': 11374436, 'last_modified': '2019-07-03 07:12' } - } - }, - 'consensuses': { - 'files': { - 'consensuses-2007-10.tar.xz': { + ] + }, { + 'path': 'consensuses', + 'files': [ + { + 'path': 'consensuses-2007-10.tar.xz', 'size': 1061648, 'last_modified': '2012-05-15 14:35' - }, - 'consensuses-2007-11.tar.xz': { + }, { + 'path': 'consensuses-2007-11.tar.xz', 'size': 6810308, 'last_modified': '2012-05-15 14:35' - }, - 'consensuses-2007-12.tar.xz': { + }, { + 'path': 'consensuses-2007-12.tar.xz', 'size': 8106968, 'last_modified': '2012-05-15 14:35' } - } - }, - 'extra-infos': { - 'files': { - 'extra-infos-2007-08.tar.xz': { + ] + }, { + 'path': 'extra-infos', + 'files': [ + { + 'path': 'extra-infos-2007-08.tar.xz', 'size': 3016916, 'last_modified': '2016-06-23 09:53' - }, - 'extra-infos-2007-09.tar.xz': { + }, { + 'path': 'extra-infos-2007-09.tar.xz', 'size': 6459884, 'last_modified': '2016-06-23 09:54' - }, - 'extra-infos-2007-10.tar.xz': { + }, { + 'path': 'extra-infos-2007-10.tar.xz', 'size': 7326564, 'last_modified': '2016-06-23 09:54' } - } - }, - 'microdescs': { - 'files': { - 'microdescs-2014-01.tar.xz': { + ] + }, { + 'path': 'microdescs', + 'files': [ + { + 'path': 'microdescs-2014-01.tar.xz', 'size': 7515396, 'last_modified': '2014-02-07 03:59' - }, - 'microdescs-2014-02.tar.xz': { + }, { + 'path': 'microdescs-2014-02.tar.xz', 'size': 21822944, 'last_modified': '2014-03-07 04:54' - }, - 'microdescs-2014-03.tar.xz': { + }, { + 'path': 'microdescs-2014-03.tar.xz', 'size': 24201436, 'last_modified': '2014-04-07 03:54' } - } - }, - 'server-descriptors': { - 'files': { - 'server-descriptors-2005-12.tar.xz': { + ] + }, { + 'path': 'server-descriptors', + 'files': [ + { + 'path': 'server-descriptors-2005-12.tar.xz', 'size': 1348620, 'last_modified': '2016-06-24 08:12' - }, - 'server-descriptors-2006-02.tar.xz': { + }, { + 'path': 'server-descriptors-2006-02.tar.xz', 'size': 28625408, 'last_modified': '2016-06-24 08:14' - }, - 'server-descriptors-2006-03.tar.xz': { + }, { + 'path': 'server-descriptors-2006-03.tar.xz', 'size': 49548736, 'last_modified': '2016-06-24 08:17' } - } - }, - 'statuses': { - 'files': { - 'statuses-2005-12.tar.xz': { + ] + }, { + 'path': 'statuses', + 'files': [ + { + 'path': 'statuses-2005-12.tar.xz', 'size': 1468844, 'last_modified': '2016-06-25 11:50' - }, - 'statuses-2006-01.tar.xz': { + }, { + 'path': 'statuses-2006-01.tar.xz', 'size': 3344280, 'last_modified': '2016-06-25 11:52' - }, - 'statuses-2006-02.tar.xz': { + }, { + 'path': 'statuses-2006-02.tar.xz', 'size': 4006336, 'last_modified': '2016-06-25 11:54' } - } - }, - 'tor': { - 'files': { - 'tor-2004-05.tar.xz': { + ] + }, { + 'path': 'tor', + 'files': [ + { + 'path': 'tor-2004-05.tar.xz', 'size': 386672, 'last_modified': '2012-05-18 14:26' - }, - 'tor-2004-06.tar.xz': { + }, { + 'path': 'tor-2004-06.tar.xz', 'size': 1087980, 'last_modified': '2012-05-18 14:26' - }, - 'tor-2004-07.tar.xz': { + }, { + 'path': 'tor-2004-07.tar.xz', 'size': 1366568, 'last_modified': '2012-05-18 14:26' } - } - }, - 'votes': { - 'files': { - 'votes-2007-10.tar.xz': { + ] + }, { + 'path': 'votes', + 'files': [ + { + 'path': 'votes-2007-10.tar.xz', 'size': 1356504, 'last_modified': '2012-05-15 14:51' - }, - 'votes-2007-11.tar.xz': { + }, { + 'path': 'votes-2007-11.tar.xz', 'size': 10641492, 'last_modified': '2012-05-15 14:51' - }, - 'votes-2007-12.tar.xz': { + }, { + 'path': 'votes-2007-12.tar.xz', 'size': 14712136, 'last_modified': '2012-05-15 14:52' } - } + ] } - } - }, - 'torperf': { - 'files': { - 'torperf-2009-07.tar.xz': { + ] + }, { + 'path': 'torperf', + 'files': [ + { + 'path': 'torperf-2009-07.tar.xz', 'size': 182712, 'last_modified': '2012-05-30 07:23' - }, - 'torperf-2009-08.tar.xz': { + }, { + 'path': 'torperf-2009-08.tar.xz', 'size': 203236, 'last_modified': '2012-05-30 07:23' - }, - 'torperf-2009-09.tar.xz': { + }, { + 'path': 'torperf-2009-09.tar.xz', 'size': 193832, 'last_modified': '2012-05-30 07:23' } - } - }, - 'webstats': { - 'files': { - 'webstats-2015-01.tar': { + ] + }, { + 'path': 'webstats', + 'files': [ + { + 'path': 'webstats-2015-01.tar', 'size': 30720, 'last_modified': '2018-03-19 16:07' - }, - 'webstats-2015-02.tar': { + }, { + 'path': 'webstats-2015-02.tar', 'size': 20480, 'last_modified': '2018-03-19 16:07' - }, - 'webstats-2015-03.tar': { + }, { + 'path': 'webstats-2015-03.tar', 'size': 20480, 'last_modified': '2018-03-19 16:07' } - } + ] } - } - }, - 'contrib': {}, - 'recent': { - 'directories': { - 'bridge-descriptors': { - 'directories': { - 'extra-infos': { - 'files': { - '2019-07-03-02-09-00-extra-infos': { + ] + }, { + 'path': 'contrib' + }, { + 'path': 'recent', + 'directories': [ + { + 'path': 'bridge-descriptors', + 'directories': [ + { + 'path': 'extra-infos', + 'files': [ + { + 'path': '2019-07-03-02-09-00-extra-infos', 'size': 507816, 'last_modified': '2019-07-03 02:09' - }, - '2019-07-03-03-09-00-extra-infos': { + }, { + 'path': '2019-07-03-03-09-00-extra-infos', 'size': 558790, 'last_modified': '2019-07-03 03:09' - }, - '2019-07-03-04-09-05-extra-infos': { + }, { + 'path': '2019-07-03-04-09-05-extra-infos', 'size': 592279, 'last_modified': '2019-07-03 04:09' } - } - }, - 'server-descriptors': { - 'files': { - '2019-07-03-02-09-00-server-descriptors': { + ] + }, { + 'path': 'server-descriptors', + 'files': [ + { + 'path': '2019-07-03-02-09-00-server-descriptors', 'size': 274355, 'last_modified': '2019-07-03 02:09' - }, - '2019-07-03-03-09-00-server-descriptors': { + }, { + 'path': '2019-07-03-03-09-00-server-descriptors', 'size': 295671, 'last_modified': '2019-07-03 03:09' - }, - '2019-07-03-04-09-05-server-descriptors': { + }, { + 'path': '2019-07-03-04-09-05-server-descriptors', 'size': 312822, 'last_modified': '2019-07-03 04:09' } - } - }, - 'statuses': { - 'files': { - '20190703-011231-BA44A889E64B93FAA2B114E02C2A279A8555C533': { + ] + }, { + 'path': 'statuses', + 'files': [ + { + 'path': '20190703-011231-BA44A889E64B93FAA2B114E02C2A279A8555C533', 'size': 238340, 'last_modified': '2019-07-03 02:09' - }, - '20190703-014231-BA44A889E64B93FAA2B114E02C2A279A8555C533': { + }, { + 'path': '20190703-014231-BA44A889E64B93FAA2B114E02C2A279A8555C533', 'size': 238147, 'last_modified': '2019-07-03 02:09' - }, - '20190703-021231-BA44A889E64B93FAA2B114E02C2A279A8555C533': { + }, { + 'path': '20190703-021231-BA44A889E64B93FAA2B114E02C2A279A8555C533', 'size': 238231, 'last_modified': '2019-07-03 03:09' } - } + ] } - } - }, - 'exit-lists': { - 'files': { - '2019-07-03-02-02-00': { + ] + }, { + 'path': 'exit-lists', + 'files': [ + { + 'path': '2019-07-03-02-02-00', 'size': 158516, 'last_modified': '2019-07-03 02:02' - }, - '2019-07-03-03-02-00': { + }, { + 'path': '2019-07-03-03-02-00', 'size': 158830, 'last_modified': '2019-07-03 03:02' - }, - '2019-07-03-04-02-00': { + }, { + 'path': '2019-07-03-04-02-00', 'size': 158831, 'last_modified': '2019-07-03 04:02' } - } - }, - 'relay-descriptors': { - 'directories': { - 'bandwidths': { - 'files': { - '2019-07-03-00-30-57-bandwidth-616DF2278EE2C90F475E4EA2562E2C00EB9F10E517FB901229F331AEB70B8AD7': { + ] + }, { + 'path': 'relay-descriptors', + 'directories': [ + { + 'path': 'bandwidths', + 'files': [ + { + 'path': '2019-07-03-00-30-57-bandwidth-616DF2278EE2C90F475E4EA2562E2C00EB9F10E517FB901229F331AEB70B8AD7', 'size': 2580044, 'last_modified': '2019-07-03 02:35' - }, - '2019-07-03-01-35-02-bandwidth-2A5B679E9AA2D5C903CD16C00E16FEFA3E21E38AB3DE338941AE4AC13FD505DE': { + }, { + 'path': '2019-07-03-01-35-02-bandwidth-2A5B679E9AA2D5C903CD16C00E16FEFA3E21E38AB3DE338941AE4AC13FD505DE', 'size': 4210146, 'last_modified': '2019-07-03 02:35' - }, - '2019-07-03-01-35-03-bandwidth-3B06DAD9EDE8E1D08CC494E856AFD9D49256ECC2D68456799DD1EC7ED3436875': { + }, { + 'path': '2019-07-03-01-35-03-bandwidth-3B06DAD9EDE8E1D08CC494E856AFD9D49256ECC2D68456799DD1EC7ED3436875', 'size': 4249699, 'last_modified': '2019-07-03 02:35' } - } - }, - 'consensuses': { - 'files': { - '2019-07-03-02-00-00-consensus': { + ] + }, { + 'path': 'consensuses', + 'files': [ + { + 'path': '2019-07-03-02-00-00-consensus', 'size': 2211265, 'last_modified': '2019-07-03 02:05' - }, - '2019-07-03-03-00-00-consensus': { + }, { + 'path': '2019-07-03-03-00-00-consensus', 'size': 2204478, 'last_modified': '2019-07-03 03:05' - }, - '2019-07-03-04-00-00-consensus': { + }, { + 'path': '2019-07-03-04-00-00-consensus', 'size': 2202554, 'last_modified': '2019-07-03 04:05' } - } - }, - 'extra-infos': { - 'files': { - '2019-07-03-02-05-00-extra-infos': { + ] + }, { + 'path': 'extra-infos', + 'files': [ + { + 'path': '2019-07-03-02-05-00-extra-infos', 'size': 1162899, 'last_modified': '2019-07-03 02:05' - }, - '2019-07-03-03-05-00-extra-infos': { + }, { + 'path': '2019-07-03-03-05-00-extra-infos', 'size': 1133425, 'last_modified': '2019-07-03 03:05' - }, - '2019-07-03-04-05-00-extra-infos': { + }, { + 'path': '2019-07-03-04-05-00-extra-infos', 'size': 1153793, 'last_modified': '2019-07-03 04:05' } - } - }, - 'microdescs': { - 'directories': { - 'consensus-microdesc': { - 'files': { - '2019-07-03-02-00-00-consensus-microdesc': { + ] + }, { + 'path': 'microdescs', + 'directories': [ + { + 'path': 'consensus-microdesc', + 'files': [ + { + 'path': '2019-07-03-02-00-00-consensus-microdesc', 'size': 2028994, 'last_modified': '2019-07-03 02:05' - }, - '2019-07-03-03-00-00-consensus-microdesc': { + }, { + 'path': '2019-07-03-03-00-00-consensus-microdesc', 'size': 2022808, 'last_modified': '2019-07-03 03:05' - }, - '2019-07-03-04-00-00-consensus-microdesc': { + }, { + 'path': '2019-07-03-04-00-00-consensus-microdesc', 'size': 2020751, 'last_modified': '2019-07-03 04:05' } - } - }, - 'micro': { - 'files': { - '2019-07-02-23-05-00-micro': { + ] + }, { + 'path': 'micro', + 'files': [ + { + 'path': '2019-07-02-23-05-00-micro', 'size': 19934, 'last_modified': '2019-07-02 23:05' - }, - '2019-07-03-00-05-00-micro': { + }, { + 'path': '2019-07-03-00-05-00-micro', 'size': 12030, 'last_modified': '2019-07-03 00:05' - }, - '2019-07-03-01-05-00-micro': { + }, { + 'path': '2019-07-03-01-05-00-micro', 'size': 14850, 'last_modified': '2019-07-03 01:05' } - } + ] } - } - }, - 'server-descriptors': { - 'files': { - '2019-07-03-02-05-00-server-descriptors': { + ] + }, { + 'path': 'server-descriptors', + 'files': [ + { + 'path': '2019-07-03-02-05-00-server-descriptors', 'size': 1374587, 'last_modified': '2019-07-03 02:05' - }, - '2019-07-03-03-05-00-server-descriptors': { + }, { + 'path': '2019-07-03-03-05-00-server-descriptors', 'size': 1353286, 'last_modified': '2019-07-03 03:05' - }, - '2019-07-03-04-05-00-server-descriptors': { + }, { + 'path': '2019-07-03-04-05-00-server-descriptors', 'size': 1336125, 'last_modified': '2019-07-03 04:05' } - } - }, - 'votes': { - 'files': { - '2019-07-03-02-00-00-vote-0232AF901C31A04EE9848595AF9BB7620D4C5B2E-D9B6923A3AF9FB5DB3CD30191BCBD29A7A936D86': { + ] + }, { + 'path': 'votes', + 'files': [ + { + 'path': '2019-07-03-02-00-00-vote-0232AF901C31A04EE9848595AF9BB7620D4C5B2E-D9B6923A3AF9FB5DB3CD30191BCBD29A7A936D86', 'size': 3449707, 'last_modified': '2019-07-03 02:05' - }, - '2019-07-03-02-00-00-vote-14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4-39429F0789A9EC07A2A8754C4C449CCABAB787CC': { + }, { + 'path': '2019-07-03-02-00-00-vote-14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4-39429F0789A9EC07A2A8754C4C449CCABAB787CC', 'size': 3435482, 'last_modified': '2019-07-03 02:05' - }, - '2019-07-03-02-00-00-vote-23D15D965BC35114467363C165C4F724B64B4F66-4337842B697467123B5351E1E8EB64ED497C0EDA': { + }, { + 'path': '2019-07-03-02-00-00-vote-23D15D965BC35114467363C165C4F724B64B4F66-4337842B697467123B5351E1E8EB64ED497C0EDA', 'size': 3571106, 'last_modified': '2019-07-03 02:05' } - } + ] } - } - }, - 'torperf': { - 'files': { - 'op-ab-1048576-2019-07-03.tpf': { + ] + }, { + 'path': 'torperf', + 'files': [ + { + 'path': 'op-ab-1048576-2019-07-03.tpf', 'size': 39152, 'last_modified': '2019-07-04 00:01' - }, - 'op-ab-51200-2019-07-03.tpf': { + }, { + 'path': 'op-ab-51200-2019-07-03.tpf', 'size': 248173, 'last_modified': '2019-07-04 00:01' - }, - 'op-ab-5242880-2019-07-03.tpf': { + }, { + 'path': 'op-ab-5242880-2019-07-03.tpf', 'size': 15900, 'last_modified': '2019-07-04 00:01' } - } - }, - 'webstats': { - 'files': { - '2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20190620.xz': { + ] + }, { + 'path': 'webstats', + 'files': [ + { + 'path': '2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20190620.xz', 'size': 8872, 'last_modified': '2019-07-03 04:21' - }, - '2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20190621.xz': { + }, { + 'path': '2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20190621.xz', 'size': 9312, 'last_modified': '2019-07-04 04:21' - }, - '2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20190622.xz': { + }, { + 'path': '2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20190622.xz', 'size': 9836, 'last_modified': '2019-07-05 04:21' } - } + ] } - } + ] } - } + ] } [View Less]

1 0

[stem/master] CollecTor Index class
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit 860fda10124767e68e713436a4e191a2b0db4bfd Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jul 7 11:21:00 2019 -0700 CollecTor Index class Class to provide the methods we'll need to work with the index. --- stem/descriptor/collector.py | 52 +++++++++++++++++++++++++-------------- test/unit/descriptor/collector.py | 13 +++++----- 2 files changed, 40 insertions(+), 25 deletions(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.… [View More]py index 2633558e..770a1296 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -121,29 +121,44 @@ def _download(url, compression, timeout, retries): return stem.util.str_tools._to_unicode(response) -def _convert_index_paths(val): +class Index(object): + """ + Index of CollecTor's content. """ - Key files and directories off their paths so we can work with them more - efficiently. - :val dict val: index to convert + def __init__(self, content): + self._str_content = content + self._hash_content = Index._convert_paths(json.loads(content)) - :returns: index with files and directories converted to path-keyed hashes + def __str__(self): + return self._str_content - :raises: **ValueError** if unable to parse the index - """ + def __iter__(self): + for k, v in self._hash_content.items(): + yield k, v + + @staticmethod + def _convert_paths(val): + """ + Key files and directories off their paths so we can transverse them more + efficiently. - if isinstance(val, dict): - return dict([(k, _convert_index_paths(v)) for (k, v) in val.items()]) - elif isinstance(val, list) and all([isinstance(entry, dict) and 'path' in entry for entry in val]): - contents = {} + :val dict val: index to convert - for entry in val: - contents[entry['path']] = dict((k, _convert_index_paths(v)) for (k, v) in entry.items() if k != 'path') + :returns: index with files and directories converted to path-keyed hashes + """ + + if isinstance(val, dict): + return dict([(k, Index._convert_paths(v)) for (k, v) in val.items()]) + elif isinstance(val, list) and all([isinstance(entry, dict) and 'path' in entry for entry in val]): + contents = {} - return contents - else: - return val + for entry in val: + contents[entry['path']] = dict((k, Index._convert_paths(v)) for (k, v) in entry.items() if k != 'path') + + return contents + else: + return val class CollecTor(object): @@ -179,7 +194,8 @@ class CollecTor(object): """ Provides the archives available in CollecTor. - :returns: **dict** with the archive contents + :returns: :class:`~stem.descriptor.collector.Index` with the archive + contents :raises: If unable to retrieve the index this provide... @@ -192,7 +208,7 @@ class CollecTor(object): if not self._cached_index or time.time() - self._cached_index_at >= REFRESH_INDEX_RATE: response = _download(COLLECTOR_URL + 'index/index.json', self.compression, self.timeout, self.retries) - self._cached_index = _convert_index_paths(json.loads(response)) + self._cached_index = Index(response) self._cached_index_at = time.time() return self._cached_index diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index 56ab19c1..2e11c89d 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -35,7 +35,7 @@ class TestCollector(unittest.TestCase): urlopen_mock.return_value = io.BytesIO(MINIMAL_INDEX_JSON) collector = CollecTor(compression = Compression.PLAINTEXT) - self.assertEqual(MINIMAL_INDEX, collector.index()) + self.assertEqual(MINIMAL_INDEX, dict(collector.index())) urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json', timeout = None) @patch(URL_OPEN) @@ -48,7 +48,7 @@ class TestCollector(unittest.TestCase): urlopen_mock.return_value = io.BytesIO(zlib.compress(MINIMAL_INDEX_JSON)) collector = CollecTor(compression = Compression.GZIP) - self.assertEqual(MINIMAL_INDEX, collector.index()) + self.assertEqual(MINIMAL_INDEX, dict(collector.index())) urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.gz', timeout = None) @patch(URL_OPEN) @@ -61,7 +61,7 @@ class TestCollector(unittest.TestCase): urlopen_mock.return_value = io.BytesIO(bz2.compress(MINIMAL_INDEX_JSON)) collector = CollecTor(compression = Compression.BZ2) - self.assertEqual(MINIMAL_INDEX, collector.index()) + self.assertEqual(MINIMAL_INDEX, dict(collector.index())) urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.bz2', timeout = None) @patch(URL_OPEN) @@ -74,7 +74,7 @@ class TestCollector(unittest.TestCase): urlopen_mock.return_value = io.BytesIO(lzma.compress(MINIMAL_INDEX_JSON)) collector = CollecTor(compression = Compression.LZMA) - self.assertEqual(MINIMAL_INDEX, collector.index()) + self.assertEqual(MINIMAL_INDEX, dict(collector.index())) urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.lzma', timeout = None) @patch(URL_OPEN) @@ -94,7 +94,7 @@ class TestCollector(unittest.TestCase): @patch(URL_OPEN, Mock(return_value = io.BytesIO(MINIMAL_INDEX_JSON))) def test_index(self): collector = CollecTor(compression = Compression.PLAINTEXT) - self.assertEqual(MINIMAL_INDEX, collector.index()) + self.assertEqual(MINIMAL_INDEX, dict(collector.index())) @patch(URL_OPEN, Mock(return_value = io.BytesIO(b'not json'))) def test_index_malformed_json(self): @@ -119,6 +119,5 @@ class TestCollector(unittest.TestCase): with open(get_resource('collector_index.json'), 'rb') as index_file: urlopen_mock.return_value = io.BytesIO(index_file.read()) - self.maxDiff = None collector = CollecTor(compression = Compression.PLAINTEXT) - self.assertEqual(EXAMPLE_INDEX, collector.index()) + self.assertEqual(EXAMPLE_INDEX, dict(collector.index())) [View Less]

1 0

[stem/master] Pre-emptively sort cached files
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit 41b63747e5b4a477b6eca3b72baf861a03047aaa Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jul 22 15:49:50 2019 -0700 Pre-emptively sort cached files Stupid me. There's no value in sorting filtered lists upon every query - might as well sort our cache so this only needs to be done once. --- stem/descriptor/collector.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py … [View More]

1 0

[stem/master] Initial descriptor reading function
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit 07bf976cbf57824e0425ee33bfd9763934915bbb Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Jul 24 18:12:29 2019 -0700 Initial descriptor reading function Still quite a few rough edges, but works. Exercised with... import stem.descriptor.collector collector = stem.descriptor.collector.CollecTor() f = list(filter(lambda x: 'server-descriptor 1.0' in x._guess_descriptor_types(), collector.files()))[0] for desc in f.read('/… [View More]home/atagar/Desktop/foo'): print(desc) We're splitting the download and read methods so descriptor archives can optionally cache locally. --- stem/descriptor/collector.py | 123 ++++++++++++++++++++++++++++++++++++-- test/unit/descriptor/collector.py | 8 +-- 2 files changed, 122 insertions(+), 9 deletions(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index 754a96b9..1f7ddb96 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -51,11 +51,15 @@ With this you can either download and read directly from CollecTor... import datetime import json +import os import re import sys +import tempfile import time -from stem.descriptor import Compression +import stem.util.str_tools + +from stem.descriptor import Compression, parse_file from stem.util import log try: @@ -121,7 +125,6 @@ def _download(url, timeout, retries): :returns: content of the given url :raises: - * **IOError** if unable to decompress * **socket.timeout** if our request timed out * **urllib2.URLError** for most request failures @@ -174,8 +177,118 @@ class File(object): self.last_modified = datetime.datetime.strptime(last_modified, '%Y-%m-%d %H:%M') self._guessed_type = None + self._downloaded_to = None # location we last downloaded to + + def read(self, directory = None, descriptor_type = None, timeout = None, retries = 3): + """ + Provides descriptors from this archive. Descriptors are downloaded or read + from disk as follows... + + * If this file has already been downloaded through + :func:`~stem.descriptor.collector.CollecTor.download' these descriptors + are read from disk. + + * If a **directory** argument is provided and the file is already present + these descriptors are read from disk. + + * If a **directory** argument is provided and the file is not present the + file is downloaded this location then read. + + * If the file has neither been downloaded and no **directory** argument + is provided then the file is downloaded to a temporary directory that's + deleted after it is read. + + :param str directory: destination to download into + :param str descriptor_type: `descriptor type + <https://metrics.torproject.org/collector.html#data-formats>`_, this is + guessed if not provided + :param int timeout: timeout when connection becomes idle, no timeout + applied if **None** + :param int retires: maximum attempts to impose + + :returns: iterator for :class:`~stem.descriptor.__init__.Descriptor` + instances in the file + + :raises: + * **ValueError** if unable to determine the descirptor type + * **TypeError** if we cannot parse this descriptor type + * **socket.timeout** if our request timed out + * **urllib2.URLError** for most request failures + + Note that the urllib2 module may fail with other exception types, in + which case we'll pass it along. + """ + + if descriptor_type is None: + descriptor_types = self._guess_descriptor_types() + + if not descriptor_types: + raise ValueError("Unable to determine this file's descriptor type") + elif len(descriptor_types) > 1: + raise ValueError("Unable to determine disambiguate file's descriptor type from %s" % ', '.join(descriptor_types)) + + descriptor_type = descriptor_types[0] + + if directory is None: + if self._downloaded_to and os.path.exists(self._downloaded_to): + directory = os.path.dirname(self._downloaded_to) + else: + with tempfile.TemporaryDirectory() as tmp_directory: + return self.read(tmp_directory, timeout, retries) + + path = self.download(directory, True, timeout, retries) + return parse_file(path, descriptor_type) + + def download(self, directory, decompress = True, timeout = None, retries = 3): + """ + Downloads this file to the given location. If a file already exists this is + a no-op. + + :param str directory: destination to download into + :param bool decompress: decompress written file + :param int timeout: timeout when connection becomes idle, no timeout + applied if **None** + :param int retires: maximum attempts to impose + + :returns: **str** with the path we downloaded to + + :raises: + * **socket.timeout** if our request timed out + * **urllib2.URLError** for most request failures + + Note that the urllib2 module may fail with other exception types, in + which case we'll pass it along. + """ + + # TODO: If checksums get added to the index we should replace + # the path check below to verify that... + # + # https://trac.torproject.org/projects/tor/ticket/31204 + + filename = self.path.split('/')[-1] + + if decompress: + filename = filename.rsplit('.', 1)[0] + + path = os.path.join(directory, filename) + + if not os.path.exists(directory): + os.makedirs(directory) + elif os.path.exists(path): + return path # file already exists + + with open(path, 'wb') as output_file: + response = _download(COLLECTOR_URL + self.path, timeout, retries) + + if decompress: + response = self.compression.decompress(response) + + output_file.write(response) + + self._downloaded_to = path + return path - def guess_descriptor_types(self): + def _guess_descriptor_types(self): """ Descriptor @type this file is expected to have based on its path. If unable to determine any this tuple is empty. @@ -290,7 +403,7 @@ class CollecTor(object): url = COLLECTOR_URL + 'index/index.json' + extension response = compression.decompress(_download(url, self.timeout, self.retries)) - self._cached_index = json.loads(response) + self._cached_index = json.loads(stem.util.str_tools._to_unicode(response)) self._cached_index_at = time.time() return self._cached_index @@ -325,7 +438,7 @@ class CollecTor(object): elif end and (entry.end is None or entry.end > end): continue - if descriptor_type is None or any([desc_type.startswith(descriptor_type) for desc_type in entry.guess_descriptor_types()]): + if descriptor_type is None or any([desc_type.startswith(descriptor_type) for desc_type in entry._guess_descriptor_types()]): matches.append(entry) return matches diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index 86641f32..914b52b9 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -209,13 +209,13 @@ class TestCollector(unittest.TestCase): def test_guess_descriptor_types(self): f = File('archive/bridge-descriptors/extra-infos/bridge-extra-infos-2008-05.tar.xz', 377644, '2016-09-04 09:21') - self.assertEqual(('bridge-extra-info 1.3',), f.guess_descriptor_types()) + self.assertEqual(('bridge-extra-info 1.3',), f._guess_descriptor_types()) f = File('archive/relay-descriptors/microdescs/microdescs-2014-01.tar.xz', 7515396, '2014-02-07 03:59') - self.assertEqual(('network-status-microdesc-consensus-3 1.0', 'microdescriptor 1.0'), f.guess_descriptor_types()) + self.assertEqual(('network-status-microdesc-consensus-3 1.0', 'microdescriptor 1.0'), f._guess_descriptor_types()) f = File('archive/webstats/webstats-2015-03.tar', 20480, '2018-03-19 16:07') - self.assertEqual((), f.guess_descriptor_types()) + self.assertEqual((), f._guess_descriptor_types()) f = File('archive/no_such_file.tar', 20480, '2018-03-19 16:07') - self.assertEqual((), f.guess_descriptor_types()) + self.assertEqual((), f._guess_descriptor_types()) [View Less]

1 0

[stem/master] Estimate descriptor dates from filenames
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit 8be08440456199873a1d3c6eb2f8e3ed968eedc0 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jul 14 14:37:52 2019 -0700 Estimate descriptor dates from filenames Now that we have descriptor types and compression the last ingredient we need are the time ranges a file is for. Guessing this from filenames. --- stem/descriptor/collector.py | 65 ++++++++++++++++++++++++++++++++++----- test/unit/descriptor/collector.py | 21 +++++++++++++ 2 files changed, … [View More]78 insertions(+), 8 deletions(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index a47612cb..c1458784 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -51,6 +51,7 @@ With this you can either download and read directly from CollecTor... import datetime import json +import re import sys import time @@ -70,6 +71,10 @@ import stem.util.str_tools COLLECTOR_URL = 'https://collector.torproject.org/' REFRESH_INDEX_RATE = 3600 # get new index if cached copy is an hour old +YEAR_DATE = re.compile('-(\\d{4})-(\\d{2})\\.') +SEC_DATE = re.compile('(\\d{4}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-\\d{2})') + + # mapping of path prefixes to their descriptor type (sampled 7/11/19) COLLECTOR_DESC_TYPES = { @@ -165,23 +170,24 @@ class File(object): this cannot be determined :var bool tar: **True** if a tarball, **False** otherwise :var int size: size of the file + + :var datetime start: beginning of the time range descriptors are for, + **None** if this cannot be determined + :var datetime end: ending of the time range descriptors are for, + **None** if this cannot be determined :var datetime last_modified: when the file was last modified """ def __init__(self, path, size, last_modified): self.path = path - self.compression = None + self.compression = File._guess_compression(path) self.tar = path.endswith('.tar') or '.tar.' in path self.size = size + + self.start, self.end = File._guess_time_range(path) self.last_modified = datetime.datetime.strptime(last_modified, '%Y-%m-%d %H:%M') - self._guessed_type = None - if '.' not in self.path or self.path.endswith('.tar'): - self.compression = Compression.PLAINTEXT - else: - for compression in (Compression.LZMA, Compression.BZ2, Compression.GZIP): - if self.path.endswith(compression.extension): - self.compression = compression + self._guessed_type = None def guess_descriptor_types(self): """ @@ -203,6 +209,49 @@ class File(object): return self._guessed_type + @staticmethod + def _guess_compression(path): + """ + Determine file comprssion from CollecTor's filename. + """ + + if '.' not in path or path.endswith('.tar'): + return Compression.PLAINTEXT + else: + for compression in (Compression.LZMA, Compression.BZ2, Compression.GZIP): + if path.endswith(compression.extension): + return compression + + @staticmethod + def _guess_time_range(path): + """ + Attemt to determine the (start, end) time range from CollecTor's filename. + This provides (None, None) if this cannot be determined. + """ + + year_match = YEAR_DATE.search(path) + + if year_match: + year, month = map(int, year_match.groups()) + start = datetime.datetime(year, month, 1) + + if month < 12: + return (start, datetime.datetime(year, month + 1, 1)) + else: + return (start, datetime.datetime(year + 1, 1, 1)) + + sec_match = SEC_DATE.search(path) + + if sec_match: + # Descriptors in the 'recent/*' section have filenames with second level + # granularity. Not quite sure why, but since consensus documents are + # published hourly we'll use that as the delta here. + + start = datetime.datetime.strptime(sec_match.group(1), '%Y-%m-%d-%H-%M-%S') + return (start, start + datetime.timedelta(seconds = 3600)) + + return (None, None) + class CollecTor(object): """ diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index 3c4d39a0..297050ea 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -151,6 +151,27 @@ class TestCollector(unittest.TestCase): self.assertEqual(Compression.PLAINTEXT, f.compression) self.assertEqual(False, f.tar) + def test_file_date_attributes(self): + f = File('archive/relay-descriptors/microdescs/microdescs-2014-01.tar.xz', 7515396, '2014-02-07 03:59') + self.assertEqual(datetime.datetime(2014, 1, 1), f.start) + self.assertEqual(datetime.datetime(2014, 2, 1), f.end) + + f = File('recent/relay-descriptors/extra-infos/2019-07-03-02-05-00-extra-infos', 1162899, '2019-07-03 02:05') + self.assertEqual(datetime.datetime(2019, 7, 3, 2, 5, 0), f.start) + self.assertEqual(datetime.datetime(2019, 7, 3, 3, 5, 0), f.end) + + f = File('archive/relay-descriptors/certs.tar.xz', 144696, '2019-07-03 03:29') + self.assertEqual(None, f.start) + self.assertEqual(None, f.end) + + # check date boundaries + + f = File('archive/relay-descriptors/microdescs/microdescs-2014-12.tar.xz', 7515396, '2014-02-07 03:59') + self.assertEqual(datetime.datetime(2015, 1, 1), f.end) + + f = File('recent/relay-descriptors/extra-infos/2019-07-03-23-05-00-extra-infos', 1162899, '2019-07-03 02:05') + self.assertEqual(datetime.datetime(2019, 7, 4, 0, 5, 0), f.end) + def test_guess_descriptor_types(self): f = File('archive/bridge-descriptors/extra-infos/bridge-extra-infos-2008-05.tar.xz', 377644, '2016-09-04 09:21') self.assertEqual(('bridge-extra-info 1.3',), f.guess_descriptor_types()) [View Less]

1 0

[stem/master] CollecTor File class
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit 790af0715095e06d559a963dfeff0a3b41e1eb51 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jul 7 13:01:25 2019 -0700 CollecTor File class On reflection, think a file-level object will be more useful than representing the index (we don't need anything from it beside files). --- stem/descriptor/collector.py | 45 +++++++++++++++++++++++++++++++++++++++ test/unit/descriptor/collector.py | 24 ++++++++++++++++----- 2 files changed, 64 insertions(+), 5 … [View More]deletions(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index 770a1296..135a5d00 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -49,6 +49,7 @@ With this you can either download and read directly from CollecTor... .. versionadded:: 1.8.0 """ +import datetime import json import sys import time @@ -124,12 +125,16 @@ def _download(url, compression, timeout, retries): class Index(object): """ Index of CollecTor's content. + + :var hash files: mapping of paths to thier content """ def __init__(self, content): self._str_content = content self._hash_content = Index._convert_paths(json.loads(content)) + self.files = Index._get_files(self._hash_content, []) + def __str__(self): return self._str_content @@ -138,6 +143,31 @@ class Index(object): yield k, v @staticmethod + def _get_files(val, path): + """ + Provies a mapping of paths to files within the index. + + :param dict val: index hash + :param list path: path we've transversed into + + :returns: **dict** mapping paths to files + """ + + files = {} + + if isinstance(val, dict): + for k, v in val.items(): + if k == 'files': + for filename, attr in v.items(): + file_path = '/'.join(path + [filename]) + files[file_path] = File(file_path, attr.get('size'), attr.get('last_modified')) + elif k == 'directories': + for filename, attr in v.items(): + files.update(Index._get_files(attr, path + [filename])) + + return files + + @staticmethod def _convert_paths(val): """ Key files and directories off their paths so we can transverse them more @@ -161,6 +191,21 @@ class Index(object): return val +class File(object): + """ + File within CollecTor. + + :var str path: file path within collector + :var int size: size of the file + :var datetime last_modified: when the file was last modified + """ + + def __init__(self, path, size, last_modified): + self.path = path + self.size = size + self.last_modified = datetime.datetime.strptime(last_modified, '%Y-%m-%d %H:%M') + + class CollecTor(object): """ Downloader for descriptors from CollecTor. The contents of CollecTor are diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index 2e11c89d..cc12ee00 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -2,6 +2,7 @@ Unit tests for stem.descriptor.collector. """ +import datetime import io import unittest @@ -28,6 +29,9 @@ MINIMAL_INDEX = { MINIMAL_INDEX_JSON = b'{"index_created":"2017-12-25 21:06","build_revision":"56a303e","path":"https://collector.torproject.org"}' +with open(get_resource('collector_index.json'), 'rb') as index_file: + EXAMPLE_INDEX_CONTENT = index_file.read() + class TestCollector(unittest.TestCase): @patch(URL_OPEN) @@ -114,10 +118,20 @@ class TestCollector(unittest.TestCase): collector = CollecTor(compression = compression) self.assertRaisesRegexp(IOError, 'Unable to decompress %s response' % compression, collector.index) - @patch(URL_OPEN) - def test_real_index(self, urlopen_mock): - with open(get_resource('collector_index.json'), 'rb') as index_file: - urlopen_mock.return_value = io.BytesIO(index_file.read()) - + @patch(URL_OPEN, Mock(return_value = io.BytesIO(EXAMPLE_INDEX_CONTENT))) + def test_real_index(self): collector = CollecTor(compression = Compression.PLAINTEXT) self.assertEqual(EXAMPLE_INDEX, dict(collector.index())) + + @patch(URL_OPEN, Mock(return_value = io.BytesIO(EXAMPLE_INDEX_CONTENT))) + def test_contents(self): + collector = CollecTor(compression = Compression.PLAINTEXT) + index = collector.index() + + self.assertEqual(85, len(index.files)) + test_path = 'archive/relay-descriptors/extra-infos/extra-infos-2007-09.tar.xz' + + extrainfo_file = index.files[test_path] + self.assertEqual(test_path, extrainfo_file.path) + self.assertEqual(6459884, extrainfo_file.size) + self.assertEqual(datetime.datetime(2016, 6, 23, 9, 54), extrainfo_file.last_modified) [View Less]

1 0

[stem/master] Drop zstd from CollecTor unit tests
by atagar＠torproject.org 17 Aug '19

17 Aug '19

commit ae1970e2d8f25c097a61a48347a520d25314a5bc Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jul 5 14:49:15 2019 -0700 Drop zstd from CollecTor unit tests Oops! Just remembered why I didn't originally include zstd tests. CollecTor doesn't do zstd compression (just tor's DirPort). --- test/unit/descriptor/collector.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/test/unit/descriptor/collector.py b/test/unit/… [View More]

1 0