[tor-commits] [stem/master] Converting stem.descriptor.* to reStructuredText
atagar at torproject.org
atagar at torproject.org
Wed Jun 6 04:30:17 UTC 2012
commit 4819e2fc97f76424a3dead741d327be711f53a8d
Author: Damian Johnson <atagar at torproject.org>
Date: Tue Jun 5 21:15:37 2012 -0700
Converting stem.descriptor.* to reStructuredText
Fingers so sore...
---
stem/descriptor/__init__.py | 74 ++++-----
stem/descriptor/extrainfo_descriptor.py | 275 +++++++++++++++----------------
stem/descriptor/reader.py | 110 ++++++-------
stem/descriptor/server_descriptor.py | 212 +++++++++++-------------
4 files changed, 318 insertions(+), 353 deletions(-)
diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py
index 1de1dac..37f9ec7 100644
--- a/stem/descriptor/__init__.py
+++ b/stem/descriptor/__init__.py
@@ -1,11 +1,15 @@
"""
Package for parsing and processing descriptor data.
-parse_file - Iterates over the descriptors in a file.
-Descriptor - Common parent for all descriptor file types.
- |- get_path - location of the descriptor on disk if it came from a file
- |- get_unrecognized_lines - unparsed descriptor content
- +- __str__ - string that the descriptor was made from
+**Module Overview:**
+
+::
+
+ parse_file - Iterates over the descriptors in a file.
+ Descriptor - Common parent for all descriptor file types.
+ |- get_path - location of the descriptor on disk if it came from a file
+ |- get_unrecognized_lines - unparsed descriptor content
+ +- __str__ - string that the descriptor was made from
"""
__all__ = ["descriptor", "reader", "extrainfo_descriptor", "server_descriptor", "parse_file", "Descriptor"]
@@ -23,16 +27,14 @@ def parse_file(path, descriptor_file):
"""
Provides an iterator for the descriptors within a given file.
- Arguments:
- path (str) - absolute path to the file's location on disk
- descriptor_file (file) - opened file with the descriptor contents
+ :param str path: absolute path to the file's location on disk
+ :param file descriptor_file: opened file with the descriptor contents
- Returns:
- iterator for Descriptor instances in the file
+ :returns: iterator for :class:`stem.descriptor.Descriptor` instances in the file
- Raises:
- TypeError if we can't match the contents of the file to a descriptor type
- IOError if unable to read from the descriptor_file
+ :raises:
+ * TypeError if we can't match the contents of the file to a descriptor type
+ * IOError if unable to read from the descriptor_file
"""
import stem.descriptor.server_descriptor
@@ -93,8 +95,7 @@ class Descriptor:
"""
Provides the absolute path that we loaded this descriptor from.
- Returns:
- str with the absolute path of the descriptor source
+ :returns: str with the absolute path of the descriptor source
"""
return self._path
@@ -105,8 +106,7 @@ class Descriptor:
not know how to process. This is most common due to new descriptor fields
that this library does not yet know how to process. Patches welcome!
- Returns:
- list of lines of unrecognized content
+ :returns: list of lines of unrecognized content
"""
raise NotImplementedError
@@ -122,13 +122,11 @@ def _read_until_keyword(keyword, descriptor_file, inclusive = False):
Reads from the descriptor file until we get to the given keyword or reach the
end of the file.
- Arguments:
- keyword (str) - keyword we want to read until
- descriptor_file (file) - file with the descriptor content
- inclusive (bool) - includes the line with the keyword if True
+ :param str keyword: keyword we want to read until
+ :param file descriptor_file: file with the descriptor content
+ :param bool inclusive: includes the line with the keyword if True
- Returns:
- list with the lines until we find the keyword
+ :returns: list with the lines until we find the keyword
"""
content = []
@@ -156,15 +154,11 @@ def _get_pseudo_pgp_block(remaining_contents):
Checks if given contents begins with a pseudo-Open-PGP-style block and, if
so, pops it off and provides it back to the caller.
- Arguments:
- remaining_contents (list) - lines to be checked for a public key block
+ :param list remaining_contents: lines to be checked for a public key block
- Returns:
- str with the armor wrapped contents or None if it doesn't exist
+ :returns: str with the armor wrapped contents or None if it doesn't exist
- Raises:
- ValueError if the contents starts with a key block but it's malformed (for
- instance, if it lacks an ending line)
+ :raises: ValueError if the contents starts with a key block but it's malformed (for instance, if it lacks an ending line)
"""
if not remaining_contents:
@@ -202,19 +196,17 @@ def _get_descriptor_components(raw_contents, validate, extra_keywords):
entries because this influences the resulting exit policy, but for everything
else in server descriptors the order does not matter.
- Arguments:
- raw_contents (str) - descriptor content provided by the relay
- validate (bool) - checks the validity of the descriptor's content if
- True, skips these checks otherwise
- extra_keywords (list) - entity keywords to put into a separate listing with
- ordering intact
+ :param str raw_contents: descriptor content provided by the relay
+ :param bool validate: checks the validity of the descriptor's content if True, skips these checks otherwise
+ :param list extra_keywords: entity keywords to put into a separate listing with ordering intact
- Returns:
+ :returns:
tuple with the following attributes...
- entries (dict) - keyword => (value, pgp key) entries
- first_keyword (str) - keyword of the first line
- last_keyword (str) - keyword of the last line
- extra_entries (list) - lines containing entries matching extra_keywords
+
+ * **entries (dict)** - keyword => (value, pgp key) entries
+ * **first_keyword (str)** - keyword of the first line
+ * **last_keyword (str)** - keyword of the last line
+ * **extra_entries (list)** - lines containing entries matching extra_keywords
"""
entries = {}
diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py
index 1c00d34..ff37064 100644
--- a/stem/descriptor/extrainfo_descriptor.py
+++ b/stem/descriptor/extrainfo_descriptor.py
@@ -10,33 +10,39 @@ cannot be requested of bridges.
Extra-info descriptors are available from a few sources...
-- if you have 'DownloadExtraInfo 1' in your torrc...
- - control port via 'GETINFO extra-info/digest/*' queries
- - the 'cached-extrainfo' file in tor's data directory
-- tor metrics, at https://metrics.torproject.org/data.html
-- directory authorities and mirrors via their DirPort
+* if you have 'DownloadExtraInfo 1' in your torrc...
-DirResponses - known statuses for ExtraInfoDescriptor's dir_*_responses
- |- OK - network status requests that were answered
- |- NOT_ENOUGH_SIGS - network status wasn't signed by enough authorities
- |- UNAVAILABLE - requested network status was unavailable
- |- NOT_FOUND - requested network status was not found
- |- NOT_MODIFIED - network status unmodified since If-Modified-Since time
- +- BUSY - directory was busy
+ * control port via 'GETINFO extra-info/digest/*' queries
+ * the 'cached-extrainfo' file in tor's data directory
-DirStats - known stats for ExtraInfoDescriptor's dir_*_direct_dl and dir_*_tunneled_dl
- |- COMPLETE - requests that completed successfully
- |- TIMEOUT - requests that didn't complete within a ten minute timeout
- |- RUNNING - requests still in procress when measurement's taken
- |- MIN - smallest rate at which a descriptor was downloaded in B/s
- |- MAX - largest rate at which a descriptor was downloaded in B/s
- |- D1-4 and D6-9 - rate of the slowest x/10 download rates in B/s
- |- Q1 and Q3 - rate of the slowest and fastest querter download rates in B/s
- +- MD - median download rate in B/s
+* tor metrics, at https://metrics.torproject.org/data.html
+* directory authorities and mirrors via their DirPort
-parse_file - Iterates over the extra-info descriptors in a file.
-ExtraInfoDescriptor - Tor extra-info descriptor.
- +- get_unrecognized_lines - lines with unrecognized content
+**Module Overview:**
+
+::
+
+ DirResponses - known statuses for ExtraInfoDescriptor's dir_*_responses
+ |- OK - network status requests that were answered
+ |- NOT_ENOUGH_SIGS - network status wasn't signed by enough authorities
+ |- UNAVAILABLE - requested network status was unavailable
+ |- NOT_FOUND - requested network status was not found
+ |- NOT_MODIFIED - network status unmodified since If-Modified-Since time
+ +- BUSY - directory was busy
+
+ DirStats - known stats for ExtraInfoDescriptor's dir_*_direct_dl and dir_*_tunneled_dl
+ |- COMPLETE - requests that completed successfully
+ |- TIMEOUT - requests that didn't complete within a ten minute timeout
+ |- RUNNING - requests still in procress when measurement's taken
+ |- MIN - smallest rate at which a descriptor was downloaded in B/s
+ |- MAX - largest rate at which a descriptor was downloaded in B/s
+ |- D1-4 and D6-9 - rate of the slowest x/10 download rates in B/s
+ |- Q1 and Q3 - rate of the slowest and fastest querter download rates in B/s
+ +- MD - median download rate in B/s
+
+ parse_file - Iterates over the extra-info descriptors in a file.
+ ExtraInfoDescriptor - Tor extra-info descriptor.
+ +- get_unrecognized_lines - lines with unrecognized content
"""
import re
@@ -112,17 +118,14 @@ def parse_file(descriptor_file, validate = True):
"""
Iterates over the extra-info descriptors in a file.
- Arguments:
- descriptor_file (file) - file with descriptor content
- validate (bool) - checks the validity of the descriptor's content if
- True, skips these checks otherwise
+ :param file descriptor_file: file with descriptor content
+ :param bool validate: checks the validity of the descriptor's content if True, skips these checks otherwise
- Returns:
- iterator for ExtraInfoDescriptor instances in the file
+ :returns: iterator for ExtraInfoDescriptor instances in the file
- Raises:
- ValueError if the contents is malformed and validate is True
- IOError if the file can't be read
+ :raises:
+ * ValueError if the contents is malformed and validate is True
+ * IOError if the file can't be read
"""
while True:
@@ -140,16 +143,12 @@ def _parse_timestamp_and_interval(keyword, content):
"""
Parses a 'YYYY-MM-DD HH:MM:SS (NSEC s) *' entry.
- Arguments:
- keyword (str) - line's keyword
- content (str) - line content to be parsed
+ :param str keyword: line's keyword
+ :param str content: line content to be parsed
- Returns:
- tuple of the form...
- (timestamp (datetime), interval (int), remaining content (str))
+ :returns: tuple of the form ``(timestamp (datetime), interval (int), remaining content (str))``
- Raises:
- ValueError if the content is malformed
+ :raises: ValueError if the content is malformed
"""
line = "%s %s" % (keyword, content)
@@ -174,92 +173,97 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor):
"""
Extra-info descriptor document.
- Attributes:
- nickname (str) - relay's nickname (*)
- fingerprint (str) - identity key fingerprint (*)
- published (datetime) - time in GMT when this descriptor was made (*)
- geoip_db_digest (str) - sha1 of geoIP database file
- signature (str) - signature for this extrainfo descriptor (*)
-
- conn_bi_direct_end (datetime) - end of the sampling interval
- conn_bi_direct_interval (int) - seconds per interval
- conn_bi_direct_below (int) - connections that read/wrote less than 20 KiB
- conn_bi_direct_read (int) - connections that read at least 10x more than wrote
- conn_bi_direct_write (int) - connections that wrote at least 10x more than read
- conn_bi_direct_both (int) - remaining connections
-
- Bytes read/written for relayed traffic:
- read_history_end (datetime) - end of the sampling interval
- read_history_interval (int) - seconds per interval
- read_history_values (list) - bytes read during each interval
-
- write_history_end (datetime) - end of the sampling interval
- write_history_interval (int) - seconds per interval
- write_history_values (list) - bytes written during each interval
-
- Cell relaying statistics:
- cell_stats_end (datetime) - end of the period when stats were gathered
- cell_stats_interval (int) - length in seconds of the interval
- cell_processed_cells (list) - measurement of processed cells per circuit
- cell_queued_cells (list) - measurement of queued cells per circuit
- cell_time_in_queue (list) - mean enqueued time in milliseconds for cells
- cell_circuits_per_decile (int) - mean number of circuits in a deciles
-
- Directory Mirror Attributes:
- dir_stats_end (datetime) - end of the period when stats were gathered
- dir_stats_interval (int) - length in seconds of the interval
- dir_v2_ips (dict) - mapping of locales to rounded count of requester ips
- dir_v3_ips (dict) - mapping of locales to rounded count of requester ips
- dir_v2_share (float) - percent of total directory traffic it expects to serve
- dir_v3_share (float) - percent of total directory traffic it expects to serve
- dir_v2_requests (dict) - mapping of locales to rounded count of requests
- dir_v3_requests (dict) - mapping of locales to rounded count of requests
-
- dir_v2_responses (dict) - mapping of DirResponses to their rounded count
- dir_v3_responses (dict) - mapping of DirResponses to their rounded count
- dir_v2_responses_unknown (dict) - mapping of unrecognized statuses to their count
- dir_v3_responses_unknown (dict) - mapping of unrecognized statuses to their count
-
- dir_v2_direct_dl (dict) - mapping of DirStats to measurement over DirPort
- dir_v3_direct_dl (dict) - mapping of DirStats to measurement over DirPort
- dir_v2_direct_dl_unknown (dict) - mapping of unrecognized stats to their measurement
- dir_v3_direct_dl_unknown (dict) - mapping of unrecognized stats to their measurement
-
- dir_v2_tunneled_dl (dict) - mapping of DirStats to measurement over ORPort
- dir_v3_tunneled_dl (dict) - mapping of DirStats to measurement over ORPort
- dir_v2_tunneled_dl_unknown (dict) - mapping of unrecognized stats to their measurement
- dir_v3_tunneled_dl_unknown (dict) - mapping of unrecognized stats to their measurement
-
- Bytes read/written for directory mirroring:
- dir_read_history_end (datetime) - end of the sampling interval
- dir_read_history_interval (int) - seconds per interval
- dir_read_history_values (list) - bytes read during each interval
-
- dir_write_history_end (datetime) - end of the sampling interval
- dir_write_history_interval (int) - seconds per interval
- dir_write_history_values (list) - bytes read during each interval
-
- Guard Attributes:
- entry_stats_end (datetime) - end of the period when stats were gathered
- entry_stats_interval (int) - length in seconds of the interval
- entry_ips (dict) - mapping of locales to rounded count of unique user ips
-
- Exit Attributes:
- exit_stats_end (datetime) - end of the period when stats were gathered
- exit_stats_interval (int) - length in seconds of the interval
- exit_kibibytes_written (dict) - traffic per port (keys are ints or 'other')
- exit_kibibytes_read (dict) - traffic per port (keys are ints or 'other')
- exit_streams_opened (dict) - streams per port (keys are ints or 'other')
-
- Bridge Attributes:
- bridge_stats_end (datetime) - end of the period when stats were gathered
- bridge_stats_interval (int) - length in seconds of the interval
- bridge_ips (dict) - mapping of locales to rounded count of unique user ips
- geoip_start_time (datetime) - (deprecated) replaced by bridge_stats_end
- geoip_client_origins (dict) - (deprecated) replaced by bridge_ips
-
- (*) attribute is either required when we're parsed with validation or has a
- default value, others are left as None if undefined
+ :var str nickname: **\*** relay's nickname
+ :var str fingerprint: **\*** identity key fingerprint
+ :var datetime published: **\*** time in GMT when this descriptor was made
+ :var str geoip_db_digest: sha1 of geoIP database file
+ :var str signature: **\*** signature for this extrainfo descriptor
+
+ :var datetime conn_bi_direct_end: end of the sampling interval
+ :var int conn_bi_direct_interval: seconds per interval
+ :var int conn_bi_direct_below: connections that read/wrote less than 20 KiB
+ :var int conn_bi_direct_read: connections that read at least 10x more than wrote
+ :var int conn_bi_direct_write: connections that wrote at least 10x more than read
+ :var int conn_bi_direct_both: remaining connections
+
+ **Bytes read/written for relayed traffic:**
+
+ :var datetime read_history_end: end of the sampling interval
+ :var int read_history_interval: seconds per interval
+ :var list read_history_values: bytes read during each interval
+
+ :var datetime write_history_end: end of the sampling interval
+ :var int write_history_interval: seconds per interval
+ :var list write_history_values: bytes written during each interval
+
+ **Cell relaying statistics:**
+
+ :var datetime cell_stats_end: end of the period when stats were gathered
+ :var int cell_stats_interval: length in seconds of the interval
+ :var list cell_processed_cells: measurement of processed cells per circuit
+ :var list cell_queued_cells: measurement of queued cells per circuit
+ :var list cell_time_in_queue: mean enqueued time in milliseconds for cells
+ :var int cell_circuits_per_decile: mean number of circuits in a deciles
+
+ **Directory Mirror Attributes:**
+
+ :var datetime dir_stats_end: end of the period when stats were gathered
+ :var int dir_stats_interval: length in seconds of the interval
+ :var dict dir_v2_ips: mapping of locales to rounded count of requester ips
+ :var dict dir_v3_ips: mapping of locales to rounded count of requester ips
+ :var float dir_v2_share: percent of total directory traffic it expects to serve
+ :var float dir_v3_share: percent of total directory traffic it expects to serve
+ :var dict dir_v2_requests: mapping of locales to rounded count of requests
+ :var dict dir_v3_requests: mapping of locales to rounded count of requests
+
+ :var dict dir_v2_responses: mapping of DirResponses to their rounded count
+ :var dict dir_v3_responses: mapping of DirResponses to their rounded count
+ :var dict dir_v2_responses_unknown: mapping of unrecognized statuses to their count
+ :var dict dir_v3_responses_unknown: mapping of unrecognized statuses to their count
+
+ :var dict dir_v2_direct_dl: mapping of DirStats to measurement over DirPort
+ :var dict dir_v3_direct_dl: mapping of DirStats to measurement over DirPort
+ :var dict dir_v2_direct_dl_unknown: mapping of unrecognized stats to their measurement
+ :var dict dir_v3_direct_dl_unknown: mapping of unrecognized stats to their measurement
+
+ :var dict dir_v2_tunneled_dl: mapping of DirStats to measurement over ORPort
+ :var dict dir_v3_tunneled_dl: mapping of DirStats to measurement over ORPort
+ :var dict dir_v2_tunneled_dl_unknown: mapping of unrecognized stats to their measurement
+ :var dict dir_v3_tunneled_dl_unknown: mapping of unrecognized stats to their measurement
+
+ **Bytes read/written for directory mirroring:**
+
+ :var datetime dir_read_history_end: end of the sampling interval
+ :var int dir_read_history_interval: seconds per interval
+ :var list dir_read_history_values: bytes read during each interval
+
+ :var datetime dir_write_history_end: end of the sampling interval
+ :var int dir_write_history_interval: seconds per interval
+ :var list dir_write_history_values: bytes read during each interval
+
+ **Guard Attributes:**
+
+ :var datetime entry_stats_end: end of the period when stats were gathered
+ :var int entry_stats_interval: length in seconds of the interval
+ :var dict entry_ips: mapping of locales to rounded count of unique user ips
+
+ **Exit Attributes:**
+
+ :var datetime exit_stats_end: end of the period when stats were gathered
+ :var int exit_stats_interval: length in seconds of the interval
+ :var dict exit_kibibytes_written: traffic per port (keys are ints or 'other')
+ :var dict exit_kibibytes_read: traffic per port (keys are ints or 'other')
+ :var dict exit_streams_opened: streams per port (keys are ints or 'other')
+
+ **Bridge Attributes:**
+
+ :var datetime bridge_stats_end: end of the period when stats were gathered
+ :var int bridge_stats_interval: length in seconds of the interval
+ :var dict bridge_ips: mapping of locales to rounded count of unique user ips
+ :var datetime geoip_start_time: replaced by bridge_stats_end (deprecated)
+ :var dict geoip_client_origins: replaced by bridge_ips (deprecated)
+
+ **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined
"""
def __init__(self, raw_contents, validate = True):
@@ -272,13 +276,10 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor):
validation can be disables to either improve performance or be accepting of
malformed data.
- Arguments:
- raw_contents (str) - extra-info content provided by the relay
- validate (bool) - checks the validity of the extra-info descriptor if
- True, skips these checks otherwise
+ :param str raw_contents: extra-info content provided by the relay
+ :param bool validate: checks the validity of the extra-info descriptor if True, skips these checks otherwise
- Raises:
- ValueError if the contents is malformed and validate is True
+ :raises: ValueError if the contents is malformed and validate is True
"""
stem.descriptor.Descriptor.__init__(self, raw_contents)
@@ -385,12 +386,10 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor):
Parses a series of 'keyword => (value, pgp block)' mappings and applies
them as attributes.
- Arguments:
- entries (dict) - descriptor contents to be applied
- validate (bool) - checks the validity of descriptor content if True
+ :param dict entries: descriptor contents to be applied
+ :param bool validate: checks the validity of descriptor content if True
- Raises:
- ValueError if an error occures in validation
+ :raises: ValueError if an error occures in validation
"""
for keyword, values in entries.items():
diff --git a/stem/descriptor/reader.py b/stem/descriptor/reader.py
index 1110854..acc9f8f 100644
--- a/stem/descriptor/reader.py
+++ b/stem/descriptor/reader.py
@@ -3,6 +3,8 @@ Utilities for reading descriptors from local directories and archives. This is
mostly done through the DescriptorReader class, which is an iterator for the
descriptor data in a series of destinations. For example...
+::
+
my_descriptors = [
"/tmp/server-descriptors-2012-03.tar.bz2",
"/tmp/archived_descriptors/",
@@ -15,7 +17,7 @@ descriptor data in a series of destinations. For example...
This ignores files that cannot be processed due to read errors or unparsable
content. To be notified of skipped files you can register a listener with
-register_skip_listener().
+:func:`stem.descriptor.reader.DescriptorReader.register_skip_listener`.
The DescriptorReader keeps track of the last modified timestamps for descriptor
files that it has read so it can skip unchanged files if ran again. This
@@ -24,6 +26,8 @@ DescriptorReaders. For instance, the following prints descriptors as they're
changed over the course of a minute, and picks up where it left off if ran
again...
+::
+
reader = DescriptorReader(["/tmp/descriptor_data"])
try:
@@ -43,25 +47,28 @@ again...
save_processed_files("/tmp/used_descriptors", reader.get_processed_files())
+**Module Overview:**
-load_processed_files - Loads a listing of processed files.
-save_processed_files - Saves a listing of processed files.
-
-DescriptorReader - Iterator for descriptor data on the local file system.
- |- get_processed_files - provides the listing of files that we've processed
- |- set_processed_files - sets our tracking of the files we have processed
- |- register_skip_listener - adds a listener that's notified of skipped files
- |- start - begins reading descriptor data
- |- stop - stops reading descriptor data
- |- __enter__ / __exit__ - manages the descriptor reader thread in the context
- +- __iter__ - iterates over descriptor data in unread files
+::
-FileSkipped - Base exception for a file that was skipped.
- |- AlreadyRead - We've already read a file with this last modified timestamp.
- |- ParsingFailure - Contents can't be parsed as descriptor data.
- |- UnrecognizedType - File extension indicates non-descriptor data.
- +- ReadFailed - Wraps an error that was raised while reading the file.
- +- FileMissing - File does not exist.
+ load_processed_files - Loads a listing of processed files.
+ save_processed_files - Saves a listing of processed files.
+
+ DescriptorReader - Iterator for descriptor data on the local file system.
+ |- get_processed_files - provides the listing of files that we've processed
+ |- set_processed_files - sets our tracking of the files we have processed
+ |- register_skip_listener - adds a listener that's notified of skipped files
+ |- start - begins reading descriptor data
+ |- stop - stops reading descriptor data
+ |- __enter__ / __exit__ - manages the descriptor reader thread in the context
+ +- __iter__ - iterates over descriptor data in unread files
+
+ FileSkipped - Base exception for a file that was skipped.
+ |- AlreadyRead - We've already read a file with this last modified timestamp.
+ |- ParsingFailure - Contents can't be parsed as descriptor data.
+ |- UnrecognizedType - File extension indicates non-descriptor data.
+ +- ReadFailed - Wraps an error that was raised while reading the file.
+ +- FileMissing - File does not exist.
"""
import os
@@ -119,17 +126,16 @@ class FileMissing(ReadFailed):
def load_processed_files(path):
"""
Loads a dictionary of 'path => last modified timestamp' mappings, as
- persisted by save_processed_files(), from a file.
+ persisted by :func:`stem.descriptor.reader.save_processed_files`, from a
+ file.
- Arguments:
- path (str) - location to load the processed files dictionary from
+ :param str path: location to load the processed files dictionary from
- Returns:
- dict of 'path (str) => last modified unix timestamp (int)' mappings
+ :returns: dict of 'path (str) => last modified unix timestamp (int)' mappings
- Raises:
- IOError if unable to read the file
- TypeError if unable to parse the file's contents
+ :raises:
+ * IOError if unable to read the file
+ * TypeError if unable to parse the file's contents
"""
processed_files = {}
@@ -160,13 +166,12 @@ def save_processed_files(path, processed_files):
provided by the DescriptorReader's get_processed_files() method) so that they
can be loaded later and applied to another DescriptorReader.
- Arguments:
- path (str) - location to save the processed files dictionary to
- processed_files (dict) - 'path => last modified' mappings
+ :param str path: location to save the processed files dictionary to
+ :param dict processed_files: 'path => last modified' mappings
- Raises:
- IOError if unable to write to the file
- TypeError if processed_files is of the wrong type
+ :raises:
+ * IOError if unable to write to the file
+ * TypeError if processed_files is of the wrong type
"""
# makes the parent directory if it doesn't already exist
@@ -196,15 +201,10 @@ class DescriptorReader:
handling. If you want that then use the load/save_processed_files functions
instead.
- Arguments:
- target (str, list) - path or list of paths for files or directories to be
- read from
- follow_links (bool) - determines if we'll follow symlinks when traversing
- directories
- buffer_size (int) - descriptors we'll buffer before waiting for some to
- be read, this is unbounded if zero
- persistence_path (str) - if set we will load and save processed file
- listings from this path, errors are ignored
+ :param str,list target: path or list of paths for files or directories to be read from
+ :param bool follow_links: determines if we'll follow symlinks when traversing directories
+ :param int buffer_size: descriptors we'll buffer before waiting for some to be read, this is unbounded if zero
+ :param str persistence_path: if set we will load and save processed file listings from this path, errors are ignored
"""
def __init__(self, target, follow_links = False, buffer_size = 100, persistence_path = None):
@@ -241,14 +241,14 @@ class DescriptorReader:
For each file that we have read descriptor data from this provides a
mapping of the form...
- absolute path (str) => last modified unix timestamp (int)
+ ::
+
+ absolute path (str) => last modified unix timestamp (int)
This includes entries set through the set_processed_files() method. After
each run is reset to only the files that were present during that run.
- Returns:
- dict with the absolute paths and unix timestamp for the last modified
- times of the files we have processed
+ :returns: dict with the absolute paths and unix timestamp for the last modified times of the files we have processed
"""
# make sure that we only provide back absolute paths
@@ -260,9 +260,7 @@ class DescriptorReader:
as a method for pre-populating the listing of descriptor files that we have
seen.
- Arguments:
- processed_files (dict) - mapping of absolute paths (str) to unix
- timestamps for the last modified time (int)
+ :param dict processed_files: mapping of absolute paths (str) to unix timestamps for the last modified time (int)
"""
self._processed_files = dict(processed_files)
@@ -272,12 +270,11 @@ class DescriptorReader:
Registers a listener for files that are skipped. This listener is expected
to be a functor of the form...
- my_listener(path, exception)
+ ::
+
+ my_listener(path, exception)
- Arguments:
- listener (functor) - functor to be notified of files that are skipped to
- read errors or because they couldn't be parsed as
- valid descriptor data
+ :param functor listener: functor to be notified of files that are skipped to read errors or because they couldn't be parsed as valid descriptor data
"""
self._skip_listeners.append(listener)
@@ -287,9 +284,7 @@ class DescriptorReader:
Provides the number of descriptors that are waiting to be iterated over.
This is limited to the buffer_size that we were constructed with.
- Returns:
- int for the estimated number of currently enqueued descriptors, this is
- not entirely reliable
+ :returns: int for the estimated number of currently enqueued descriptors, this is not entirely reliable
"""
return self._unreturned_descriptors.qsize()
@@ -298,8 +293,7 @@ class DescriptorReader:
"""
Starts reading our descriptor files.
- Raises:
- ValueError if we're already reading the descriptor files
+ :raises: ValueError if we're already reading the descriptor files
"""
with self._reader_thread_lock:
diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py
index b1b02e3..e19a3bd 100644
--- a/stem/descriptor/server_descriptor.py
+++ b/stem/descriptor/server_descriptor.py
@@ -3,24 +3,28 @@ Parsing for Tor server descriptors, which contains the infrequently changing
information about a Tor relay (contact information, exit policy, public keys,
etc). This information is provided from a few sources...
-- control port via 'GETINFO desc/*' queries
-- the 'cached-descriptors' file in tor's data directory
-- tor metrics, at https://metrics.torproject.org/data.html
-- directory authorities and mirrors via their DirPort
+* control port via 'GETINFO desc/*' queries
+* the 'cached-descriptors' file in tor's data directory
+* tor metrics, at https://metrics.torproject.org/data.html
+* directory authorities and mirrors via their DirPort
-parse_file - Iterates over the server descriptors in a file.
-ServerDescriptor - Tor server descriptor.
- | |- RelayDescriptor - Server descriptor for a relay.
- | | +- is_valid - checks the signature against the descriptor content
- | |
- | +- BridgeDescriptor - Scrubbed server descriptor for a bridge.
- | |- is_scrubbed - checks if our content has been properly scrubbed
- | +- get_scrubbing_issues - description of issues with our scrubbing
- |
- |- digest - calculates the digest value for our content
- |- get_unrecognized_lines - lines with unrecognized content
- |- get_annotations - dictionary of content prior to the descriptor entry
- +- get_annotation_lines - lines that provided the annotations
+**Module Overview:**
+
+::
+
+ parse_file - Iterates over the server descriptors in a file.
+ ServerDescriptor - Tor server descriptor.
+ | |- RelayDescriptor - Server descriptor for a relay.
+ | | +- is_valid - checks the signature against the descriptor content
+ | |
+ | +- BridgeDescriptor - Scrubbed server descriptor for a bridge.
+ | |- is_scrubbed - checks if our content has been properly scrubbed
+ | +- get_scrubbing_issues - description of issues with our scrubbing
+ |
+ |- digest - calculates the digest value for our content
+ |- get_unrecognized_lines - lines with unrecognized content
+ |- get_annotations - dictionary of content prior to the descriptor entry
+ +- get_annotation_lines - lines that provided the annotations
"""
import re
@@ -39,7 +43,7 @@ try:
import rsa
IS_RSA_AVAILABLE = True
except ImportError:
- log.info("Unable to import the rsa module. Because of this we'll be unable to verify descriptor integrity.")
+ log.info("Unable to import the rsa module. Because of this we'll be unable to verify descriptor signature integrity.")
IS_RSA_AVAILABLE = False
# relay descriptors must have exactly one of the following
@@ -75,17 +79,14 @@ def parse_file(descriptor_file, validate = True):
Iterates over the server descriptors in a file. This can read either relay or
bridge server descriptors.
- Arguments:
- descriptor_file (file) - file with descriptor content
- validate (bool) - checks the validity of the descriptor's content if
- True, skips these checks otherwise
+ :param file descriptor_file: file with descriptor content
+ :param bool validate: checks the validity of the descriptor's content if True, skips these checks otherwise
- Returns:
- iterator for ServerDescriptor instances in the file
+ :returns: iterator for ServerDescriptor instances in the file
- Raises:
- ValueError if the contents is malformed and validate is True
- IOError if the file can't be read
+ :raises:
+ * ValueError if the contents is malformed and validate is True
+ * IOError if the file can't be read
"""
# Handler for relay descriptors
@@ -134,48 +135,46 @@ class ServerDescriptor(stem.descriptor.Descriptor):
"""
Common parent for server descriptors.
- Attributes:
- nickname (str) - relay's nickname (*)
- fingerprint (str) - identity key fingerprint
- published (datetime) - time in GMT when this descriptor was made (*)
-
- address (str) - IPv4 address of the relay (*)
- or_port (int) - port used for relaying (*)
- socks_port (int) - (deprecated, always zero) port used as client (*)
- dir_port (int) - port used for descriptor mirroring (*)
-
- platform (str) - line with operating system and tor version
- tor_version (stem.version.Version) - version of tor
- operating_system (str) - operating system
- uptime (int) - uptime when published in seconds
- contact (str) - contact information
- exit_policy (stem.exit_policy.ExitPolicy) - stated exit policy (*)
- family (list) - nicknames or fingerprints of declared family (*)
-
- average_bandwidth (int) - averate rate it's willing to relay in bytes/s (*)
- burst_bandwidth (int) - burst rate it's willing to relay in bytes/s (*)
- observed_bandwidth (int) - estimated capacity based on usage in bytes/s (*)
-
- link_protocols (list) - link protocols supported by the relay
- circuit_protocols (list) - circuit protocols supported by the relay
- hibernating (bool) - hibernating when published (*)
- allow_single_hop_exits (bool) - flag if single hop exiting is allowed (*)
- extra_info_cache (bool) - flag if a mirror for extra-info documents (*)
- extra_info_digest (str) - hex encoded digest of our extra-info document
- hidden_service_dir (list) - hidden service descriptor versions it stores
- eventdns (bool) - (deprecated, always unset) flag for evdns backend
-
- Deprecated, moved to extra-info descriptor...
- read_history_end (datetime) - end of the sampling interval
- read_history_interval (int) - seconds per interval
- read_history_values (list) - bytes read during each interval
-
- write_history_end (datetime) - end of the sampling interval
- write_history_interval (int) - seconds per interval
- write_history_values (list) - bytes written during each interval
-
- (*) attribute is either required when we're parsed with validation or has a
- default value, others are left as None if undefined
+ :var str nickname: **\*** relay's nickname
+ :var str fingerprint: identity key fingerprint
+ :var datetime published: **\*** time in GMT when this descriptor was made
+
+ :var str address: **\*** IPv4 address of the relay
+ :var int or_port: **\*** port used for relaying
+ :var int socks_port: **\*** port used as client (deprecated, always zero)
+ :var int dir_port: **\*** port used for descriptor mirroring
+
+ :var str platform: line with operating system and tor version
+ :var stem.version.Version tor_version: version of tor
+ :var str operating_system: operating system
+ :var int uptime: uptime when published in seconds
+ :var str contact: contact information
+ :var stem.exit_policy.ExitPolicy exit_policy: **\*** stated exit policy
+ :var list family: **\*** nicknames or fingerprints of declared family
+
+ :var int average_bandwidth: **\*** averate rate it's willing to relay in bytes/s
+ :var int burst_bandwidth: **\*** burst rate it's willing to relay in bytes/s
+ :var int observed_bandwidth: **\*** estimated capacity based on usage in bytes/s
+
+ :var list link_protocols: link protocols supported by the relay
+ :var list circuit_protocols: circuit protocols supported by the relay
+ :var bool hibernating: **\*** hibernating when published
+ :var bool allow_single_hop_exits: **\*** flag if single hop exiting is allowed
+ :var bool extra_info_cache: **\*** flag if a mirror for extra-info documents
+ :var str extra_info_digest: hex encoded digest of our extra-info document
+ :var bool eventdns: flag for evdns backend (deprecated, always unset)
+
+ Deprecated, moved to extra-info descriptor...
+
+ :var datetime read_history_end: end of the sampling interval
+ :var int read_history_interval: seconds per interval
+ :var list read_history_values: bytes read during each interval
+
+ :var datetime write_history_end: end of the sampling interval
+ :var int write_history_interval: seconds per interval
+ :var list write_history_values: bytes written during each interval
+
+ **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined
"""
def __init__(self, raw_contents, validate = True, annotations = None):
@@ -188,14 +187,11 @@ class ServerDescriptor(stem.descriptor.Descriptor):
validation can be disables to either improve performance or be accepting of
malformed data.
- Arguments:
- raw_contents (str) - descriptor content provided by the relay
- validate (bool) - checks the validity of the descriptor's content if
- True, skips these checks otherwise
- annotations (list) - lines that appeared prior to the descriptor
+ :param str raw_contents: descriptor content provided by the relay
+ :param bool validate: checks the validity of the descriptor's content if True, skips these checks otherwise
+ :param list annotations: lines that appeared prior to the descriptor
- Raises:
- ValueError if the contents is malformed and validate is True
+ :raises: ValueError if the contents is malformed and validate is True
"""
stem.descriptor.Descriptor.__init__(self, raw_contents)
@@ -262,11 +258,10 @@ class ServerDescriptor(stem.descriptor.Descriptor):
server descriptor entry for this relay.
Note that network status entries exclude the padding, so you'll need to add
- a '=' to it so they'll match...
- https://en.wikipedia.org/wiki/Base64#Padding
+ a '=' to it so they'll match (`explanation
+ <https://en.wikipedia.org/wiki/Base64#Padding>`_).
- Returns:
- str with the digest value for this server descriptor
+ :returns: str with the digest value for this server descriptor
"""
raise NotImplementedError("Unsupported Operation: this should be implemented by the ServerDescriptor subclass")
@@ -279,11 +274,12 @@ class ServerDescriptor(stem.descriptor.Descriptor):
Provides content that appeard prior to the descriptor. If this comes from
the cached-descriptors file then this commonly contains content like...
+ ::
+
@downloaded-at 2012-03-18 21:18:29
@source "173.254.216.66"
- Returns:
- dict with the key/value pairs in our annotations
+ :returns: dict with the key/value pairs in our annotations
"""
if self._annotation_dict is None:
@@ -305,8 +301,7 @@ class ServerDescriptor(stem.descriptor.Descriptor):
is the same as the get_annotations() results, but with the unparsed lines
and ordering retained.
- Returns:
- list with the lines of annotation that came before this descriptor
+ :returns: list with the lines of annotation that came before this descriptor
"""
return self._annotation_lines
@@ -316,12 +311,10 @@ class ServerDescriptor(stem.descriptor.Descriptor):
Parses a series of 'keyword => (value, pgp block)' mappings and applies
them as attributes.
- Arguments:
- entries (dict) - descriptor contents to be applied
- validate (bool) - checks the validity of descriptor content if True
+ :param dict entries: descriptor contents to be applied
+ :param bool validate: checks the validity of descriptor content if True
- Raises:
- ValueError if an error occures in validation
+ :raises: ValueError if an error occures in validation
"""
for keyword, values in entries.items():
@@ -516,13 +509,11 @@ class ServerDescriptor(stem.descriptor.Descriptor):
Does a basic check that the entries conform to this descriptor type's
constraints.
- Arguments:
- entries (dict) - keyword => (value, pgp key) entries
- first_keyword (str) - keyword of the first line
- last_keyword (str) - keyword of the last line
+ :param dict entries: keyword => (value, pgp key) entries
+ :param str first_keyword: keyword of the first line
+ :param str last_keyword: keyword of the last line
- Raises:
- ValueError if an issue arises in validation
+ :raises: ValueError if an issue arises in validation
"""
required_fields = self._required_fields()
@@ -558,16 +549,13 @@ class ServerDescriptor(stem.descriptor.Descriptor):
class RelayDescriptor(ServerDescriptor):
"""
- Server descriptor, as specified in...
- https://gitweb.torproject.org/torspec.git/blob/HEAD:/dir-spec.txt
+ Server descriptor (`specification <https://gitweb.torproject.org/torspec.git/blob/HEAD:/dir-spec.txt>`_)
- Attributes:
- onion_key (str) - key used to encrypt EXTEND cells (*)
- signing_key (str) - relay's long-term identity key (*)
- signature (str) - signature for this descriptor (*)
+ :var str onion_key: **\*** key used to encrypt EXTEND cells
+ :var str signing_key: **\*** relay's long-term identity key
+ :var str signature: **\*** signature for this descriptor
- (*) attribute is either required when we're parsed with validation or has a
- default value, others are left as None if undefined
+ **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined
"""
def __init__(self, raw_contents, validate = True, annotations = None):
@@ -593,8 +581,7 @@ class RelayDescriptor(ServerDescriptor):
"""
Validates that our content matches our signature.
- Returns:
- True if our signature matches our content, False otherwise
+ :returns: True if our signature matches our content, False otherwise
"""
raise NotImplementedError # TODO: finish implementing
@@ -668,13 +655,9 @@ class RelayDescriptor(ServerDescriptor):
class BridgeDescriptor(ServerDescriptor):
"""
- Bridge descriptor, as specified in...
- https://metrics.torproject.org/formats.html#bridgedesc
+ Bridge descriptor (`specification <https://metrics.torproject.org/formats.html#bridgedesc>`_)
- Attributes:
- address_alt (list) - alternative for our address/or_port attributes, each
- entry is a tuple of the form...
- (address (str), port (int), is_ipv6 (bool))
+ :var list address_alt: alternative for our address/or_port attributes, each entry is a tuple of the form ``(address (str), port (int), is_ipv6 (bool))``
"""
def __init__(self, raw_contents, validate = True, annotations = None):
@@ -737,8 +720,7 @@ class BridgeDescriptor(ServerDescriptor):
descriptor specification. Validation is a moving target so this may not
be fully up to date.
- Returns:
- True if we're scrubbed, False otherwise
+ :returns: True if we're scrubbed, False otherwise
"""
return self.get_scrubbing_issues() == []
@@ -747,9 +729,7 @@ class BridgeDescriptor(ServerDescriptor):
"""
Provides issues with our scrubbing.
- Returns:
- list of strings which describe issues we have with our scrubbing, this
- list is empty if we're properly scrubbed
+ :returns: list of strings which describe issues we have with our scrubbing, this list is empty if we're properly scrubbed
"""
if self._scrubbing_issues == None:
More information about the tor-commits
mailing list