January 2014 - tor-commits - lists.torproject.org

[bridgedb/master] Change cfg→config in Main.startup() to avoid name conflict.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit 01573a3e2f3581901a7548cbdc050de2d83aac41 Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Nov 15 14:33:51 2013 +0000 Change cfg→config in Main.startup() to avoid name conflict. Otherwise, the cfg variable in Main.startup().reload() would be overriding the parent namespace in potentially bad/buggy ways. --- lib/bridgedb/Main.py | 45 +++++++++++++++++++++------------------------ 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 42f361d..0b1512a 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -321,7 +321,7 @@ def startup(options, rundir, configFile): # and another in the directory it changes into. os.chdir(rundir) - cfg = loadConfig(configFile) + config = loadConfig(configFile) # Set up logging as early as possible. We cannot import from the bridgedb # package any of our modules which import :mod:`logging` and start using @@ -329,17 +329,17 @@ def startup(options, rundir, configFile): # default handler that logs to the console will be created by the imported # module, and all further calls to :func:`logging.basicConfig` will be # ignored. - configureLogging(cfg) + configureLogging(config) if options['dump-bridges'] or (options.subCommand is not None): - runSubcommand(options, cfg) + runSubcommand(options, config) # Write the pidfile only after any options.subCommands are run (because # these exit when they are finished). Otherwise, if there is a subcommand, # the real PIDFILE would get overwritten with the PID of the temporary # bridgedb process running the subcommand. - if cfg.PIDFILE: - f = open(cfg.PIDFILE, 'w') + if config.PIDFILE: + f = open(config.PIDFILE, 'w') f.write("%s\n" % os.getpid()) f.close() @@ -350,15 +350,15 @@ def startup(options, rundir, configFile): from bridgedb import HTTPServer # Load the master key, or create a new one. - key = bridgedb.crypto.getKey(cfg.MASTER_KEY_FILE) + key = crypto.getKey(config.MASTER_KEY_FILE) # Initialize our DB file. - db = bridgedb.Storage.Database(cfg.DB_FILE + ".sqlite", cfg.DB_FILE) + db = bridgedb.Storage.Database(config.DB_FILE + ".sqlite", config.DB_FILE) bridgedb.Storage.setGlobalDB(db) # Get a proxy list. proxyList = ProxyCategory() - proxyList.replaceProxyList(loadProxyList(cfg)) + proxyList.replaceProxyList(loadProxyList(config)) # Create a BridgeSplitter to assign the bridges to the different # distributors. @@ -366,8 +366,8 @@ def startup(options, rundir, configFile): logging.debug("Created splitter: %r" % splitter) # Create ring parameters. - forcePorts = getattr(cfg, "FORCE_PORTS") - forceFlags = getattr(cfg, "FORCE_FLAGS") + forcePorts = getattr(config, "FORCE_PORTS") + forceFlags = getattr(config, "FORCE_FLAGS") if not forcePorts: forcePorts = [] if not forceFlags: forceFlags = [] ringParams=Bridges.BridgeRingParameters(needPorts=forcePorts, @@ -375,7 +375,7 @@ def startup(options, rundir, configFile): emailDistributor = ipDistributor = None # As appropriate, create an IP-based distributor. - if cfg.HTTPS_DIST and cfg.HTTPS_SHARE: + if config.HTTPS_DIST and config.HTTPS_SHARE: logging.debug("Setting up HTTPS Distributor...") categories = [] if proxyList.ipset: @@ -385,38 +385,35 @@ def startup(options, rundir, configFile): ipDistributor = Dist.IPBasedDistributor( Dist.uniformMap, - cfg.N_IP_CLUSTERS, + config.N_IP_CLUSTERS, Bridges.get_hmac(key, "HTTPS-IP-Dist-Key"), categories, answerParameters=ringParams) - splitter.addRing(ipDistributor, "https", cfg.HTTPS_SHARE) + splitter.addRing(ipDistributor, "https", config.HTTPS_SHARE) #webSchedule = Time.IntervalSchedule("day", 2) webSchedule = Time.NoSchedule() # As appropriate, create an email-based distributor. - if cfg.EMAIL_DIST and cfg.EMAIL_SHARE: - for d in cfg.EMAIL_DOMAINS: - cfg.EMAIL_DOMAIN_MAP[d] = d - logging.debug("New email domain map: '%s'" % cfg.EMAIL_DOMAIN_MAP) + if config.EMAIL_DIST and config.EMAIL_SHARE: logging.debug("Setting up Email Distributor...") emailDistributor = Dist.EmailBasedDistributor( Bridges.get_hmac(key, "Email-Dist-Key"), - cfg.EMAIL_DOMAIN_MAP.copy(), - cfg.EMAIL_DOMAIN_RULES.copy(), + config.EMAIL_DOMAIN_MAP.copy(), + config.EMAIL_DOMAIN_RULES.copy(), answerParameters=ringParams) - splitter.addRing(emailDistributor, "email", cfg.EMAIL_SHARE) + splitter.addRing(emailDistributor, "email", config.EMAIL_SHARE) #emailSchedule = Time.IntervalSchedule("day", 1) emailSchedule = Time.NoSchedule() # As appropriate, tell the splitter to leave some bridges unallocated. - if cfg.RESERVED_SHARE: + if config.RESERVED_SHARE: splitter.addRing(Bridges.UnallocatedHolder(), "unallocated", - cfg.RESERVED_SHARE) + config.RESERVED_SHARE) # Add pseudo distributors to splitter - for p in cfg.FILE_BUCKETS.keys(): - splitter.addPseudoRing(p) + for pseudoRing in config.FILE_BUCKETS.keys(): + splitter.addPseudoRing(pseudoRing) def reload(*args): """Reload settings, proxy lists, and bridges.

1 0

[bridgedb/master] Update startup.reload() docstring.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit aafc3bfaccc69455894a3938e03c53d53c7964f7 Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Nov 15 15:18:23 2013 +0000 Update startup.reload() docstring. --- lib/bridgedb/Main.py | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 29678e6..f22e568 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -427,34 +427,31 @@ def startup(options, rundir, configFile): def reload(): """Reload settings, proxy lists, and bridges. - The contents of the config file should be compiled (it's roughly 20-30 - times faster to use the ``compile`` builtin on a string before - ``exec``ing it) first, and then ``exec``ed -- not ``execfile``! -- in - order to get the contents of the config file to exist within the scope - of the configuration object. Otherwise, Python *will* default to - placing them directly within the ``globals()`` scope. + State should be saved before calling this method, and will be saved + again at the end of it. - For a more detailed explanation, see http://stackoverflow.com/q/17470193 - and http://lucumr.pocoo.org/2011/2/1/exec-in-python/ + The internal variables, ``cfg``, ``splitter``, ``proxyList``, + ``ipDistributor``, and ``emailDistributor`` are all taken from a + :class:`~bridgedb.persistent.State` instance, which has been saved to + a statefile with :meth:`bridgedb.persistent.State.save`. :type cfg: :class:`Conf` - :param cfg: The current configuration, including any in-memory + :ivar cfg: The current configuration, including any in-memory settings (i.e. settings whose values were not obtained from the config file, but were set via a function somewhere) - :type options: :class:`twisted.python.usage.Options` - :param options: Any commandline options. - - :param splitter: XXX - + :type splitter: A :class:`bridgedb.Bridges.BridgeHolder` + :ivar splitter: A class which takes an HMAC key and splits bridges + into their hashring assignments. :type proxyList: :class:`ProxyCategory` - :param proxyList: The container for the IP addresses of any currently + :ivar proxyList: The container for the IP addresses of any currently known open proxies. - - :param IPDistributor: A :class:`Dist.IPBasedDistributor`. - :param emailDistributor: A :class:`Dist.EmailDistributor`. - :param dict tasks: A dictionary of {'name': Task}, where Task is some + :ivar ipDistributor: A :class:`Dist.IPBasedDistributor`. + :ivar emailDistributor: A :class:`Dist.EmailBasedDistributor`. + :ivar dict tasks: A dictionary of ``{name: task}``, where name is a + string to associate with the ``task``, and ``task`` is some scheduled event, repetitive or otherwise, for the :class:`reactor - <twisted.internet.epollreactor.EPollReactor>` + <twisted.internet.epollreactor.EPollReactor>`. See the classes + within the :mod:`twisted.internet.tasks` module. """ logging.debug("Caught SIGHUP") logging.info("Reloading...")

1 0

[bridgedb/master] Start using bridgedb.persistent.State in bridgedb.Main.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit fb972d8a425ceee76a388202196642e0479fb3f5 Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Nov 15 15:16:09 2013 +0000 Start using bridgedb.persistent.State in bridgedb.Main. --- lib/bridgedb/Main.py | 53 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 3a22914..a2503ee 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -344,6 +344,11 @@ def startup(options, rundir, configFile): pidfile.write("%s\n" % os.getpid()) pidfile.flush() + from bridgedb import persistent + + state = persistent.State(config=config) + state.RUN_IN_DIR = rundir + from bridgedb import EmailServer from bridgedb import HTTPServer @@ -352,6 +357,7 @@ def startup(options, rundir, configFile): # Initialize our DB file. db = bridgedb.Storage.Database(config.DB_FILE + ".sqlite", config.DB_FILE) + # TODO: move setGlobalDB to bridgedb.persistent.State class bridgedb.Storage.setGlobalDB(db) # Get a proxy list. @@ -413,6 +419,11 @@ def startup(options, rundir, configFile): for pseudoRing in config.FILE_BUCKETS.keys(): splitter.addPseudoRing(pseudoRing) + # Save our state + state.proxyList = proxyList + state.key = key + state.save() + def reload(): """Reload settings, proxy lists, and bridges. @@ -448,17 +459,26 @@ def startup(options, rundir, configFile): logging.debug("Caught SIGHUP") logging.info("Reloading...") - cfg = loadConfig(options, cfg) + logging.info("Loading saved state...") + state = persistent.load() + cfg = loadConfig(state.CONFIG_FILE, state.config) + logging.info("Updating any changed settings...") + state.useChangedSettings(cfg) - # update loglevel on (re)load - level = getattr(cfg, 'LOGLEVEL', 'WARNING') + level = getattr(state, 'LOGLEVEL', 'WARNING') + logging.info("Updating log level to: '%s'" % level) level = getattr(logging, level) logging.getLogger().setLevel(level) - load(cfg, splitter, clear=True) - proxyList.replaceProxyList(loadProxyList(cfg)) - logging.info("%d bridges loaded", len(splitter)) - if emailDistributor: + logging.debug("Saving state again before reparsing descriptors...") + state.save() + + state = persistent.load() + logging.info("Bridges loaded: %d" % len(splitter)) + logging.debug("Replacing the list of open proxies...") + state.proxyList.replaceProxyList(loadProxyList(cfg)) + + if emailDistributor is not None: logging.debug("Prepopulating email distributor hashrings...") emailDistributor.prepopulateRings() # create default rings logging.info("Bridges allotted for email distribution: %d" @@ -477,15 +497,18 @@ def startup(options, rundir, configFile): # Dump bridge pool assignments to disk. try: logging.debug("Dumping pool assignments to file: '%s'" - % cfg.ASSIGNMENTS_FILE) - f = open(cfg.ASSIGNMENTS_FILE, 'a') - f.write("bridge-pool-assignment %s\n" % - time.strftime("%Y-%m-%d %H:%M:%S")) - splitter.dumpAssignments(f) - f.flush() - f.close() + % state.ASSIGNMENTS_FILE) + fh = open(state.ASSIGNMENTS_FILE, 'a') + fh.write("bridge-pool-assignment %s\n" % + time.strftime("%Y-%m-%d %H:%M:%S")) + splitter.dumpAssignments(fh) + fh.flush() + fh.close() except IOError: - logging.info("I/O error while writing assignments") + logging.info("I/O error while writing assignments to: '%s'" + % state.ASSIGNMENTS_FILE) + + state.save() global _reloadFn _reloadFn = reload

1 0

[bridgedb/master] Also change cfg→config after def reload() in startup().
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit ec76a0ac71da3421dfcb8fd8f7708198ff89ee6d Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Nov 15 15:17:24 2013 +0000 Also change cfg→config after def reload() in startup(). --- lib/bridgedb/Main.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index a2503ee..29678e6 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -517,13 +517,11 @@ def startup(options, rundir, configFile): # And actually load it to start parsing. reload() - # Configure HTTP and/or HTTPS servers. - if cfg.HTTPS_DIST and cfg.HTTPS_SHARE: - HTTPServer.addWebServer(cfg, ipDistributor, webSchedule) - - # Configure Email servers. - if cfg.EMAIL_DIST and cfg.EMAIL_SHARE: - EmailServer.addSMTPServer(cfg, emailDistributor, emailSchedule) + # Configure all servers: + if config.HTTPS_DIST and config.HTTPS_SHARE: + HTTPServer.addWebServer(config, ipDistributor, webSchedule) + if config.EMAIL_DIST and config.EMAIL_SHARE: + EmailServer.addSMTPServer(config, emailDistributor, emailSchedule) # Actually run the servers. try: @@ -531,8 +529,8 @@ def startup(options, rundir, configFile): reactor.run() finally: db.close() - if cfg.PIDFILE: - os.unlink(cfg.PIDFILE) + if config.PIDFILE: + os.unlink(config.PIDFILE) sys.exit() def runSubcommand(options, config):

1 0

[bridgedb/master] Remove unnecessary config parameter coercion.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit 849b08a014c82deeaac67ef168c33ed51542ad2c Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Nov 15 15:20:19 2013 +0000 Remove unnecessary config parameter coercion. FORCE_PORTS and FORCE_FLAGS are already set to lists. We don't need to coerce them if they weren't set anymore. --- lib/bridgedb/Main.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index ad51631..7dfd41f 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -370,14 +370,11 @@ def startup(options, rundir, configFile): logging.debug("Created splitter: %r" % splitter) # Create ring parameters. - forcePorts = getattr(config, "FORCE_PORTS") - forceFlags = getattr(config, "FORCE_FLAGS") - if not forcePorts: forcePorts = [] - if not forceFlags: forceFlags = [] - ringParams=Bridges.BridgeRingParameters(needPorts=forcePorts, - needFlags=forceFlags) + ringParams = Bridges.BridgeRingParameters(needPorts=config.FORCE_PORTS, + needFlags=config.FORCE_FLAGS) emailDistributor = ipDistributor = None + # As appropriate, create an IP-based distributor. if config.HTTPS_DIST and config.HTTPS_SHARE: logging.debug("Setting up HTTPS Distributor...")

1 0

[bridgedb/master] Give a warning if no emailDistributor was created.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit 80c14e831fa2208a21a1f7a262eefebe938d93ab Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Nov 15 15:18:55 2013 +0000 Give a warning if no emailDistributor was created. --- lib/bridgedb/Main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index f22e568..f62b54d 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -480,8 +480,10 @@ def startup(options, rundir, configFile): emailDistributor.prepopulateRings() # create default rings logging.info("Bridges allotted for email distribution: %d" % len(emailDistributor.splitter)) + else: + logging.warn("No email distributor created!") - if ipDistributor: + if ipDistributor is not None: logging.debug("Prepopulating HTTPS distributor hashrings...") ipDistributor.prepopulateRings() # create default rings logging.info("Bridges allotted for web distribution: %d"

1 0

[bridgedb/master] Handle KeyboardInterrupts in Main.startup() loop.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit f075404fcd6c0209099dda50206716700b125fd9 Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Nov 15 15:19:35 2013 +0000 Handle KeyboardInterrupts in Main.startup() loop. --- lib/bridgedb/Main.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index f62b54d..ad51631 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -526,10 +526,14 @@ def startup(options, rundir, configFile): try: logging.info("Starting reactors.") reactor.run() + except KeyboardInterrupt: + logging.fatal("Received keyboard interrupt. Shutting down...") finally: + logging.info("Closing databases...") db.close() if config.PIDFILE: os.unlink(config.PIDFILE) + logging.info("Exiting...") sys.exit() def runSubcommand(options, config):

1 0

[bridgedb/master] Fix a bug due to parantheses instead of square brackets.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit a1206cd5f56bf21375015f98e064c4e62f8edace Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Nov 15 15:21:40 2013 +0000 Fix a bug due to parantheses instead of square brackets. --- lib/bridgedb/Main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 7dfd41f..8ef3fee 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -549,7 +549,7 @@ def runSubcommand(options, config): # mentioned above with the EmailServer and HTTPServer. from bridgedb import runner - if options('dump-bridges'): + if options['dump-bridges']: runner.doDumpBridges(config) if options.subCommand is not None:

1 0

[bridgedb/master] Add docstrings for Dist.IPBasedDistributor.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit d14f886e7ae640d5c22e02ebd359441980ed394d Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Nov 15 15:23:41 2013 +0000 Add docstrings for Dist.IPBasedDistributor. --- lib/bridgedb/Dist.py | 56 ++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 43 insertions(+), 13 deletions(-) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index aebb1f8..e3481f0 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -42,26 +42,56 @@ def getNumBridgesPerAnswer(ring, max_bridges_per_answer=3): return n_bridges_per_answer class IPBasedDistributor(bridgedb.Bridges.BridgeHolder): - """Object that hands out bridges based on the IP address of an incoming - request and the current time period. + """A Distributor that hands out bridges based on the IP address of an + incoming request and the current time period. + + :ivar areaOrderHmac: An HMAC function used to order areas within rings. + :ivar areaClusterHmac: An HMAC function used to assign areas to rings. + :ivar list rings: A list of :class:`bridgedb.Bridges.BridgeHolder` + hashrings, one for each area in the ``areaMapper``. Every inserted + bridge will go into one of these rings, and every area is associated + with one. + :ivar categories: DOCDOC See :param:`ipCategories`. + :type splitter: :class:`bridgedb.Bridges.FixedBridgeSplitter` + :ivar splitter: A hashring that assigns bridges to subrings with fixed + proportions. Used to assign bridges into the subrings of this + distributor. """ - ## Fields: - ## areaMapper -- a function that maps an IP address to a string such - ## that addresses mapping to the same string are in the same "area". - ## rings -- a list of BridgeRing objects. Every bridge goes into one - ## of these rings, and every area is associated with one. - ## splitter -- a FixedBridgeSplitter to assign bridges into the - ## rings of this distributor. - ## areaOrderHmac -- an hmac function used to order areas within rings. - ## areaClusterHmac -- an hmac function used to assign areas to rings. - def __init__(self, areaMapper, nClusters, key, ipCategories=(), answerParameters=None): + + def __init__(self, areaMapper, nClusters, key, + ipCategories=None, answerParameters=None): + """Create a Distributor that decides which bridges to distribute based + upon the client's IP address and the current time. + + :type areaMapper: callable + :param areaMapper: A function that maps IP addresses arbitrarily to + strings, such that addresses which map to identical strings are + considered to be in the same "area" (for some arbitrary definition + of "area"). See :func:`bridgedb.Dist.uniformMap` for an example. + :param integer nClusters: The number of clusters to group IP addresses + into. Note that if PROXY_LIST_FILES is set in bridgedb.conf, then + the actual number of clusters is one higher than ``nClusters``, + because the set of known open proxies constitutes its own + category. + DOCDOC What exactly does a cluster *do*? + :param bytearray key: The master HMAC key for this distributor. All + added bridges are HMACed with this key in order to place them into + the hashrings. + :type ipCategories: iterable or None + :param ipCategories: DOCDOC + :type answerParameters: :class:`bridgedb.Bridges.BridgeRingParameters` + :param answerParameters: A mechanism for ensuring that the set of + bridges that this distributor answers a client with fit certain + parameters, i.e. that an answer has "at least two obfsproxy + bridges" or "at least one bridge on port 443", etc. + """ self.areaMapper = areaMapper self.nClusters = nClusters self.answerParameters = answerParameters self.rings = [] - self.categories = [] #DOCDOC + self.categories = [] for c in ipCategories: self.categories.append(c)

1 0

[bridgedb/master] Add bridgedb.parse.addr module for parsing IPs and ports.
by isis＠torproject.org 12 Jan '14

12 Jan '14

commit 1f111e56428e89b731d832e20b7c5876af736943 Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Nov 15 15:26:32 2013 +0000 Add bridgedb.parse.addr module for parsing IPs and ports. * MOVE class bridgedb.Bridges.PortList to bridgedb.parse.addr. --- lib/bridgedb/Bridges.py | 64 -------------- lib/bridgedb/parse/addr.py | 208 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 208 insertions(+), 64 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index a11cd48..723d3c9 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -361,54 +361,6 @@ def parseDescFile(f, bridge_purpose='bridge'): yield b nickname = ip = orport = fingerprint = purpose = None -class PortList: - """ container class for port ranges - """ - - def __init__(self, *args, **kwargs): - self.ports = set() - self.add(*args) - - def _sanitycheck(self, val): - #XXX: if debug=False this is disabled. bad! - assert type(val) is int - assert(0 < val <= 65535) - - def __contains__(self, val1): - return val1 in self.ports - - def add(self, *args): - for arg in args: - try: - if type(arg) is str: - ports = set([int(p) for p in arg.split(',')][:PORTSPEC_LEN]) - [self._sanitycheck(p) for p in ports] - self.ports.update(ports) - if type(arg) is int: - self._sanitycheck(arg) - self.ports.update([arg]) - if type(arg) is PortList: - self.add(list(arg.ports)) - except AssertionError: raise ValueError - except ValueError: raise - - def __iter__(self): - return self.ports.__iter__() - - def __str__(self): - s = "" - for p in self.ports: - s += "".join(",%s"%p) - return s.lstrip(",") - - def __repr__(self): - return "PortList('%s')" % self.__str__() - - def __len__(self): - return len(self.ports) - - def __getitem__(self, x): - return list(self.ports)[x] class ParseORAddressError(Exception): def __init__(self): @@ -418,22 +370,6 @@ class ParseORAddressError(Exception): re_ipv6 = re.compile("\[([a-fA-F0-9:]+)\]:(.*$)") re_ipv4 = re.compile("((?:\d{1,3}\.?){4}):(.*$)") -def parseORAddressLine(line): - address = None - portlist = None - # try regexp to discover ip version - for regex in [re_ipv4, re_ipv6]: - m = regex.match(line) - if m: - # get an address and portspec, or raise ParseError - try: - address = ipaddr.IPAddress(m.group(1)) - portlist = PortList(m.group(2)) - except (IndexError, ValueError): raise ParseORAddressError - - # return a valid address, portlist or raise ParseORAddressError - if address and portlist and len(portlist): return address,portlist - raise ParseORAddressError class PluggableTransport: """ diff --git a/lib/bridgedb/parse/addr.py b/lib/bridgedb/parse/addr.py new file mode 100644 index 0000000..6e5daa6 --- /dev/null +++ b/lib/bridgedb/parse/addr.py @@ -0,0 +1,208 @@ +# -*- coding: utf-8 -*- +# +# This file is part of BridgeDB, a Tor bridge distribution system. +# +# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis(a)torproject.org> +# please also see AUTHORS file +# :copyright: (c) 2013 Isis Lovecruft +# (c) 2007-2013, The Tor Project, Inc. +# (c) 2007-2013, all entities within the AUTHORS file +# :license: 3-clause BSD, see included LICENSE for information + +"""Utilities for parsing IP addresses. + +** Module Overview: ** + +.. + parse + ||_ parse.addr + | |_ isIPAddress - Check if an arbitrary string is an IP address. + | |_ isIPv4 - Check if an arbitrary string is an IPv4 address. + | |_ isIPv6 - Check if an arbitrary string is an IPv6 address. + | \_ isValidIP - Check that an IP address is valid. + | + |__ :mod:`bridgedbparse.headers` + |__ :mod:`bridgedb.parse.options` + \__ :mod:`bridgedb.parse.versions` +""" + +import logging + +import ipaddr + + +class InvalidPort(ValueError): + """Raised when a given port number is invalid.""" + + +def isIPAddress(ip, compressed=True): + """Check if an arbitrary string is an IP address, and that it's valid. + + :type ip: basestring or int + :param ip: The IP address to check. + :param boolean compressed: If True, return a string representing the + compressed form of the address. Otherwise, return an + :class:`ipaddr.IPAddress` instance. + :rtype: A :class:`ipaddr.IPAddress`, or a string, or False + :returns: The IP, as a string or a class, if it passed the + checks. Otherwise, returns False. + """ + try: + ip = ipaddr.IPAddress(ip) + except ValueError: + return False + except Exception as error: + logging.exception(error) + else: + if isValidIP(ip): + if compressed: + return ip.compressed + else: + return ip + return False + +def _isIPv(version, ip): + """Check if an address is a certain ``version``, either IPv4 or IPv6. + + :param integer version: The IPv[4|6] version to check; must be either + ``4`` or ``6``. + :type ip: basestring or int + :param ip: The IP address to check. + :rtype: boolean + :returns: True if the address is an IPv4 address. + """ + ip = isIPAddress(ip, compressed=False) + if ip and (ip.version == version): + return True + return False + +def isIPv4(ip): + """Check if an address is IPv4. + + :type ip: basestring or int + :param ip: The IP address to check. + :rtype: boolean + :returns: True if the address is an IPv4 address. + """ + return _isIPv(4, ip) + +def isIPv6(ip): + """Check if an address is IPv6. + + :type ip: basestring or int + :param ip: The IP address to check. + :rtype: boolean + :returns: True if the address is an IPv6 address. + """ + return _isIPv(6, ip) + +def isValidIP(ipaddress): + """Check that an IP (v4 or v6) is public and not reserved. + + The IP address, ``ip``, must not be any of the following: + * A link-local address, such as ``169.254.0.0/16`` or ``fe80::/64``. + * The address of a loopback interface, i.e. ``127.0.0.1`` or ``::1``. + * A multicast address, for example, ``255.255.255.0``. + * An unspecified address, for example ``0.0.0.0/32`` in IPv4 or + ``::/128`` in IPv6. + * A default route address, for example ``0.0.0.0/0`` or ``::/0``. + * Any other address within a private networks, such as the IANA + reserved Shared Address Space, defined in RFC6598_ as + ``100.64.0.0/10``. + + If it is an IPv4 address, it also must not be: + * A reserved address vis-á-vis RFC1918_ + + If it is an IPv6 address, it also must not be: + * A "site-local", or Unique Local Address (ULA_), address vis-á-vis + RFC4193_ (i.e. within the ``fc00::/7`` netblock) + + .. _RFC6598: https://tools.ietf.org/htmłrfc6598 + .. _RFC1918: https://tools.ietf.org/html/rfc1918 + .. _ULA: https://en.wikipedia.org/wiki/Unique_local_address + .. _RFC4193: https://tools.ietf.org/html/rfc4193 + + :type ipaddress: An :class:`ipaddr.IPAddress`, + :class:`ipaddr.IPv4Address`, or :class:`ipaddr.IPv6Address`. + :param ipaddress: An IPAddress class. + :rtype: boolean + :returns: True if the address passes the checks, False otherwise. + """ + if not (ipaddress.is_link_local or ipaddress.is_loopback + or ipaddress.is_multicast or ipaddress.is_private + or ipaddress.is_unspecified): + if (ipaddress.version == 6) and (not ipaddress.is_site_local): + return True + elif (ipaddress.version == 4) and (not ipaddress.is_reserved): + return True + return False + + +class PortList(object): + """ container class for port ranges + """ + + #: The maximum number of allowed ports per IP address. + PORTSPEC_LEN = 16 + + def __init__(self, *args, **kwargs): + self.ports = set() + self.add(*args) + + def _sanitycheck(self, port): + if (not isinstance(port, int)) or not (0 < port <= 65535): + raise InvalidPort("%s is not a valid port number!" % port) + + def __contains__(self, port): + return port in self.ports + + def add(self, *args): + """Add a port (or ports) to this PortList.""" + for arg in args: + portlist = [] + try: + if isinstance(arg, basestring): + ports = set([int(p) + for p in arg.split(',')][:self.PORTSPEC_LEN]) + portlist.extend([p for p in ports]) + if isinstance(arg, int): + portlist.extend(arg) + if isinstance(arg, PortList): + self.add(list(arg.ports)) + except ValueError: + raise InvalidPort("%s is not a valid port number!" % arg) + except InvalidPort: + raise + + [self._sanitycheck(port) for port in portlist] + self.ports.update(portlist) + + def __iter__(self): + """Iterate through all ports in this PortList.""" + return self.ports.__iter__() + + def __str__(self): + """Returns a pretty string representation of this PortList.""" + ret = [] + for port in self.ports: + ret.append(',%s' % port) + ret = ''.join([piece for piece in ret]) + return ret.lstrip(",") + + def __repr__(self): + """Returns a raw depiction of this PortList.""" + return "PortList('%s')" % self.__str__() + + def __len__(self): + """Returns the total number of ports in this PortList.""" + return len(self.ports) + + def __getitem__(self, x): + """Get a port if it is in this PortList. + + :raises: ValueError if ``x`` isn't in this PortList. + :rtype: integer + :returns: The port ``x``, if it is in this PortList. + """ + portlist = list(self.ports) + return portlist[portlist.index(x)]

1 0