commit eef1f3f1975881f08090b6825369126f10c77031 Author: ilv <ilv@users.noreply.github.com> Date: Mon Jun 2 00:42:00 2014 -0400 Created the basis of GetTor's Core class. Modified the proposed design to read links from files instead of generating on demand. --- spec/design/core.txt | 133 +++++++++++++++++++++++---------------- src/core_demo.py | 19 ++++++ src/gettor.py | 145 +++++++++++++++++++++++++++++++++++++++++++ src/gettor.pyc | Bin 0 -> 3934 bytes src/providers/dropbox.links | 5 ++ src/providers/gdrive.links | 5 ++ src/providers/mirrors.links | 5 ++ 7 files changed, 257 insertions(+), 55 deletions(-) diff --git a/spec/design/core.txt b/spec/design/core.txt index ecb6951..f90c6f6 100644 --- a/spec/design/core.txt +++ b/spec/design/core.txt @@ -1,8 +1,12 @@ - Google Summer of Code 2014 GetTor Revamp - Code module + Google Summer of Code 2014 GetTor Revamp - Core module Author: Israel Leiva - <israel.leiva@usach.cl> - Last update: 2014-05-16 - Version: 0.01 - Changes: First version + Last update: 2014-06-01 + Version: 0.02 + Changes: Combine official mirrors with providers links (as another provider). + Eliminated on demand link generation. Now it reads from files. + Modified description according to PEP-8. + First version. + 1. Preface @@ -18,90 +22,109 @@ The main functionalities the core module should provide are: - * Receive a request with OS version, architecture, bundle - language, and respond with the respective links. - * Generate links, per request or at demand, depending on if the - former is accepted as part of the new design. + * Receive a request with OS version, bundle language, and respond + with the respective links. + * Read links from providers files. * Log anonymous transactions. 3. Design The new design should consist of the following files, directories and - functions: + methods: * core.conf: Configuration values, e.g. base directory. - * providers/: Directory for providers configuration. + * providers/: Directory for generated links. - ----- providersList.txt: list of valid providers. - ----- provider1.conf: configuration for provider1. - ----- provider2.conf: configuration for provider2. + ----- provider1.links: links from provider1. + ----- provider2.links: links from provider2. + ----- mirrors.links: links of official mirrors. - All this data is added manually. - - * mirrors.txt: Contains official mirrors. One per line. Added manually. + All this data is generated automatically. * logs/: Directory for logs. Added automatically. - ----- yyyy-mm-dd.log: daily log of requests. + ----- core_yyyy-mm-dd.log: daily log of requests. - * lib/gettor/core.py: Core module of GetTor. + * Core module of GetTor. - getLinks(os_version, locale) - Returns links for os_version (in both archs) in locale language. - This will read the providers list and call __generateLinks() - for each one of them, plus calling to __getMirrors(). - - Example: getLinks('OSX', 'en') + get_links(operating_system, locale) + Public method to obtain the links. Returns links for + operating_system in locale language. It checks if the operating + system and locale are supported and then calls the private + method _get_links() + - __generateLinks(options, provider) - Generate links for a specific provider according to the options - received (os_version, locale). This will try to import the - provider module and call the uploadBundle function. + Example: get_links('linux', 'en') - Example (within the module): __generateLinks(options, 'dropbox') + _get_links(operating_system, locale) + Gets the links for a specific operating system and locale + according to the options received. It reads all the .links + files inside the providers directory. Each one of these files + contains the provider's name in the first line and all the + links after. The format of these files is as following: + + PROVIDER NAME + operating_system locale link package_signature key_fingerprint + + Example: + + Dropbox + linux en https://foo.bar https://foo.bar.asc 111-222-333-444 + osx es https://bar.baz https://bar.baz.asc 555-666-777-888 + + The key fingerprint is joined by '-' characters for convenience, + but it's sent back to the service module separated by spaces. + + The official mirrors are considered as just another provider. - __getMirrors() - Obtains mirrors from mirrors.txt. - __logRequest(options) + _log_request(operating_system, locale) Log information about the request for future stats (e.g. which - OS from which service is the most required). + OS from which service is the most required). All logging + should be done with the logging module. - * lib/gettor/providers/provider.py: There should be one module per - provider with the uploadBundle public function. There should be - at least three modules at the end of GSoC: dropbox.py, drive.py, - github.py - - uploadBundle(options) - Calls the provider internal functions to upload the required - bundle according to the options received. This internal - functions will depend solely on the API requirements from - the provider. + * Providers: There should be one module/script per provider in charge + of generating the .links file with the proper format, including + the official mirrors. + 4. Roadmap An example of how the core module work: a. The SMTP service receives a request. - b. The SMTP calls getLinks() with the options sent by the user. - c. getLinks() calls to __generateLinks() and then to __getMirrors() - d. getLinks() constructs a message with the information obtained. - e. getLinks() calls to __logRequest(). - f. getLinks() returns the message previously constructed. + b. The SMTP calls core.get_links() with the options sent by the user. + c. get_links() calls to _log_request(). + d. get_links() calls to _get_links(). + e. get_links() constructs a message with the information obtained. + f. get_links() returns the message previously constructed. g. The SMTP service creates a message with the links obtained and send it to the user. 5. Discussion -5.1 Cache +5.1 On demand generation + + Now we consider links generation over a period of time, say, once a week. + Generating links on demand don't seem to be necessary and it would + certainly be resource consuming. The core module doesn't care about + how links are generated, it just assumes they exist. Is there any case + where on demand link generation should be considered? - The above design was thought for per request links generation. Another - way of doing this would be to maintain a cache of generated links and - call __generateLinks() depending on the cache last modified time. - Reading links from this cache should consider to check if the given - links still exists. +5.2 Core class + + Should the Core be a module on its own, or a class of GetTor? + Currently it's just a class, after realizing that + + core = gettor.core.core() + + Was too much. It's better to do + + core = gettor.Core() + + Isn't? -5.2 Logs +5.3 Logs Should we mantain separate logs for successful and fail requests? diff --git a/src/core_demo.py b/src/core_demo.py new file mode 100644 index 0000000..19ffd58 --- /dev/null +++ b/src/core_demo.py @@ -0,0 +1,19 @@ +#!/usr/bin/python +# +# Dummy script to test GetTore's Core module progress +# + +import gettor + +core = gettor.Core() + +try: + links = core.get_links('linux', 'en') + print links +except ValueError as e: + print "Value error! " + str(e) +except RuntimeError as e: + print "Internal error! " + str(e) + + + diff --git a/src/gettor.py b/src/gettor.py new file mode 100644 index 0000000..573bae7 --- /dev/null +++ b/src/gettor.py @@ -0,0 +1,145 @@ +import os +import re +import inspect +import logging + + +class Core: + + """ + Gets links from providers and delivers them to other modules. + + Public methods: + get_links(operating_system, locale) + """ + + def __init__(self): + """ + Initialize a Core object by reading a configuration file. + + Parameters: none + """ + + # Read configuration file + + + # Dummy info. This should be read from a configuration file. + self.basedir = './providers/' + self.supported_locales = [ 'en', 'es' ] + self.supported_os = [ 'windows', 'linux', 'osx' ] + + def get_links(self, operating_system, locale): + """ + Public method to obtain links. + + Checks for supported locales and operating systems. It returns + ValueError if the locale or operating system is not supported. + It raises RuntimeError if something goes wrong while trying + to obtain the links. It returns a string on success. This + method should be called from the services modules of GetTor + (e.g. SMTP). + """ + + # Log requests + self._log_request(operating_system, locale) + + if locale not in self.supported_locales: + raise ValueError("Locale %s not supported at the moment" % locale) + + if operating_system not in self.supported_os: + raise ValueError("Operating system %s not supported at the moment" + % operating_system) + + # This could change in the future, let's leave it isolated. + links = self._get_links(operating_system, locale) + + if links is None: + raise RuntimeError("Something went wrong with GetTor's core") + + return links + + def _get_links(self, operating_system, locale): + """ + Private method to obtain the links. + + Looks for the links inside each provider file. On success + returns a string with the links. On failure returns None. + This should only be called from get_links() method. + + Parameters: + os: string describing the operating system + locale: string describing the locale + """ + + # There should be a 'providers' directory inside self.basedir + # + # Each .links file begins with a string describing the provider. + # After that, every line should have the following format: + # + # operating_system locale link package_signature key_fingerprint + # + # e.g. + # + # Dropbox + # linux en https://foo.bar https://foo.bar.asc 111-222-333-444 + # osx es https://bar.baz https://bar.baz.asc 555-666-777-888 + # + links = [] + + # Look for files ending with .links + p = re.compile('.*\.links$') + + for name in os.listdir(self.basedir): + path = os.path.abspath(os.path.join(self.basedir, name)) + if os.path.isfile(path) and p.match(path): + links.append(path) + + # Let's create a dictionary linking each provider with the links + # found for operating_system and locale. This way makes it easy + # to check if no links were found. + providers = {} + + # We trust links have been generated properly + for name in links: + link_file = open(name, 'r') + provider_name = link_file.readline() + + for line in link_file: + words = line.split(' ') + if words[0] == operating_system and words[1] == locale: + providers.setdefault(provider_name, []).append( + "%s %s %s" % (words[2], words[3], words[4].replace('-', ' ')) + ) + + link_file.close() + + # Create the final links list with all providers + all_links = [] + + for key in providers.keys(): + all_links.append( + "\n%s\n%s\n" % (key, ''.join(providers[key])) + ) + + if all_links: + return "".join(all_links) + else: + return None + + + + def _log_request(self, operating_system, locale): + """ + Private method to log what service module called to get the links. + + Parameters: none + """ + + caller = inspect.stack()[2] + module = inspect.getmodule(caller[0]) + + # Dummy print for now. Should be done with logging + print "\nCalled by %s\nOS: %s\nLocale: %s\n" % \ + (str(module), operating_system, locale) + + diff --git a/src/gettor.pyc b/src/gettor.pyc new file mode 100644 index 0000000..81b41ac Binary files /dev/null and b/src/gettor.pyc differ diff --git a/src/providers/dropbox.links b/src/providers/dropbox.links new file mode 100644 index 0000000..53d4a04 --- /dev/null +++ b/src/providers/dropbox.links @@ -0,0 +1,5 @@ +Dropbox + +linux en https://foo.bar https://foo.bar.asc 111-222-333-444 +linux es https://bar.baz https://bar.baz.asc 555-666-777-888 +linux en https://baz.foo https://baz.foo.asc 999-111-222-333 diff --git a/src/providers/gdrive.links b/src/providers/gdrive.links new file mode 100644 index 0000000..570c8e2 --- /dev/null +++ b/src/providers/gdrive.links @@ -0,0 +1,5 @@ +Google Drive + +linux en https://foo.bar https://foo.bar.asc 111-222-333-444 +linux es https://bar.baz https://bar.baz.asc 555-666-777-888 +linux en https://baz.foo https://baz.foo.asc 999-111-222-333 diff --git a/src/providers/mirrors.links b/src/providers/mirrors.links new file mode 100644 index 0000000..0eb5b10 --- /dev/null +++ b/src/providers/mirrors.links @@ -0,0 +1,5 @@ +Official Mirrors + +linux en https://foo.bar https://foo.bar.asc 111-222-333-444 +linux es https://bar.baz https://bar.baz.asc 555-666-777-888 +linux en https://baz.foo https://baz.foo.asc 999-111-222-333