[tor-commits] [gettor/develop] RESTful API
ilv at torproject.org
ilv at torproject.org
Wed Dec 16 19:52:48 UTC 2015
commit 6b12e964082feb9ffc1d5b7da84ead87378b8ae9
Author: ilv <ilv at users.noreply.github.com>
Date: Wed Dec 16 16:55:25 2015 -0300
RESTful API
---
gettor/http.py | 493 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
process_http.py | 13 ++
2 files changed, 506 insertions(+)
diff --git a/gettor/http.py b/gettor/http.py
new file mode 100644
index 0000000..c1441c6
--- /dev/null
+++ b/gettor/http.py
@@ -0,0 +1,493 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of GetTor.
+#
+# :authors: Israel Leiva <ilv at torproject.org>
+# see also AUTHORS file
+#
+# :copyright: (c) 2008-2015, The Tor Project, Inc.
+# (c) 2015, Israel Leiva
+#
+# :license: This is Free Software. See LICENSE for license information.
+
+import os
+import re
+import json
+import codecs
+import urllib2
+import ConfigParser
+
+from time import gmtime, strftime
+
+from flask import Flask
+from flask_restful import Api, Resource, reqparse
+
+import core
+import utils
+
+"""GetTor RESTful API"""
+
+# currently supported locales for Tor Browser
+LC = ['ar', 'de', 'en-US', 'es-ES', 'fa', 'fr', 'it', 'ko', 'nl', 'pl',
+ 'pt-PT', 'ru', 'tr', 'vi', 'zh-CN']
+
+# https://gitweb.tpo/tor-browser-spec.git/tree/processes/VersionNumbers
+# does not say anything about operating systems, so it's possible the
+# notation might change in the future. We should always use the same three
+# strings though: linux, windows, osx.
+OS = {
+ 'Linux': 'linux',
+ 'Windows': 'windows',
+ 'MacOS': 'osx'
+}
+
+# based on
+# https://gitweb.tpo.org/tor-browser-spec.git/tree/processes/VersionNumbers
+# except for the first one, which is based on current RecommendedTBBVersions
+RE = {
+ 'os': '(.*)-(\w+)',
+ 'alpha': '\d\.\d(\.\d)*a\d+',
+ 'beta': '\d\.\d(\.\d)*b\d+',
+ 'stable': '\d\.\d(\.\d)*'
+}
+
+# strings to build names of packages depending on OS.
+PKG = {
+ 'windows': 'torbrowser-install-%s_%s.exe',
+ 'linux': 'tor-browser-linux%s-%s_%s.tar.xz',
+ 'osx': 'TorBrowser-%s-osx64_%s.dmg'
+}
+
+# bin and asc are used to build the download links for each version, os and lc
+URL = {
+ 'version': 'https://www.torproject.org/projects/torbrowser/RecommendedTBBVersions',
+ 'bin': 'https://www.torproject.org/dist/torbrowser/%s/%s',
+ 'asc': 'https://www.torproject.org/dist/torbrowser/%s/%s.asc'
+}
+
+# filters for API resources
+parser = reqparse.RequestParser()
+parser.add_argument('os', type=str, help='Operating System')
+parser.add_argument('lc', type=str, help='Locale')
+
+
+class ConfigError(Exception):
+ pass
+
+
+class InternalError(Exception):
+ pass
+
+
+class HTTP(object):
+ """ Provide useful resources via RESTful API. """
+ def __init__(self, cfg=None):
+ """ Create new object by reading a configuration file.
+
+ :param: cfg (string) path of the configuration file.
+
+ """
+ default_cfg = 'http.cfg'
+ config = ConfigParser.ConfigParser()
+
+ if cfg is None or not os.path.isfile(cfg):
+ cfg = default_cfg
+
+ try:
+ with open(cfg) as f:
+ config.readfp(f)
+ except IOError:
+ raise ConfigError("File %s not found!" % cfg)
+
+ try:
+ # server that provides the RESTful API
+ self.server = config.get('general', 'server')
+ # path to the links files
+ self.links_path = config.get('general', 'links')
+ # path to mirrors in json
+ self.mirrors_path = config.get('general', 'mirrors')
+
+ # we will ask gettor.core for the links
+ core_cfg = config.get('general', 'core')
+ self.core = core.Core(core_cfg)
+
+ except ConfigParser.Error as e:
+ raise ConfigError("Configuration error: %s" % str(e))
+ except core.ConfigError as e:
+ raise InternalError("HTTP error: %s" % str(e))
+
+ def _is_json(self, my_json):
+ """ Check if json generated is valid.
+
+ :param: my_json (string) data to ve verified.
+
+ :return: (bool) true if data is json-valid, false otherwise.
+
+ """
+ try:
+ json_object = json.loads(my_json)
+ except ValueError, e:
+ return False
+ return True
+
+ def _get_provider_name(self, p):
+ """ Return simplified version of provider's name.
+
+ :param: p (string) provider's name.
+
+ :return: (string) provider's name in lowercase and without spaces.
+
+ """
+ p = p.replace(' ', '-')
+ return p.lower()
+
+ def _add_links(self, lv, release, version, os):
+ """ Add link for all locales in LC depending on given OS.
+
+ :param: lv (dict) latest version data structure.
+ :param: release (string) release to which add the links.
+ :param: version (string) version obtained from tpo.
+ :param: os (string) operating system.
+
+ """
+ for lc in LC:
+ if os == 'linux':
+ pkg32 = PKG['linux'] % ('32', version, lc)
+ link_bin32 = URL['bin'] % (version, pkg32)
+ link_asc32 = URL['asc'] % (version, pkg32)
+
+ pkg64 = PKG['linux'] % ('64', version, lc)
+ link_bin64 = URL['bin'] % (version, pkg64)
+ link_asc64 = URL['asc'] % (version, pkg64)
+
+ lv[release]['downloads'][os][lc] = {
+ 'binary32': link_bin32,
+ 'signature32': link_asc32,
+ 'binary64': link_bin64,
+ 'signature64': link_asc64,
+ }
+ else:
+ if os == 'windows':
+ pkg = PKG['windows'] % (version, lc)
+
+ elif os == 'osx':
+ pkg = PKG['osx'] % (version, lc)
+
+ else:
+ continue
+
+ link_bin = URL['bin'] % (version, pkg)
+ link_asc = URL['asc'] % (version, pkg)
+ lv[release]['downloads'][os][lc] = {
+ 'binary': link_bin,
+ 'signature': link_asc
+ }
+
+ def _load_latest_version(self):
+ """ Load latest version data. """
+ response = urllib2.urlopen(URL['version'])
+ json_response = json.load(response)
+
+ lv = {
+ 'stable': {
+ 'latest_version': '',
+ 'downloads': {}
+ },
+ 'alpha': {
+ 'latest_version': '',
+ 'downloads': {}
+ },
+ 'beta': {
+ 'latest_version': '',
+ 'downloads': {}
+ }
+ }
+
+ self.releases = {
+ 'alpha': '%s/latest/alpha' % self.server,
+ 'beta': '%s/latest/beta' % self.server,
+ 'stable': '%s/latest/stable' % self.server,
+ 'updated_at': strftime("%Y-%m-%d %H:%M:%S", gmtime())
+ }
+
+ # one iteration to find the latest version for each release
+ for v in json_response:
+ # latest version for each release
+ if not re.match(RE['os'], v):
+ if re.match(RE['alpha'], v):
+ if v > lv['alpha']['latest_version']:
+ # we'll use the latest one
+ lv['alpha']['latest_version'] = v
+
+ elif re.match(RE['beta'], v):
+ if v > lv['beta']['latest_version']:
+ # we'll use the latest one
+ lv['beta']['latest_version'] = v
+
+ elif re.match(RE['stable'], v):
+ if v > lv['stable']['latest_version']:
+ # we'll use the latest one
+ lv['stable']['latest_version'] = v
+
+ latest_alpha = lv['alpha']['latest_version']
+ latest_beta = lv['beta']['latest_version']
+ latest_stable = lv['stable']['latest_version']
+
+ # another iteration to add the links
+ for v in json_response:
+ # based on current RecommendedTBBVersions scheme
+ # for each release and for each os we build links for all locales
+ if re.match(RE['os'], v):
+ m = re.match(RE['os'], v)
+ version = m.group(1)
+ osys = m.group(2)
+
+ if osys in OS:
+ if latest_alpha and version == latest_alpha \
+ and re.match(RE['alpha'], version):
+ lv['alpha']['downloads'][OS[osys]] = {}
+ self._add_links(lv, 'alpha', version, OS[osys])
+
+ elif latest_beta and version == latest_beta \
+ and re.match(RE['beta'], version):
+ lv['beta']['downloads'][OS[osys]] = {}
+ self._add_links(lv, 'beta', version, OS[osys])
+
+ elif latest_stable and version == latest_stable \
+ and re.match(RE['stable'], version):
+ lv['stable']['downloads'][OS[osys]] = {}
+ self._add_links(lv, 'stable', version, OS[osys])
+
+ lv['updated_at'] = strftime("%Y-%m-%d %H:%M:%S", gmtime())
+ self.lv = lv
+
+ def _load_links(self):
+ """ Load links and providers data. """
+ links_files = []
+
+ # look for files ending with .links in links_path
+ p = re.compile('.*\.links$')
+ for name in os.listdir(self.links_path):
+ path = os.path.abspath(os.path.join(self.links_path, name))
+ if os.path.isfile(path) and p.match(path):
+ links_files.append(path)
+
+ links = {}
+ providers = {}
+ supported_os = self.core.get_supported_os()
+ supported_lc = self.core.get_supported_lc()
+
+ for name in links_files:
+ config = ConfigParser.ConfigParser()
+ try:
+ with open(name) as f:
+ config.readfp(f)
+ except IOError:
+ raise InternalError("File %s not found!" % name)
+
+ try:
+ pname = config.get('provider', 'name')
+ pname = self._get_provider_name(pname)
+
+ # build providers dict
+ providers[pname] = '%s/providers/%s' % (self.server, pname)
+ providers['updated_at'] = strftime(
+ "%Y-%m-%d %H:%M:%S", gmtime()
+ )
+
+ self.providers = providers
+ links[pname] = {}
+
+ # build links data.
+ for osys in supported_os:
+ links[pname][osys] = {}
+ for lc in supported_lc:
+ links[pname][osys][lc] = {}
+
+ for osys in supported_os:
+ for lc in supported_lc:
+ l_str = config.get(osys, lc)
+
+ # linux has 32 and 64 bit packages
+ if osys == 'linux':
+ l32_str, l64_str = l_str.split(',')
+
+ link32, sig32, sha32 = [
+ l for l in l32_str.split("$") if l
+ ]
+
+ link64, sig64, sha64 = [
+ l for l in l64_str.split("$") if l
+ ]
+ link64 = link64.lstrip()
+
+ links[pname][osys][lc]['binary32'] = link32
+ links[pname][osys][lc]['signature32'] = sig32
+ links[pname][osys][lc]['sha256-32'] = sha32
+ links[pname][osys][lc]['binary64'] = link64
+ links[pname][osys][lc]['signature64'] = sig64
+ links[pname][osys][lc]['sha256-64'] = sha64
+
+ else:
+ link, sig, sha = [l for l in l_str.split("$") if l]
+ links[pname][osys][lc]['binary'] = link
+ links[pname][osys][lc]['signature'] = sig
+ links[pname][osys][lc]['sha256'] = sha
+
+ except ConfigParser.Error as e:
+ raise InternalError("%s" % str(e))
+
+ links['updated_at'] = strftime("%Y-%m-%d %H:%M:%S", gmtime())
+ self.links = links
+
+ def _load_mirrors(self):
+ """ Load mirrors data. """
+ mirrors = []
+
+ # json of mirrors should be obtained from get_mirrors.py
+ json_data = open(self.mirrors_path).read()
+ mirrors = json.loads(json_data)
+
+ self.mirrors = mirrors
+
+ def _load_resources(self):
+ """ Load available resources data. """
+
+ self.resources = {
+ 'providers': '%s/providers' % self.server,
+ 'mirrors': '%s/mirrors' % self.server,
+ 'latest_version': '%s/latest' % self.server,
+ 'updated_at': strftime("%Y-%m-%d %H:%M:%S", gmtime())
+ }
+
+ def load_data(self):
+ """ Load all data.
+
+ Since data is not frequently updated, we load all data before
+ running the RESTful API. Every time the links/mirrors/version
+ data is updated we should restart the API.
+
+ """
+ self._load_links()
+ self._load_mirrors()
+ self._load_resources()
+ self._load_latest_version()
+
+ def run(self):
+ """ Run RESTful API. """
+ app = Flask(__name__)
+ api = Api(app)
+
+ api.add_resource(
+ AvailableResources,
+ '/',
+ resource_class_kwargs={
+ 'resources': self.resources
+ }
+ )
+
+ api.add_resource(
+ Providers,
+ '/providers',
+ '/providers/<string:provider>',
+ resource_class_kwargs={
+ 'links': self.links,
+ 'providers': self.providers
+ }
+ )
+
+ api.add_resource(
+ LatestVersion,
+ '/latest',
+ '/latest/<string:release>',
+ resource_class_kwargs={
+ 'latest_version': self.lv,
+ 'releases': self.releases
+ }
+ )
+
+ api.add_resource(
+ Mirrors,
+ '/mirrors',
+ resource_class_kwargs={
+ 'mirrors': self.mirrors
+ }
+ )
+
+ app.run(debug=True)
+
+
+class AvailableResources(Resource):
+ def __init__(self, resources):
+ """ Set initial data. """
+ self.resources = resources
+
+ def get(self):
+ """ Return available resources on the API. """
+ return self.resources
+
+
+class Providers(Resource):
+ def __init__(self, providers, links):
+ """ Set initial data. """
+ self.providers = providers
+ self.links = links
+
+ def get(self, provider=None):
+ """ Return providers and links data. """
+
+ # we use arg to filter results by os and lc (in that order)
+ arg = parser.parse_args()
+
+ if provider:
+ if arg['os']:
+ if arg['lc']:
+ # links by provider, os, and lc (in that order)
+ return self.links[provider][arg['os']][arg['lc']]
+ else:
+ # links by provider and os (in that order)
+ return self.links[provider][arg['os']]
+ else:
+ # links by provider
+ return self.links[provider]
+ else:
+ # list of providers
+ return self.providers
+
+
+class LatestVersion(Resource):
+ def __init__(self, latest_version, releases):
+ """ Set initial data. """
+ self.lv = latest_version
+ self.releases = releases
+
+ def get(self, release=None):
+ """ Return latest version data. """
+
+ # we use arg to filter results by os and lc (in that order)
+ arg = parser.parse_args()
+
+ if release:
+ if arg['os']:
+ if arg['lc']:
+ # tpo links by release, os and lc (in that order)
+ return self.lv[release]['downloads'][arg['os']][arg['lc']]
+ else:
+ # tpo links by release and os (in that order)
+ return self.lv[release]['downloads'][arg['os']]
+ else:
+ # version and tpo links by release
+ return self.lv[release]
+ else:
+ # list of releases
+ return self.releases
+
+
+class Mirrors(Resource):
+ def __init__(self, mirrors):
+ """ Set initial data. """
+ self.mirrors = mirrors
+
+ def get(self):
+ """ Return mirrors data. """
+ return self.mirrors
diff --git a/process_http.py b/process_http.py
new file mode 100644
index 0000000..31475f1
--- /dev/null
+++ b/process_http.py
@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import gettor.http
+
+
+def main():
+ api = gettor.http.HTTP('http.cfg')
+ api.load_data()
+ api.run()
+
+if __name__ == '__main__':
+ main()
More information about the tor-commits
mailing list