- tor-commits - lists.torproject.org

[gettor/master] Check SSL certificates at API requests
by ilv＠torproject.org 15 Feb '16

15 Feb '16

commit cfde87075bf5a19da3fa8bfac7c7c2e157f1f0a3 Author: Donncha O'Cearbhaill <donncha(a)donncha.is> Date: Fri Feb 12 19:33:46 2016 +0000 Check SSL certificates at API requests --- upload/bundles2github.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/upload/bundles2github.py b/upload/bundles2github.py index 1252163..d081ddd 100644 --- a/upload/bundles2github.py +++ b/upload/bundles2github.py @@ -11,23 +11,25 @@ # :license: This is Free Software. See LICENSE for license information. # +# Use pyopenssl to verify TLS certifcates +try: + import urllib3.contrib.pyopenssl + urllib3.contrib.pyopenssl.inject_into_urllib3() +except ImportError: + pass + import os -import re import sys import argparse import ConfigParser +import gnupg import github3 -import gnupg + import gettor.core from gettor.utils import (get_bundle_info, get_file_sha256, find_files_to_upload) -import urllib3 - -# Actually verify Github's cert! -urllib3.disable_warnings() - def upload_new_release(github_repo, version, upload_dir): """ @@ -144,13 +146,10 @@ if __name__ == '__main__': print("Creating links file") for asset in release.assets: - url = ("https://github.com/{user}/{repo}/releases/download/" - "v{tag}/{file}".format( - user=github_user, - repo=github_repo, - tag=version, - file=asset.name, - )) + url = asset.browser_download_url + if url.endswith('.asc'): + continue + osys, arch, lc = get_bundle_info(asset.name) sha256 = get_file_sha256( os.path.abspath(os.path.join(tb_path, asset.name))

1 0

[gettor/master] Improvement of links message. See #17424
by ilv＠torproject.org 15 Feb '16

15 Feb '16

commit 676ee2890ea9d078fc33ac8b990f2cf655b3a086 Author: ilv <ilv(a)users.noreply.github.com> Date: Fri Dec 11 10:18:25 2015 -0300 Improvement of links message. See #17424 --- gettor/core.py | 22 ++++++++++++++++------ gettor/smtp.py | 8 +++++++- lang/smtp/i18n/en/LC_MESSAGES/en.po | 37 ++++++++++++------------------------- 3 files changed, 35 insertions(+), 32 deletions(-) diff --git a/gettor/core.py b/gettor/core.py index 3d946c9..97f7ce3 100644 --- a/gettor/core.py +++ b/gettor/core.py @@ -253,43 +253,49 @@ class Core(object): t32, t64 = [t for t in providers[pname].split(",") if t] link, signature, chs32 = [l for l in t32.split("$") if l] + link = "%s: %s" % (pname, link) links32[link] = signature link, signature, chs64 = [l for l in t64.split("$") if l] + link = "%s: %s" % (pname, link.lstrip()) links64[link] = signature else: link, signature, chs32 = [l for l in providers[pname].split("$") if l] + link = "%s: %s" % (pname, link) links32[link] = signature #providers[pname] = providers[pname].replace(",", "") #providers[pname] = providers[pname].replace("$", "\n\n") + ### We will improve and add the verification section soon ### # all packages are signed with same key # (Tor Browser developers) - fingerprint = config.get('key', 'fingerprint') + # fingerprint = config.get('key', 'fingerprint') # for now, english messages only - fingerprint_msg = self._get_msg('fingerprint', 'en') - fingerprint_msg = fingerprint_msg % fingerprint + # fingerprint_msg = self._get_msg('fingerprint', 'en') + # fingerprint_msg = fingerprint_msg % fingerprint except ConfigParser.Error as e: raise InternalError("%s" % str(e)) # create the final links list with all providers all_links = [] - msg = "Tor Browser %s-bit:" % arch + msg = "Tor Browser %s-bit:\n" % arch for link in links32: msg = "%s\n%s" % (msg, link) all_links.append(msg) if osys == 'linux': - msg = "\n\n\nTor Browser 64-bit:" + msg = "\n\n\nTor Browser 64-bit:\n" for link in links64: msg = "%s\n%s" % (msg, link) all_links.append(msg) + ### We will improve and add the verification section soon ### + """ msg = "\n\n\nTor Browser's signature %s-bit:" %\ arch for link in links32: @@ -313,6 +319,9 @@ class Core(object): all_links.append(msg) """ + ### end verification ### + + """ for key in providers.keys(): # get more friendly description of the provider try: @@ -328,8 +337,9 @@ class Core(object): raise InternalError("%s" % str(e)) """ + ### We will improve and add the verification section soon ### # add fingerprint after the links - all_links.append(fingerprint_msg) + # all_links.append(fingerprint_msg) if all_links: return "".join(all_links) diff --git a/gettor/smtp.py b/gettor/smtp.py index ed6a4fc..1cc8db3 100644 --- a/gettor/smtp.py +++ b/gettor/smtp.py @@ -33,6 +33,12 @@ import blacklist """SMTP module for processing email requests.""" +OS = { + 'osx': 'Mac OS X', + 'linux': 'Linux', + 'windows': 'Windows' +} + class ConfigError(Exception): pass @@ -368,7 +374,7 @@ class SMTP(object): try: links_subject = self._get_msg('links_subject', 'en') links_msg = self._get_msg('links_msg', 'en') - links_msg = links_msg % (os, lc, links) + links_msg = links_msg % (os, links) self._send_email( from_addr, diff --git a/lang/smtp/i18n/en/LC_MESSAGES/en.po b/lang/smtp/i18n/en/LC_MESSAGES/en.po index ee6a24e..f987648 100644 --- a/lang/smtp/i18n/en/LC_MESSAGES/en.po +++ b/lang/smtp/i18n/en/LC_MESSAGES/en.po @@ -22,42 +22,29 @@ msgstr "The locale you requested '%s' is not supported." #: Links message msgid "links_msg" -msgstr "Hello there! this is the 'GetTor' robot.\n\ +msgstr "You requested Tor Browser for %s.\n\ \n\ -Below are the links for your request (Tor Browser for %s, %s package):\n\ +You will need only one of the links below to download the bundle. If a link\n\ +does not work for you, try the next one.\n\ \n\ %s\n\ \n\ -===========================================================================\n\ -Still need help? If you have any questions, trouble connecting to Tor\n\ -network, or need to talk to a human, please contact our support team at:\n\ \n\ - help(a)rt.torproject.org\n\ -\n\ -We are ready to answer your queries in English, Farsi, Chinese, Arabic,\n\ -French and Spanish." +All the best.\n\ +--\n\ +GetTor team" #: Mirrors message msgid "mirrors_msg" -msgstr "Hello there! this is the 'GetTor' robot.\n\ +msgstr "Hi! this is the GetTor robot.\n\ \n\ Thank you for your request. Attached to this email you will find\n\ -an updated list of mirrors of Tor Project's website.\n\ -\n\ -Still need help? If you have any questions, trouble connecting to Tor\n\ -network, or need to talk to a human, please contact our support team at:\n\ -\n\ - help(a)rt.torproject.org\n\ -\n\ -We are ready to answer your queries in English, Farsi, Chinese, Arabic,\n\ -French and Spanish." +an updated list of mirrors of Tor Project's website." #: Help message msgid "help_msg" -msgstr "Hello there! this is the 'GetTor' robot.\n\ -\n\ -Thank you for your request. I am here to help you download the latest\n\ -Tor Browser.\n\ +msgstr "Hi! This is the GetTor robot. I am here to help you download the\n\ +latest version of Tor Browser.\n\ \n\ Please reply to this message with one of the options below:\n\ \n\ @@ -66,6 +53,6 @@ Please reply to this message with one of the options below:\n\ osx\n\ mirrors\n\ \n\ -And I will send you the download/access instructions quickly.\n\ +I will then send you the download instructions.\n\ \n\ -Tip: Just send a blank reply to this message if you are not sure." +If you are unsure, just send a blank reply to this message."

1 0

[gettor/master] Add script for generating automatic reports (#17588)
by ilv＠torproject.org 15 Feb '16

15 Feb '16

commit 35d304c9c0426946c17313ecc0e24f777e582b4a Author: Sukhbir Singh <sukhbir(a)torproject.org> Date: Tue Dec 8 00:40:38 2015 -0500 Add script for generating automatic reports (#17588) --- report.py | 129 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/report.py b/report.py new file mode 100644 index 0000000..b75d7c8 --- /dev/null +++ b/report.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python + +# This file is part of GetTor, a Tor Browser distribution system. + +# :authors: Sukhbir Singh <sukhbir(a)torproject.org> +# see also AUTHORS file +# +# :copyright: (c) 2008-2015, The Tor Project, Inc. + +# To read the log files and populate the db, run the script with `--logs' once +# every day and then clear the log files. To generate the report, run it with +# `--report', once (or twice?) a month. + +# TODO: Logging? argparse? + +import os +import sys +import ConfigParser +from datetime import datetime as dt + +import sqlite3 + +CORE_CFG = "core.cfg" + +CHANNELS = ["smtp", "xmpp", "twitter"] +LOG_FILES = [channel + ".log" for channel in CHANNELS] + +DB = "gettor2.db" + +OUTPUT = "report.log" + +REPORT = """@ GetTor Report for {} + +We received a total of {} requests in {}, with a peak of {} requests on {}. + +[*] Request +{} + +[*] OS +{} + +[*] Language +{} + +[*] Channel +{} + +""" + + +class Report(object): + def __init__(self): + self.config = ConfigParser.ConfigParser() + self.config.read(CORE_CFG) + + self.log_dir = self.config.get("log", "dir") + # This is defined in the sample log file. + self.log_format = ["date", "request", "os", "locale"] + + self.conn = sqlite3.connect(DB) + + def db_write(self): + self.cursor = self.conn.cursor() + for each in self.logs: + self.cursor.execute("""INSERT INTO requests + (date, request, os, locale, channel) VALUES + (:date, :request, :os, :locale, :channel)""", + each) + self.conn.commit() + self.conn.close() + + def get_logs(self): + self.logs = [] + for each in LOG_FILES: + with open(os.path.join(self.log_dir, each)) as f: + for line in f: + if line.startswith("[INFO] "): + # The "7" here is for the "[INFO] " text. + logs = [log.strip() for log in line[7:].split(";")] + log_data = {key: value for key, value in + zip(self.log_format, logs)} + # We also need the channel. + log_data["channel"] = each.split(".")[0] + self.logs.append(log_data) + self.db_write() + self.conn.close() + + def generate_report(self): + self.cursor = self.conn.cursor() + self.requests = [] + + self.cursor.execute("SELECT COUNT(*) FROM requests") + self.requests.append(self.cursor.fetchone()) + + self.cursor.execute("""SELECT date, COUNT(*) FROM requests +GROUP BY date +ORDER BY COUNT(date) +DESC""") + self.requests.append(self.cursor.fetchone()) + + self.columns = ["request", "os", "locale", "channel"] + for each in self.columns: + self.cursor.execute("""SELECT {0}, COUNT(*) FROM requests +GROUP BY {0} +ORDER BY COUNT({0}) +DESC""".format(each)) + result = "\n".join(["{0:>16}: {1}".format(each[0], each[1]) + for each in self.cursor.fetchall() + if not each[0] == "none"]) + self.requests.append(result) + + with open(OUTPUT, "a") as f: + f.write(REPORT.format(dt.now().strftime("%B %Y"), + self.requests[0][0], + dt.now().strftime("%B"), + self.requests[1][1], + dt.strptime(self.requests[1][0], "%Y-%m-%d") + .strftime("%B %-d"), + *self.requests[2:])) + self.conn.close() + +if __name__ == "__main__": + try: + if sys.argv[1] == "--logs": + Report().get_logs() + if sys.argv[1] == "--report": + Report().generate_report() + except IndexError: + sys.exit("You need either `--logs' or `--report.'")

1 0

[gettor/master] Merge branch 'develop' of ssh://git-rw.torproject.org/gettor into develop
by ilv＠torproject.org 15 Feb '16

15 Feb '16

commit b0261b785ec3bab2cc0324900036c5114590fab9 Merge: 3cc92b1 b2b07f3 Author: ilv <ilv(a)users.noreply.github.com> Date: Thu Dec 31 17:18:57 2015 -0300 Merge branch 'develop' of ssh://git-rw.torproject.org/gettor into develop lang/smtp/i18n/en/LC_MESSAGES/en.po | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)

1 0

[gettor/master] RESTful API
by ilv＠torproject.org 15 Feb '16

15 Feb '16

commit 6b12e964082feb9ffc1d5b7da84ead87378b8ae9 Author: ilv <ilv(a)users.noreply.github.com> Date: Wed Dec 16 16:55:25 2015 -0300 RESTful API --- gettor/http.py | 493 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ process_http.py | 13 ++ 2 files changed, 506 insertions(+) diff --git a/gettor/http.py b/gettor/http.py new file mode 100644 index 0000000..c1441c6 --- /dev/null +++ b/gettor/http.py @@ -0,0 +1,493 @@ +# -*- coding: utf-8 -*- +# +# This file is part of GetTor. +# +# :authors: Israel Leiva <ilv(a)torproject.org> +# see also AUTHORS file +# +# :copyright: (c) 2008-2015, The Tor Project, Inc. +# (c) 2015, Israel Leiva +# +# :license: This is Free Software. See LICENSE for license information. + +import os +import re +import json +import codecs +import urllib2 +import ConfigParser + +from time import gmtime, strftime + +from flask import Flask +from flask_restful import Api, Resource, reqparse + +import core +import utils + +"""GetTor RESTful API""" + +# currently supported locales for Tor Browser +LC = ['ar', 'de', 'en-US', 'es-ES', 'fa', 'fr', 'it', 'ko', 'nl', 'pl', + 'pt-PT', 'ru', 'tr', 'vi', 'zh-CN'] + +# https://gitweb.tpo/tor-browser-spec.git/tree/processes/VersionNumbers +# does not say anything about operating systems, so it's possible the +# notation might change in the future. We should always use the same three +# strings though: linux, windows, osx. +OS = { + 'Linux': 'linux', + 'Windows': 'windows', + 'MacOS': 'osx' +} + +# based on +# https://gitweb.tpo.org/tor-browser-spec.git/tree/processes/VersionNumbers +# except for the first one, which is based on current RecommendedTBBVersions +RE = { + 'os': '(.*)-(\w+)', + 'alpha': '\d\.\d(\.\d)*a\d+', + 'beta': '\d\.\d(\.\d)*b\d+', + 'stable': '\d\.\d(\.\d)*' +} + +# strings to build names of packages depending on OS. +PKG = { + 'windows': 'torbrowser-install-%s_%s.exe', + 'linux': 'tor-browser-linux%s-%s_%s.tar.xz', + 'osx': 'TorBrowser-%s-osx64_%s.dmg' +} + +# bin and asc are used to build the download links for each version, os and lc +URL = { + 'version': 'https://www.torproject.org/projects/torbrowser/RecommendedTBBVersions', + 'bin': 'https://www.torproject.org/dist/torbrowser/%s/%s', + 'asc': 'https://www.torproject.org/dist/torbrowser/%s/%s.asc' +} + +# filters for API resources +parser = reqparse.RequestParser() +parser.add_argument('os', type=str, help='Operating System') +parser.add_argument('lc', type=str, help='Locale') + + +class ConfigError(Exception): + pass + + +class InternalError(Exception): + pass + + +class HTTP(object): + """ Provide useful resources via RESTful API. """ + def __init__(self, cfg=None): + """ Create new object by reading a configuration file. + + :param: cfg (string) path of the configuration file. + + """ + default_cfg = 'http.cfg' + config = ConfigParser.ConfigParser() + + if cfg is None or not os.path.isfile(cfg): + cfg = default_cfg + + try: + with open(cfg) as f: + config.readfp(f) + except IOError: + raise ConfigError("File %s not found!" % cfg) + + try: + # server that provides the RESTful API + self.server = config.get('general', 'server') + # path to the links files + self.links_path = config.get('general', 'links') + # path to mirrors in json + self.mirrors_path = config.get('general', 'mirrors') + + # we will ask gettor.core for the links + core_cfg = config.get('general', 'core') + self.core = core.Core(core_cfg) + + except ConfigParser.Error as e: + raise ConfigError("Configuration error: %s" % str(e)) + except core.ConfigError as e: + raise InternalError("HTTP error: %s" % str(e)) + + def _is_json(self, my_json): + """ Check if json generated is valid. + + :param: my_json (string) data to ve verified. + + :return: (bool) true if data is json-valid, false otherwise. + + """ + try: + json_object = json.loads(my_json) + except ValueError, e: + return False + return True + + def _get_provider_name(self, p): + """ Return simplified version of provider's name. + + :param: p (string) provider's name. + + :return: (string) provider's name in lowercase and without spaces. + + """ + p = p.replace(' ', '-') + return p.lower() + + def _add_links(self, lv, release, version, os): + """ Add link for all locales in LC depending on given OS. + + :param: lv (dict) latest version data structure. + :param: release (string) release to which add the links. + :param: version (string) version obtained from tpo. + :param: os (string) operating system. + + """ + for lc in LC: + if os == 'linux': + pkg32 = PKG['linux'] % ('32', version, lc) + link_bin32 = URL['bin'] % (version, pkg32) + link_asc32 = URL['asc'] % (version, pkg32) + + pkg64 = PKG['linux'] % ('64', version, lc) + link_bin64 = URL['bin'] % (version, pkg64) + link_asc64 = URL['asc'] % (version, pkg64) + + lv[release]['downloads'][os][lc] = { + 'binary32': link_bin32, + 'signature32': link_asc32, + 'binary64': link_bin64, + 'signature64': link_asc64, + } + else: + if os == 'windows': + pkg = PKG['windows'] % (version, lc) + + elif os == 'osx': + pkg = PKG['osx'] % (version, lc) + + else: + continue + + link_bin = URL['bin'] % (version, pkg) + link_asc = URL['asc'] % (version, pkg) + lv[release]['downloads'][os][lc] = { + 'binary': link_bin, + 'signature': link_asc + } + + def _load_latest_version(self): + """ Load latest version data. """ + response = urllib2.urlopen(URL['version']) + json_response = json.load(response) + + lv = { + 'stable': { + 'latest_version': '', + 'downloads': {} + }, + 'alpha': { + 'latest_version': '', + 'downloads': {} + }, + 'beta': { + 'latest_version': '', + 'downloads': {} + } + } + + self.releases = { + 'alpha': '%s/latest/alpha' % self.server, + 'beta': '%s/latest/beta' % self.server, + 'stable': '%s/latest/stable' % self.server, + 'updated_at': strftime("%Y-%m-%d %H:%M:%S", gmtime()) + } + + # one iteration to find the latest version for each release + for v in json_response: + # latest version for each release + if not re.match(RE['os'], v): + if re.match(RE['alpha'], v): + if v > lv['alpha']['latest_version']: + # we'll use the latest one + lv['alpha']['latest_version'] = v + + elif re.match(RE['beta'], v): + if v > lv['beta']['latest_version']: + # we'll use the latest one + lv['beta']['latest_version'] = v + + elif re.match(RE['stable'], v): + if v > lv['stable']['latest_version']: + # we'll use the latest one + lv['stable']['latest_version'] = v + + latest_alpha = lv['alpha']['latest_version'] + latest_beta = lv['beta']['latest_version'] + latest_stable = lv['stable']['latest_version'] + + # another iteration to add the links + for v in json_response: + # based on current RecommendedTBBVersions scheme + # for each release and for each os we build links for all locales + if re.match(RE['os'], v): + m = re.match(RE['os'], v) + version = m.group(1) + osys = m.group(2) + + if osys in OS: + if latest_alpha and version == latest_alpha \ + and re.match(RE['alpha'], version): + lv['alpha']['downloads'][OS[osys]] = {} + self._add_links(lv, 'alpha', version, OS[osys]) + + elif latest_beta and version == latest_beta \ + and re.match(RE['beta'], version): + lv['beta']['downloads'][OS[osys]] = {} + self._add_links(lv, 'beta', version, OS[osys]) + + elif latest_stable and version == latest_stable \ + and re.match(RE['stable'], version): + lv['stable']['downloads'][OS[osys]] = {} + self._add_links(lv, 'stable', version, OS[osys]) + + lv['updated_at'] = strftime("%Y-%m-%d %H:%M:%S", gmtime()) + self.lv = lv + + def _load_links(self): + """ Load links and providers data. """ + links_files = [] + + # look for files ending with .links in links_path + p = re.compile('.*\.links$') + for name in os.listdir(self.links_path): + path = os.path.abspath(os.path.join(self.links_path, name)) + if os.path.isfile(path) and p.match(path): + links_files.append(path) + + links = {} + providers = {} + supported_os = self.core.get_supported_os() + supported_lc = self.core.get_supported_lc() + + for name in links_files: + config = ConfigParser.ConfigParser() + try: + with open(name) as f: + config.readfp(f) + except IOError: + raise InternalError("File %s not found!" % name) + + try: + pname = config.get('provider', 'name') + pname = self._get_provider_name(pname) + + # build providers dict + providers[pname] = '%s/providers/%s' % (self.server, pname) + providers['updated_at'] = strftime( + "%Y-%m-%d %H:%M:%S", gmtime() + ) + + self.providers = providers + links[pname] = {} + + # build links data. + for osys in supported_os: + links[pname][osys] = {} + for lc in supported_lc: + links[pname][osys][lc] = {} + + for osys in supported_os: + for lc in supported_lc: + l_str = config.get(osys, lc) + + # linux has 32 and 64 bit packages + if osys == 'linux': + l32_str, l64_str = l_str.split(',') + + link32, sig32, sha32 = [ + l for l in l32_str.split("$") if l + ] + + link64, sig64, sha64 = [ + l for l in l64_str.split("$") if l + ] + link64 = link64.lstrip() + + links[pname][osys][lc]['binary32'] = link32 + links[pname][osys][lc]['signature32'] = sig32 + links[pname][osys][lc]['sha256-32'] = sha32 + links[pname][osys][lc]['binary64'] = link64 + links[pname][osys][lc]['signature64'] = sig64 + links[pname][osys][lc]['sha256-64'] = sha64 + + else: + link, sig, sha = [l for l in l_str.split("$") if l] + links[pname][osys][lc]['binary'] = link + links[pname][osys][lc]['signature'] = sig + links[pname][osys][lc]['sha256'] = sha + + except ConfigParser.Error as e: + raise InternalError("%s" % str(e)) + + links['updated_at'] = strftime("%Y-%m-%d %H:%M:%S", gmtime()) + self.links = links + + def _load_mirrors(self): + """ Load mirrors data. """ + mirrors = [] + + # json of mirrors should be obtained from get_mirrors.py + json_data = open(self.mirrors_path).read() + mirrors = json.loads(json_data) + + self.mirrors = mirrors + + def _load_resources(self): + """ Load available resources data. """ + + self.resources = { + 'providers': '%s/providers' % self.server, + 'mirrors': '%s/mirrors' % self.server, + 'latest_version': '%s/latest' % self.server, + 'updated_at': strftime("%Y-%m-%d %H:%M:%S", gmtime()) + } + + def load_data(self): + """ Load all data. + + Since data is not frequently updated, we load all data before + running the RESTful API. Every time the links/mirrors/version + data is updated we should restart the API. + + """ + self._load_links() + self._load_mirrors() + self._load_resources() + self._load_latest_version() + + def run(self): + """ Run RESTful API. """ + app = Flask(__name__) + api = Api(app) + + api.add_resource( + AvailableResources, + '/', + resource_class_kwargs={ + 'resources': self.resources + } + ) + + api.add_resource( + Providers, + '/providers', + '/providers/<string:provider>', + resource_class_kwargs={ + 'links': self.links, + 'providers': self.providers + } + ) + + api.add_resource( + LatestVersion, + '/latest', + '/latest/<string:release>', + resource_class_kwargs={ + 'latest_version': self.lv, + 'releases': self.releases + } + ) + + api.add_resource( + Mirrors, + '/mirrors', + resource_class_kwargs={ + 'mirrors': self.mirrors + } + ) + + app.run(debug=True) + + +class AvailableResources(Resource): + def __init__(self, resources): + """ Set initial data. """ + self.resources = resources + + def get(self): + """ Return available resources on the API. """ + return self.resources + + +class Providers(Resource): + def __init__(self, providers, links): + """ Set initial data. """ + self.providers = providers + self.links = links + + def get(self, provider=None): + """ Return providers and links data. """ + + # we use arg to filter results by os and lc (in that order) + arg = parser.parse_args() + + if provider: + if arg['os']: + if arg['lc']: + # links by provider, os, and lc (in that order) + return self.links[provider][arg['os']][arg['lc']] + else: + # links by provider and os (in that order) + return self.links[provider][arg['os']] + else: + # links by provider + return self.links[provider] + else: + # list of providers + return self.providers + + +class LatestVersion(Resource): + def __init__(self, latest_version, releases): + """ Set initial data. """ + self.lv = latest_version + self.releases = releases + + def get(self, release=None): + """ Return latest version data. """ + + # we use arg to filter results by os and lc (in that order) + arg = parser.parse_args() + + if release: + if arg['os']: + if arg['lc']: + # tpo links by release, os and lc (in that order) + return self.lv[release]['downloads'][arg['os']][arg['lc']] + else: + # tpo links by release and os (in that order) + return self.lv[release]['downloads'][arg['os']] + else: + # version and tpo links by release + return self.lv[release] + else: + # list of releases + return self.releases + + +class Mirrors(Resource): + def __init__(self, mirrors): + """ Set initial data. """ + self.mirrors = mirrors + + def get(self): + """ Return mirrors data. """ + return self.mirrors diff --git a/process_http.py b/process_http.py new file mode 100644 index 0000000..31475f1 --- /dev/null +++ b/process_http.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import gettor.http + + +def main(): + api = gettor.http.HTTP('http.cfg') + api.load_data() + api.run() + +if __name__ == '__main__': + main()

1 0

[gettor/master] Update signature body for GetTor (see 17424#comment:6)
by ilv＠torproject.org 15 Feb '16

15 Feb '16

commit b2b07f3eabfdc2672e55124cd9d633e8a58fca93 Author: Sukhbir Singh <sukhbir(a)torproject.org> Date: Thu Dec 17 23:58:36 2015 -0500 Update signature body for GetTor (see 17424#comment:6) --- lang/smtp/i18n/en/LC_MESSAGES/en.po | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lang/smtp/i18n/en/LC_MESSAGES/en.po b/lang/smtp/i18n/en/LC_MESSAGES/en.po index f987648..13ea727 100644 --- a/lang/smtp/i18n/en/LC_MESSAGES/en.po +++ b/lang/smtp/i18n/en/LC_MESSAGES/en.po @@ -30,9 +30,8 @@ does not work for you, try the next one.\n\ %s\n\ \n\ \n\ -All the best.\n\ --\n\ -GetTor team" +GetTor" #: Mirrors message msgid "mirrors_msg"

1 0

[gettor/master] Change from flask to build an static API
by ilv＠torproject.org 15 Feb '16

15 Feb '16

commit 3cc92b1b881ffdc2b6c441b44455560c11a6f907 Author: ilv <ilv(a)users.noreply.github.com> Date: Thu Dec 31 17:16:36 2015 -0300 Change from flask to build an static API --- gettor/http.py | 249 +++++++++++++++++++++++++++++--------------------------- process_http.py | 7 +- 2 files changed, 135 insertions(+), 121 deletions(-) diff --git a/gettor/http.py b/gettor/http.py index c1441c6..2a63502 100644 --- a/gettor/http.py +++ b/gettor/http.py @@ -19,9 +19,6 @@ import ConfigParser from time import gmtime, strftime -from flask import Flask -from flask_restful import Api, Resource, reqparse - import core import utils @@ -65,10 +62,6 @@ URL = { 'asc': 'https://www.torproject.org/dist/torbrowser/%s/%s.asc' } -# filters for API resources -parser = reqparse.RequestParser() -parser.add_argument('os', type=str, help='Operating System') -parser.add_argument('lc', type=str, help='Locale') class ConfigError(Exception): @@ -100,8 +93,10 @@ class HTTP(object): raise ConfigError("File %s not found!" % cfg) try: + # path to static tree of API + self.tree = config.get('general', 'tree') # server that provides the RESTful API - self.server = config.get('general', 'server') + self.server = config.get('general', 'url') # path to the links files self.links_path = config.get('general', 'links') # path to mirrors in json @@ -129,6 +124,29 @@ class HTTP(object): except ValueError, e: return False return True + + def _write_json(self, path, content): + """ + """ + try: + with codecs.open( + path, + "w", + "utf-8" + ) as jsonfile: + # Make pretty json + json.dump( + content, + jsonfile, + sort_keys=True, + indent=4, + separators=(',', ': '), + encoding="utf-8", + ) + except IOError as e: + #logging.error("Couldn't write json: %s" % str(e)) + print "Error building %s: %s" % (path, str(e)) + print "%s built" % path def _get_provider_name(self, p): """ Return simplified version of provider's name. @@ -373,121 +391,116 @@ class HTTP(object): self._load_resources() self._load_latest_version() - def run(self): - """ Run RESTful API. """ - app = Flask(__name__) - api = Api(app) - - api.add_resource( - AvailableResources, - '/', - resource_class_kwargs={ - 'resources': self.resources - } + def build(self): + """ Build RESTful API. """ + + print "Building API" + + # resources + self._write_json( + os.path.join(self.tree, 'api'), + self.resources ) - api.add_resource( - Providers, - '/providers', - '/providers/<string:provider>', - resource_class_kwargs={ - 'links': self.links, - 'providers': self.providers - } + api_path = os.path.join(self.tree, 'api-content') + if not os.path.isdir(api_path): + os.mkdir(api_path) + + # providers + self._write_json( + os.path.join(api_path, 'providers'), + self.providers ) - api.add_resource( - LatestVersion, - '/latest', - '/latest/<string:release>', - resource_class_kwargs={ - 'latest_version': self.lv, - 'releases': self.releases - } - ) + providers_path = os.path.join(api_path, 'providers-content') + if not os.path.isdir(providers_path): + os.mkdir(providers_path) + + for provider in self.links: + if provider == 'updated_at': + continue + + self._write_json( + os.path.join(providers_path, provider), + self.links[provider] + ) + + provider_path = os.path.join( + providers_path, + "%s-content" % provider + ) + + if not os.path.isdir(provider_path): + os.mkdir(provider_path) + + for osys in self.links[provider]: + self._write_json( + os.path.join(provider_path, osys), + self.links[provider][osys] + ) - api.add_resource( - Mirrors, - '/mirrors', - resource_class_kwargs={ - 'mirrors': self.mirrors - } + provider_os_path = os.path.join( + provider_path, "%s-content" % osys + ) + + if not os.path.isdir(provider_os_path): + os.mkdir(provider_os_path) + + for lc in self.links[provider][osys]: + self._write_json( + os.path.join(provider_os_path, lc), + self.links[provider][osys][lc] + ) + + # latest version + self._write_json( + os.path.join(api_path, 'latest'), + self.lv ) + + lv_path = os.path.join(api_path, 'latest-content') + if not os.path.isdir(lv_path): + os.mkdir(lv_path) + + for release in self.lv: + if release == 'updated_at': + continue + + self._write_json( + os.path.join(lv_path, release), + self.lv[release] + ) + + release_path = os.path.join( + lv_path, + "%s-content" % release + ) + + if not os.path.isdir(release_path): + os.mkdir(release_path) + + for osys in self.lv[release]['downloads']: + self._write_json( + os.path.join(release_path, osys), + self.lv[release]['downloads'][osys] + ) - app.run(debug=True) - - -class AvailableResources(Resource): - def __init__(self, resources): - """ Set initial data. """ - self.resources = resources - - def get(self): - """ Return available resources on the API. """ - return self.resources - - -class Providers(Resource): - def __init__(self, providers, links): - """ Set initial data. """ - self.providers = providers - self.links = links - - def get(self, provider=None): - """ Return providers and links data. """ - - # we use arg to filter results by os and lc (in that order) - arg = parser.parse_args() - - if provider: - if arg['os']: - if arg['lc']: - # links by provider, os, and lc (in that order) - return self.links[provider][arg['os']][arg['lc']] - else: - # links by provider and os (in that order) - return self.links[provider][arg['os']] - else: - # links by provider - return self.links[provider] - else: - # list of providers - return self.providers - - -class LatestVersion(Resource): - def __init__(self, latest_version, releases): - """ Set initial data. """ - self.lv = latest_version - self.releases = releases - - def get(self, release=None): - """ Return latest version data. """ - - # we use arg to filter results by os and lc (in that order) - arg = parser.parse_args() - - if release: - if arg['os']: - if arg['lc']: - # tpo links by release, os and lc (in that order) - return self.lv[release]['downloads'][arg['os']][arg['lc']] - else: - # tpo links by release and os (in that order) - return self.lv[release]['downloads'][arg['os']] - else: - # version and tpo links by release - return self.lv[release] - else: - # list of releases - return self.releases - - -class Mirrors(Resource): - def __init__(self, mirrors): - """ Set initial data. """ - self.mirrors = mirrors + release_os_path = os.path.join( + release_path, + "%s-content" % osys + ) - def get(self): - """ Return mirrors data. """ - return self.mirrors + if not os.path.isdir(release_os_path): + os.mkdir(release_os_path) + + for lc in self.lv[release]['downloads'][osys]: + self._write_json( + os.path.join(release_os_path, lc), + self.lv[release]['downloads'][osys][lc] + ) + + # mirrors + self._write_json( + os.path.join(api_path, 'mirrors'), + self.mirrors + ) diff --git a/process_http.py b/process_http.py index 31475f1..ec20d5f 100644 --- a/process_http.py +++ b/process_http.py @@ -1,13 +1,14 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import gettor.http +import gettor.http2 def main(): - api = gettor.http.HTTP('http.cfg') + api = gettor.http2.HTTP('http.cfg') api.load_data() - api.run() + # api.run() + api.build() if __name__ == '__main__': main()

1 0

[gettor/master] Script and sample htaccess for creating GetTor API static mirror
by ilv＠torproject.org 15 Feb '16

15 Feb '16

commit 7735f5b69383ce26da39dc0dcdfd82f364ec10b8 Author: ilv <ilv(a)users.noreply.github.com> Date: Thu Dec 31 18:46:29 2015 -0300 Script and sample htaccess for creating GetTor API static mirror --- scripts/create_api_mirror.py | 305 +++++++++++++++++++++++++++++++++++++ scripts/sample-htaccess-api-mirror | 21 +++ 2 files changed, 326 insertions(+) diff --git a/scripts/create_api_mirror.py b/scripts/create_api_mirror.py new file mode 100644 index 0000000..e80ec4a --- /dev/null +++ b/scripts/create_api_mirror.py @@ -0,0 +1,305 @@ +# -*- coding: utf-8 -*- +# +# This file is part of GetTor. +# +# :authors: Israel Leiva <ilv(a)torproject.org> +# see also AUTHORS file +# +# :copyright: (c) 2008-2015, The Tor Project, Inc. +# (c) 2015, Israel Leiva +# +# :license: This is Free Software. See LICENSE for license information. + +import os +import re +import json +import codecs +import urllib2 +import ConfigParser + +from time import gmtime, strftime + + +"""Script to build a static mirror of GetTor RESTful API""" + +# currently supported locales for Tor Browser +LC = ['ar', 'de', 'en-US', 'es-ES', 'fa', 'fr', 'it', 'ko', 'nl', 'pl', + 'pt-PT', 'ru', 'tr', 'vi', 'zh-CN'] + +# https://gitweb.tpo/tor-browser-spec.git/tree/processes/VersionNumbers +# does not say anything about operating systems, so it's possible the +# notation might change in the future. We should always use the same three +# strings though: linux, windows, osx. +OS = { + 'Linux': 'linux', + 'Windows': 'windows', + 'MacOS': 'osx' +} + +# based on +# https://gitweb.tpo.org/tor-browser-spec.git/tree/processes/VersionNumbers +# except for the first one, which is based on current RecommendedTBBVersions +RE = { + 'os': '(.*)-(\w+)', + 'alpha': '\d\.\d(\.\d)*a\d+', + 'beta': '\d\.\d(\.\d)*b\d+', + 'stable': '\d\.\d(\.\d)*' +} + +# strings to build names of packages depending on OS. +PKG = { + 'windows': 'torbrowser-install-%s_%s.exe', + 'linux': 'tor-browser-linux%s-%s_%s.tar.xz', + 'osx': 'TorBrowser-%s-osx64_%s.dmg' +} + +# bin and asc are used to build the download links for each version, os and lc +URL = { + 'version': 'https://gettor.torproject.org/api/latest.json', + 'mirrors': 'https://gettor.torproject.org/api/mirrors.json', + 'providers': 'https://gettor.torproject.org/api/providers.json' +} + +GETTOR_URL = 'http://gettor.torproject.org' +SERVER_URL = 'https://yourprivateserver.com' +API_TREE = '/home/ilv/Proyectos/tor/static_api/' + +class ConfigError(Exception): + pass + + +class InternalError(Exception): + pass + + +class APIMirror(object): + """ Provide useful resources via RESTful API. """ + def __init__(self, cfg=None): + self.url = SERVER_URL + self.tree = API_TREE + + def _is_json(self, my_json): + """ Check if json generated is valid. + + :param: my_json (string) data to ve verified. + + :return: (bool) true if data is json-valid, false otherwise. + + """ + try: + json_object = json.loads(my_json) + except ValueError, e: + return False + return True + + def _write_json(self, path, content): + """ + """ + try: + with codecs.open( + path, + "w", + "utf-8" + ) as jsonfile: + # Make pretty json + json.dump( + content, + jsonfile, + sort_keys=True, + indent=4, + separators=(',', ': '), + encoding="utf-8", + ) + except IOError as e: + #logging.error("Couldn't write json: %s" % str(e)) + print "Error building %s: %s" % (path, str(e)) + print "%s built" % path + + def _get_provider_name(self, p): + """ Return simplified version of provider's name. + + :param: p (string) provider's name. + + :return: (string) provider's name in lowercase and without spaces. + + """ + p = p.replace(' ', '-') + return p.lower() + + + def _load_latest_version(self): + """ Load latest version data from GetTor API. """ + response = urllib2.urlopen(URL['version']) + json_response = json.load(response) + self.lv = json_response + + def _load_links(self): + """ Load links and providers data. """ + + response = urllib2.urlopen(URL['providers']) + json_response = json.load(response) + + links = {} + + for provider in json_response: + if provider != 'updated_at': + provider_url = json_response[provider] + + provider_response = urllib2.urlopen("%s.json" % provider_url) + provider_content = json.load(provider_response) + + json_response[provider] = provider_url.replace( + GETTOR_URL, + self.url + ) + + + links[provider] = provider_content + + self.providers = json_response + self.links = links + + def _load_mirrors(self): + """ Load mirrors data from GetTor API. """ + response = urllib2.urlopen(URL['mirrors']) + json_response = json.load(response) + self.mirrors = json_response + + def _load_resources(self): + """ Load available resources data. """ + + self.resources = { + 'providers': '%s/providers' % self.url, + 'mirrors': '%s/mirrors' % self.url, + 'latest_version': '%s/latest' % self.url, + 'updated_at': strftime("%Y-%m-%d %H:%M:%S", gmtime()) + } + + def load_data(self): + """ Load all data.""" + self._load_links() + self._load_mirrors() + self._load_resources() + self._load_latest_version() + + def build(self): + """ Build RESTful API. """ + + print "Building API mirror" + + # resources + self._write_json( + os.path.join(self.tree, 'api'), + self.resources + ) + + api_path = os.path.join(self.tree, 'api-content') + if not os.path.isdir(api_path): + os.mkdir(api_path) + + # providers + self._write_json( + os.path.join(api_path, 'providers'), + self.providers + ) + + providers_path = os.path.join(api_path, 'providers-content') + if not os.path.isdir(providers_path): + os.mkdir(providers_path) + + for provider in self.links: + if provider == 'updated_at': + continue + + self._write_json( + os.path.join(providers_path, provider), + self.links[provider] + ) + + provider_path = os.path.join( + providers_path, + "%s-content" % provider + ) + + if not os.path.isdir(provider_path): + os.mkdir(provider_path) + + for osys in self.links[provider]: + self._write_json( + os.path.join(provider_path, osys), + self.links[provider][osys] + ) + + provider_os_path = os.path.join( + provider_path, "%s-content" % osys + ) + + if not os.path.isdir(provider_os_path): + os.mkdir(provider_os_path) + + for lc in self.links[provider][osys]: + self._write_json( + os.path.join(provider_os_path, lc), + self.links[provider][osys][lc] + ) + + # latest version + self._write_json( + os.path.join(api_path, 'latest'), + self.lv + ) + + lv_path = os.path.join(api_path, 'latest-content') + if not os.path.isdir(lv_path): + os.mkdir(lv_path) + + for release in self.lv: + if release == 'updated_at': + continue + + self._write_json( + os.path.join(lv_path, release), + self.lv[release] + ) + + release_path = os.path.join( + lv_path, + "%s-content" % release + ) + + if not os.path.isdir(release_path): + os.mkdir(release_path) + + for osys in self.lv[release]['downloads']: + self._write_json( + os.path.join(release_path, osys), + self.lv[release]['downloads'][osys] + ) + + release_os_path = os.path.join( + release_path, + "%s-content" % osys + ) + + if not os.path.isdir(release_os_path): + os.mkdir(release_os_path) + + for lc in self.lv[release]['downloads'][osys]: + self._write_json( + os.path.join(release_os_path, lc), + self.lv[release]['downloads'][osys][lc] + ) + + # mirrors + self._write_json( + os.path.join(api_path, 'mirrors'), + self.mirrors + ) + +def main(): + api = APIMirror() + api.load_data() + api.build() + +if __name__ == '__main__': + main() diff --git a/scripts/sample-htaccess-api-mirror b/scripts/sample-htaccess-api-mirror new file mode 100644 index 0000000..41594db --- /dev/null +++ b/scripts/sample-htaccess-api-mirror @@ -0,0 +1,21 @@ +Options +FollowSymLinks +RewriteEngine On + +# Operating System and Locale filters for providers +RewriteCond %{QUERY_STRING} os=([a-z]+)&lc=([a-z]+) +RewriteRule ^api/providers/([a-z\-]+)$ api-content/providers-content/$1-content/%1-content/%2.json +RewriteCond %{QUERY_STRING} os=([a-z]+) +RewriteRule ^api/providers/([a-z\-]+)$ api-content/providers-content/$1-content/%1 + +# Operating System and Locale filters for TPO downloads +RewriteCond %{QUERY_STRING} os=([a-z]+)&lc=([a-zA-Z\-]+) +RewriteRule ^api/latest/([a-z]+)$ api-content/latest-content/$1-content/%1-content/%2.json +RewriteCond %{QUERY_STRING} os=([a-z]+) +RewriteRule ^api/latest/([a-z]+)$ api-content/latest-content/$1-content/%1 + +# General rules +RewriteRule ^api/providers/([a-z\-]+)$ api-content/providers-content/$1 [NC] +RewriteRule ^api/latest/([a-z]+)$ api-content/latest-content/$1 [NC] +RewriteRule ^api/([a-z]+)$ api-content/$1 [NC] + +ErrorDocument 404 404-file

1 0

[metrics-db/master] Add more robust descriptor reference checker.
by karsten＠torproject.org 15 Feb '16

15 Feb '16

commit 56288d49746ace3fdc03411c79f7d9b52d66c35d Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Feb 11 22:41:52 2016 +0100 Add more robust descriptor reference checker. The new reference checker reads files in recent/relay-descriptors/, rather than keeping data structures of newly written descriptors in memory and storing them to disk later. This should be more robust against unplanned reboots. --- .../ernie/db/relaydescs/ArchiveWriter.java | 4 + .../ernie/db/relaydescs/ReferenceChecker.java | 310 +++++++++++++++++++++ 2 files changed, 314 insertions(+) diff --git a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java index 90b41c3..01b4d06 100644 --- a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java +++ b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java @@ -57,6 +57,10 @@ public class ArchiveWriter extends Thread { // Import/download relay descriptors from the various sources new ArchiveWriter(config).run(); + new ReferenceChecker(new File("recent/relay-descriptors"), + new File("stats/references"), + new File("stats/references-history")).check(); + // Remove lock file lf.releaseLock(); diff --git a/src/org/torproject/ernie/db/relaydescs/ReferenceChecker.java b/src/org/torproject/ernie/db/relaydescs/ReferenceChecker.java new file mode 100644 index 0000000..4bafa76 --- /dev/null +++ b/src/org/torproject/ernie/db/relaydescs/ReferenceChecker.java @@ -0,0 +1,310 @@ +package org.torproject.ernie.db.relaydescs; + +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Locale; +import java.util.Set; +import java.util.SortedSet; +import java.util.TimeZone; +import java.util.TreeSet; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.DescriptorFile; +import org.torproject.descriptor.DescriptorReader; +import org.torproject.descriptor.DescriptorSourceFactory; +import org.torproject.descriptor.DirSourceEntry; +import org.torproject.descriptor.ExtraInfoDescriptor; +import org.torproject.descriptor.Microdescriptor; +import org.torproject.descriptor.NetworkStatusEntry; +import org.torproject.descriptor.RelayNetworkStatusConsensus; +import org.torproject.descriptor.RelayNetworkStatusVote; +import org.torproject.descriptor.ServerDescriptor; + +import com.google.gson.Gson; + +public class ReferenceChecker { + + private Logger log = Logger.getLogger(ReferenceChecker.class.getName()); + + private File descriptorsDir; + + private File referencesFile; + + private File historyFile; + + public ReferenceChecker(File descriptorsDir, File referencesFile, + File historyFile) { + this.descriptorsDir = descriptorsDir; + this.referencesFile = referencesFile; + this.historyFile = historyFile; + } + + public void check() { + this.getCurrentTimeMillis(); + this.readReferencesFile(); + this.readNewDescriptors(); + this.dropStaleReferences(); + this.checkReferences(); + this.writeReferencesFile(); + } + + private long currentTimeMillis; + + private void getCurrentTimeMillis() { + this.currentTimeMillis = System.currentTimeMillis(); + } + + private static class Reference implements Comparable<Reference> { + + private String referencing; + + private String referenced; + + private double weight; + + private long expiresAfterMillis; + + public Reference(String referencing, String referenced, double weight, + long expiresAfterMillis) { + this.referencing = referencing; + this.referenced = referenced; + this.weight = weight; + this.expiresAfterMillis = expiresAfterMillis; + } + + @Override + public boolean equals(Object otherObject) { + if (!(otherObject instanceof Reference)) { + return false; + } + Reference other = (Reference) otherObject; + return this.referencing.equals(other.referencing) && + this.referenced.equals(other.referenced); + } + + @Override + public int hashCode() { + return this.referencing.hashCode() + this.referenced.hashCode(); + } + + @Override + public int compareTo(Reference other) { + int result = this.referencing.compareTo(other.referencing); + if (result == 0) { + result = this.referenced.compareTo(other.referenced); + } + return result; + } + } + + private SortedSet<Reference> references = new TreeSet<Reference>(); + + private void addReference(String referencing, String referenced, + double weight, long expiresAfterMillis) { + this.references.add(new Reference(referencing.toUpperCase(), + referenced.toUpperCase(), weight, expiresAfterMillis)); + } + + private void readReferencesFile() { + if (!this.referencesFile.exists()) { + return; + } + Gson gson = new Gson(); + try { + FileReader fr = new FileReader(this.referencesFile); + this.references.addAll(Arrays.asList(gson.fromJson(fr, + Reference[].class))); + fr.close(); + } catch (IOException e) { + this.log.log(Level.WARNING, "Cannot read existing references file " + + "from previous run.", e); + } + } + + private void readNewDescriptors() { + DescriptorReader descriptorReader = + DescriptorSourceFactory.createDescriptorReader(); + descriptorReader.addDirectory(this.descriptorsDir); + descriptorReader.setExcludeFiles(this.historyFile); + Iterator<DescriptorFile> descriptorFiles = + descriptorReader.readDescriptors(); + while (descriptorFiles.hasNext()) { + DescriptorFile descriptorFile = descriptorFiles.next(); + for (Descriptor descriptor : descriptorFile.getDescriptors()) { + if (descriptor instanceof RelayNetworkStatusConsensus) { + RelayNetworkStatusConsensus consensus = + (RelayNetworkStatusConsensus) descriptor; + String consensusFlavor = consensus.getConsensusFlavor(); + if (consensusFlavor == null) { + this.readRelayNetworkStatusConsensusUnflavored(consensus); + } else if (consensusFlavor.equals("microdesc")) { + this.readRelayNetworkStatusConsensusMicrodesc(consensus); + } else { + /* Ignore unknown consensus flavors. */ + } + } else if (descriptor instanceof RelayNetworkStatusVote) { + this.readRelayNetworkStatusVote( + (RelayNetworkStatusVote) descriptor); + } else if (descriptor instanceof ServerDescriptor) { + this.readServerDescriptor((ServerDescriptor) descriptor); + } else if (descriptor instanceof ExtraInfoDescriptor) { + this.readExtraInfoDescriptor((ExtraInfoDescriptor) descriptor); + } else if (descriptor instanceof Microdescriptor) { + readMicrodescriptor((Microdescriptor) descriptor); + } else { + /* Ignore unknown descriptors. */ + } + } + } + } + + private static DateFormat dateTimeFormat; + static { + dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", + Locale.US); + dateTimeFormat.setLenient(false); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + } + + private static final long ONE_HOUR = 60L * 60L * 1000L, + THREE_HOURS = 3L * ONE_HOUR, SIX_HOURS = 6L * ONE_HOUR, + ONE_DAY = 24L * ONE_HOUR, THIRTY_DAYS = 30L * ONE_DAY; + + private void readRelayNetworkStatusConsensusUnflavored( + RelayNetworkStatusConsensus consensus) { + String validAfter = dateTimeFormat.format( + consensus.getValidAfterMillis()); + String referencing = String.format("C-%s", validAfter); + this.addReference(referencing, String.format("M-%s", validAfter), 1.0, + consensus.getValidAfterMillis() + THREE_HOURS); + for (DirSourceEntry dirSourceEntry : + consensus.getDirSourceEntries().values()) { + if (!dirSourceEntry.isLegacy()) { + this.addReference(referencing, String.format("V-%s-%s", + validAfter, dirSourceEntry.getIdentity()), 1.0, + consensus.getValidAfterMillis() + THREE_HOURS); + } + } + double entryWeight = 200.0 / + ((double) consensus.getStatusEntries().size()); + for (NetworkStatusEntry entry : + consensus.getStatusEntries().values()) { + this.addReference(referencing, + String.format("S-%s", entry.getDescriptor()), entryWeight, + entry.getPublishedMillis() + THREE_HOURS); + } + } + + + private void readRelayNetworkStatusConsensusMicrodesc( + RelayNetworkStatusConsensus consensus) { + String validAfter = dateTimeFormat.format( + consensus.getValidAfterMillis()); + String referencing = String.format("M-%s", validAfter); + this.addReference(referencing, String.format("C-%s", validAfter), 1.0, + consensus.getValidAfterMillis() + THREE_HOURS); + double entryWeight = 200.0 / + ((double) consensus.getStatusEntries().size()); + for (NetworkStatusEntry entry : + consensus.getStatusEntries().values()) { + for (String digest : entry.getMicrodescriptorDigests()) { + this.addReference(referencing, String.format("D-%s", digest), + entryWeight, entry.getPublishedMillis() + THREE_HOURS); + } + } + } + + private void readRelayNetworkStatusVote(RelayNetworkStatusVote vote) { + String validAfter = dateTimeFormat.format(vote.getValidAfterMillis()); + String referencing = String.format("V-%s-%s", validAfter, + vote.getIdentity()); + double entryWeight = 200.0 / + ((double) vote.getStatusEntries().size()); + for (NetworkStatusEntry entry : vote.getStatusEntries().values()) { + this.addReference(referencing, + String.format("S-%s", entry.getDescriptor()), entryWeight, + entry.getPublishedMillis() + SIX_HOURS); + } + } + + private void readServerDescriptor(ServerDescriptor serverDescriptor) { + String referenced = serverDescriptor.getExtraInfoDigest() == null ? "" + : String.format("E-%s", serverDescriptor.getExtraInfoDigest()); + this.addReference(String.format("S-%s", + serverDescriptor.getServerDescriptorDigest()), referenced, 0.01, + serverDescriptor.getPublishedMillis() + SIX_HOURS); + } + + private void readExtraInfoDescriptor( + ExtraInfoDescriptor extraInfoDescriptor) { + this.addReference(String.format("E-%s", + extraInfoDescriptor.getExtraInfoDigest()), "", 0.005, + extraInfoDescriptor.getPublishedMillis() + SIX_HOURS); + } + + private void readMicrodescriptor(Microdescriptor microdesc) { + this.addReference( + String.format("D-%s", microdesc.getMicrodescriptorDigest()), "", + 0.0, this.currentTimeMillis + THIRTY_DAYS); + } + + private void dropStaleReferences() { + SortedSet<Reference> recentReferences = new TreeSet<Reference>(); + for (Reference reference : this.references) { + if (this.currentTimeMillis <= reference.expiresAfterMillis) { + recentReferences.add(reference); + } + } + this.references = recentReferences; + } + + private void checkReferences() { + Set<String> knownDescriptors = new HashSet<String>(); + for (Reference reference : this.references) { + knownDescriptors.add(reference.referencing); + } + double totalMissingDescriptorsWeight = 0.0; + Set<String> missingDescriptors = new TreeSet<String>(); + StringBuilder sb = new StringBuilder("Missing referenced " + + "descriptors:"); + for (Reference reference : this.references) { + if (reference.referenced.length() > 0 && + !knownDescriptors.contains(reference.referenced)) { + if (!missingDescriptors.contains(reference.referenced)) { + totalMissingDescriptorsWeight += reference.weight; + } + missingDescriptors.add(reference.referenced); + sb.append(String.format("%n%s -> %s (%.4f -> %.4f)", + reference.referencing, reference.referenced, reference.weight, + totalMissingDescriptorsWeight)); + } + } + this.log.log(Level.INFO, sb.toString()); + if (totalMissingDescriptorsWeight > 0.999) { + this.log.log(Level.WARNING, "Missing too many referenced " + + "descriptors (" + totalMissingDescriptorsWeight + ")."); + } + } + + private void writeReferencesFile() { + Gson gson = new Gson(); + try { + FileWriter fw = new FileWriter(this.referencesFile); + gson.toJson(this.references, fw); + fw.close(); + } catch (IOException e) { + this.log.log(Level.WARNING, "Cannot write references file for next " + + "run.", e); + } + } +} +

1 0

[metrics-db/master] Keep microdescriptors in recent/ for up to 30 days.
by karsten＠torproject.org 15 Feb '16

15 Feb '16

commit 7f0b062bcd0dee7658092784706cec7b860a1b66 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Feb 12 11:06:04 2016 +0100 Keep microdescriptors in recent/ for up to 30 days. We'll need to keep microdescriptors around longer than three days, because they change less frequently and are referenced much longer than, for example, server descriptors. --- src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java index 01b4d06..a85408c 100644 --- a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java +++ b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java @@ -802,17 +802,23 @@ public class ArchiveWriter extends Thread { } /* Delete all files from the rsync directory that have not been modified - * in the last three days, and remove the .tmp extension from newly + * in the last three days (except for microdescriptors which are kept + * for up to thirty days), and remove the .tmp extension from newly * written files. */ public void cleanUpRsyncDirectory() { long cutOffMillis = System.currentTimeMillis() - 3L * 24L * 60L * 60L * 1000L; + long cutOffMicroMillis = cutOffMillis - 27L * 24L * 60L * 60L * 1000L; Stack<File> allFiles = new Stack<File>(); allFiles.add(new File("recent/relay-descriptors")); while (!allFiles.isEmpty()) { File file = allFiles.pop(); if (file.isDirectory()) { allFiles.addAll(Arrays.asList(file.listFiles())); + } else if (file.getName().endsWith("-micro")) { + if (file.lastModified() < cutOffMicroMillis) { + file.delete(); + } } else if (file.lastModified() < cutOffMillis) { file.delete(); } else if (file.getName().endsWith(".tmp")) {

1 0