commit da53fc70b6871c13c5821bf588113943a49baeb3 Author: hiro hiro@torproject.org Date: Fri Mar 8 10:52:25 2019 +0100
Create script to add links to db --- bin/gettor_service | 8 +++++ gettor/__init__.py | 4 ++- gettor/main.py | 6 ++-- gettor/services/email/sendmail.py | 21 +++++-------- gettor/services/twitter/twitter.py | 5 ++- gettor/services/xmpp/xmpp.py | 5 ++- gettor/utils/db.py | 36 +++++++++++++++------ gettor/utils/options.py | 6 +++- gettor/utils/strings.py | 9 +++++- gettor/web/http.py | 27 +++++++++------- scripts/add_lins_to_db | 64 ++++++++++++++++++++++++++++++++++++++ share/locale/en.json | 2 ++ 12 files changed, 152 insertions(+), 41 deletions(-)
diff --git a/bin/gettor_service b/bin/gettor_service index ae85f56..3c1ac04 100755 --- a/bin/gettor_service +++ b/bin/gettor_service @@ -10,6 +10,14 @@ # # :license: This is Free Software. See LICENSE for license information.
+ +################################################################################ +# # +# This is how GetTor is started as a twisted application. # +# The script allows to start, stop, restart GetTor and get its status. # +# # +################################################################################ + case "$1" in start) twistd --python=scripts/gettor --logfile=log/gettor.log --pidfile=gettor.pid diff --git a/gettor/__init__.py b/gettor/__init__.py index bf73506..74a0363 100644 --- a/gettor/__init__.py +++ b/gettor/__init__.py @@ -12,6 +12,8 @@ the Tor Browser.
from .utils import strings
- +""" +This is where version and available locales get loaded. +""" __version__ = strings.get_version() __locales__ = strings.get_locales() diff --git a/gettor/main.py b/gettor/main.py index 53fbf61..0932ed5 100644 --- a/gettor/main.py +++ b/gettor/main.py @@ -10,7 +10,9 @@ the Tor Browser. :license: see included LICENSE for information """
-"""This module sets up GetTor and starts the servers running.""" +""" +This sets up GetTor and starts the servers running. +"""
import sys
@@ -22,7 +24,7 @@ from .services.email.sendmail import Sendmail
def run(gettor, app): """ - This is GetTor's main entry point and main runtime loop. + This is GetTor's main entry point and main runtime loop. """ settings = options.parse_settings()
diff --git a/gettor/services/email/sendmail.py b/gettor/services/email/sendmail.py index 8c86bdc..92aa766 100644 --- a/gettor/services/email/sendmail.py +++ b/gettor/services/email/sendmail.py @@ -103,7 +103,7 @@ class Sendmail(object):
@defer.inlineCallbacks def get_new(self): - """strings.load_strings("en") + """ Get new requests to process. This will define the `main loop` of the Sendmail service. """ @@ -117,18 +117,16 @@ class Sendmail(object): status="ONHOLD", command="links", service="email" )
+ """ + Load strings for translations + """ + # for now just english + strings.load_strings("en")
if help_requests: try: log.info("Got new help request.")
- # for now just english - en = gettext.translation( - 'email', localedir='locales', languages=['en'] - ) - en.install() - _ = en.gettext - for request in help_requests: id = request[0] date = request[4] @@ -142,8 +140,8 @@ class Sendmail(object):
yield self.sendmail( email_addr=id, - subject=_("help_subject"), - body=_("help_body") + subject=strings._("help_subject"), + body=strings._("help_body") )
yield self.conn.update_stats( @@ -162,9 +160,6 @@ class Sendmail(object): try: log.info("Got new links request.")
- # for now just english - strings.load_strings("en") - for request in link_requests: id = request[0] date = request[4] diff --git a/gettor/services/twitter/twitter.py b/gettor/services/twitter/twitter.py index b5e08ba..fcdc353 100644 --- a/gettor/services/twitter/twitter.py +++ b/gettor/services/twitter/twitter.py @@ -22,7 +22,10 @@ import core import utils import blacklist
-"""Twitter channel for distributing links to download Tor Browser.""" +""" +Twitter channel for distributing links to download Tor Browser. +Needs to be refactored to work with twisted and updates to twitter apis +"""
OS = { 'osx': 'Mac OS X', diff --git a/gettor/services/xmpp/xmpp.py b/gettor/services/xmpp/xmpp.py index ff5e207..cfcf8bf 100644 --- a/gettor/services/xmpp/xmpp.py +++ b/gettor/services/xmpp/xmpp.py @@ -28,7 +28,10 @@ import utils import blacklist
-"""XMPP module for processing requests.""" +""" +XMPP module for processing requests. +Needs to be refactored to work with twisted +"""
OS = { 'osx': 'Mac OS X', diff --git a/gettor/utils/db.py b/gettor/utils/db.py index ee4c398..99ae081 100644 --- a/gettor/utils/db.py +++ b/gettor/utils/db.py @@ -16,7 +16,7 @@ from twisted.enterprise import adbapi
class SQLite3(object): """ - + This class handles the database connections and operations. """ def __init__(self, dbname): """Constructor.""" @@ -25,18 +25,26 @@ class SQLite3(object): )
def query_callback(self, results=None): - """ """ + """ + Query callback + Log that the database query has been executed and return results + """ log.msg("Database query executed successfully.") return results
def query_errback(self, error=None): - """ """ + """ + Query error callback + Logs database error + """ if error: log.msg("Database error: {}".format(error)) return None
def new_request(self, id, command, service, platform, date, status): - """ """ + """ + Perform a new request to the database + """ query = "INSERT INTO requests VALUES(?, ?, ?, ?, ?, ?)"
return self.dbpool.runQuery( @@ -44,7 +52,9 @@ class SQLite3(object): ).addCallback(self.query_callback).addErrback(self.query_errback)
def get_requests(self, status, command, service): - """ """ + """ + Perform a SELECT request to the database + """ query = "SELECT * FROM requests WHERE service=? AND command=? AND "\ "status = ?"
@@ -53,7 +63,9 @@ class SQLite3(object): ).addCallback(self.query_callback).addErrback(self.query_errback)
def get_num_requests(self, id, service): - """ """ + """ + Get number of requests for statistics + """ query = "SELECT COUNT(rowid) FROM requests WHERE id=? AND service=?"
return self.dbpool.runQuery( @@ -61,7 +73,9 @@ class SQLite3(object): ).addCallback(self.query_callback).addErrback(self.query_errback)
def update_request(self, id, hid, status, service, date): - """ """ + """ + Update request record in the database + """ query = "UPDATE requests SET id=?, status=? WHERE id=? AND "\ "service=? AND date=?"
@@ -70,7 +84,9 @@ class SQLite3(object): ).addCallback(self.query_callback).addErrback(self.query_errback)
def update_stats(self, command, service, platform=None): - """ """ + """ + Update statistics to the database + """ now_str = datetime.now().strftime("%Y%m%d") query = "REPLACE INTO stats(num_requests, platform, "\ "command, service, date) VALUES(COALESCE((SELECT num_requests FROM stats "\ @@ -81,7 +97,9 @@ class SQLite3(object): ).addCallback(self.query_callback).addErrback(self.query_errback)
def get_links(self, platform, status): - """ """ + """ + Get links from the database per platform + """ query = "SELECT * FROM links WHERE platform=? AND status=?" return self.dbpool.runQuery( query, (platform, status) diff --git a/gettor/utils/options.py b/gettor/utils/options.py index e8e2996..ef4b346 100644 --- a/gettor/utils/options.py +++ b/gettor/utils/options.py @@ -25,6 +25,10 @@ def load_settings(config=None): return settings
def parse_settings(): + """ + Parse settings and loads strings in a given locale + This function needs to be rewritten considering passing a locale and + returing translated strings + """ strings.load_strings("en") - return load_settings(config=False) diff --git a/gettor/utils/strings.py b/gettor/utils/strings.py index 1cf0003..5231dc7 100644 --- a/gettor/utils/strings.py +++ b/gettor/utils/strings.py @@ -72,13 +72,20 @@ def get_resource_path(filename, path): return os.path.join(prefix, filename)
def get_version(): - # The current version + """ + The current version + """ + version = "" with open(get_resource_path('version.txt', '../share')) as f: version = f.read().strip() return version
def get_locales(): + """ + Get available_locales + """ + filename = get_resource_path("available_locales.json", '../share/locale') locales = {} with open(filename, encoding='utf-8') as f: diff --git a/gettor/web/http.py b/gettor/web/http.py index 77120d6..067aef8 100644 --- a/gettor/web/http.py +++ b/gettor/web/http.py @@ -22,7 +22,10 @@ from time import gmtime, strftime import core import utils
-"""GetTor RESTful API""" +""" +GetTor RESTful API +This part of GetTor has not been integrated into twisted-gettor just yet. +"""
# currently supported locales for Tor Browser LC = ['ar', 'de', 'en-US', 'es-ES', 'fa', 'fr', 'it', 'ko', 'nl', 'pl', @@ -123,7 +126,7 @@ class HTTP(object): except ValueError, e: return False return True - + def _write_json(self, path, content): """ """ @@ -376,9 +379,9 @@ class HTTP(object):
def build(self): """ Build RESTful API. """ - + print "Building API" - + # resources self._write_json( os.path.join(self.tree, 'api'), @@ -388,7 +391,7 @@ class HTTP(object): api_path = os.path.join(self.tree, 'api-content') if not os.path.isdir(api_path): os.mkdir(api_path) - + # providers self._write_json( os.path.join(api_path, 'providers'), @@ -415,7 +418,7 @@ class HTTP(object):
if not os.path.isdir(provider_path): os.mkdir(provider_path) - + for osys in self.links[provider]: self._write_json( os.path.join(provider_path, osys), @@ -424,11 +427,11 @@ class HTTP(object):
provider_os_path = os.path.join( provider_path, "%s-content" % osys - ) + )
if not os.path.isdir(provider_os_path): os.mkdir(provider_os_path) - + for lc in self.links[provider][osys]: self._write_json( os.path.join(provider_os_path, lc), @@ -440,7 +443,7 @@ class HTTP(object): os.path.join(api_path, 'latest'), self.lv ) - + lv_path = os.path.join(api_path, 'latest-content') if not os.path.isdir(lv_path): os.mkdir(lv_path) @@ -453,7 +456,7 @@ class HTTP(object): os.path.join(lv_path, release), self.lv[release] ) - + release_path = os.path.join( lv_path, "%s-content" % release @@ -461,7 +464,7 @@ class HTTP(object):
if not os.path.isdir(release_path): os.mkdir(release_path) - + for osys in self.lv[release]['downloads']: self._write_json( os.path.join(release_path, osys), @@ -475,7 +478,7 @@ class HTTP(object):
if not os.path.isdir(release_os_path): os.mkdir(release_os_path) - + for lc in self.lv[release]['downloads'][osys]: self._write_json( os.path.join(release_os_path, lc), diff --git a/scripts/add_lins_to_db b/scripts/add_lins_to_db new file mode 100644 index 0000000..5da92b3 --- /dev/null +++ b/scripts/add_lins_to_db @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# This file is part of GetTor, a Tor Browser distribution system. +# +# :authors: hiro hiro@torproject.org +# see also AUTHORS file +# +# :license: This is Free Software. See LICENSE for license information. + +import os +import sys +import sqlite3 +import urllib import request + +def main(): + args = parser.parse_args() + abs_filename = os.path.abspath(args.filename) + + webFile = request.urlopen("https://lektor-staging.torproject.org/tpo/staging/projects/torbrowser/Recomm...") + versions = webfile.read().decode('utf-8') + version = versions.split(""")[1] + + gitlab = "https://gitlab.com/hiromipaw/torbrowser/raw/releases/" + + prefixes = { + "osx": "TorBrowser-", + "windows": "torbrowser-install-", + "linux": "tor-browser-linux64-" + } + + versions = {"windows": version, 'linux': version, 'osx': version} + + suffixes = { + "osx": "-osx64_en-US.dmg", + "windows": "_en-US.exe", + "linux": "_en-US.tar.xz" + } + + keys = set().union(suffixes, versions, prefixes) + releases = {k: "".join(dic.get(k, version) for dic in (prefixes, versions, suffixes)) for k in keys} + + + if not abs_filename: + print("Missing database filename.") + elif args.new and not args.overwrite and os.path.isfile(abs_filename): + print("Database file already exists.") + elif args.new: + conn = sqlite3.connect(abs_filename) + with conn: + c = conn.cursor() + """ + Here we drop previous links TABLE but probably it would be better to just update old links to INACTIVE + """ + c.execute("DROP TABLE IF EXISTS links") + c.execute( + "CREATE TABLE links(link TEXT, platform TEXT, arch TEXT," + " version TEXT, provider TEXT, status TEXT)" + ) + for k in keys: + c.execute( + "INSERT INTO links(link, platform, arch, version, provider, status)" + "VALUES ('%s', '%s' '64', '%s', 'gitlab', 'ACTIVE')" %(releases.get(key), k, version) + ) diff --git a/share/locale/en.json b/share/locale/en.json index 0e9d47c..8824c2c 100644 --- a/share/locale/en.json +++ b/share/locale/en.json @@ -1,6 +1,8 @@ { "links_body": "GetTor Test. Please be kind.", "links_subject": "GetTor Email Test", + "help_body": "GetTor Help Test. Please be kind.", + "help_subject": "GetTor Help Email Test", "help_debug": "Log application errors to stdout", "help_config": "Custom config file location (optional)", "smtp_links_subject": "[GetTor] Links for your request",