[tor-commits] [gettor/master] Create script to add links to db
hiro at torproject.org
hiro at torproject.org
Fri May 17 14:27:50 UTC 2019
commit da53fc70b6871c13c5821bf588113943a49baeb3
Author: hiro <hiro at torproject.org>
Date: Fri Mar 8 10:52:25 2019 +0100
Create script to add links to db
---
bin/gettor_service | 8 +++++
gettor/__init__.py | 4 ++-
gettor/main.py | 6 ++--
gettor/services/email/sendmail.py | 21 +++++--------
gettor/services/twitter/twitter.py | 5 ++-
gettor/services/xmpp/xmpp.py | 5 ++-
gettor/utils/db.py | 36 +++++++++++++++------
gettor/utils/options.py | 6 +++-
gettor/utils/strings.py | 9 +++++-
gettor/web/http.py | 27 +++++++++-------
scripts/add_lins_to_db | 64 ++++++++++++++++++++++++++++++++++++++
share/locale/en.json | 2 ++
12 files changed, 152 insertions(+), 41 deletions(-)
diff --git a/bin/gettor_service b/bin/gettor_service
index ae85f56..3c1ac04 100755
--- a/bin/gettor_service
+++ b/bin/gettor_service
@@ -10,6 +10,14 @@
#
# :license: This is Free Software. See LICENSE for license information.
+
+################################################################################
+# #
+# This is how GetTor is started as a twisted application. #
+# The script allows to start, stop, restart GetTor and get its status. #
+# #
+################################################################################
+
case "$1" in
start)
twistd --python=scripts/gettor --logfile=log/gettor.log --pidfile=gettor.pid
diff --git a/gettor/__init__.py b/gettor/__init__.py
index bf73506..74a0363 100644
--- a/gettor/__init__.py
+++ b/gettor/__init__.py
@@ -12,6 +12,8 @@ the Tor Browser.
from .utils import strings
-
+"""
+This is where version and available locales get loaded.
+"""
__version__ = strings.get_version()
__locales__ = strings.get_locales()
diff --git a/gettor/main.py b/gettor/main.py
index 53fbf61..0932ed5 100644
--- a/gettor/main.py
+++ b/gettor/main.py
@@ -10,7 +10,9 @@ the Tor Browser.
:license: see included LICENSE for information
"""
-"""This module sets up GetTor and starts the servers running."""
+"""
+This sets up GetTor and starts the servers running.
+"""
import sys
@@ -22,7 +24,7 @@ from .services.email.sendmail import Sendmail
def run(gettor, app):
"""
- This is GetTor's main entry point and main runtime loop.
+ This is GetTor's main entry point and main runtime loop.
"""
settings = options.parse_settings()
diff --git a/gettor/services/email/sendmail.py b/gettor/services/email/sendmail.py
index 8c86bdc..92aa766 100644
--- a/gettor/services/email/sendmail.py
+++ b/gettor/services/email/sendmail.py
@@ -103,7 +103,7 @@ class Sendmail(object):
@defer.inlineCallbacks
def get_new(self):
- """strings.load_strings("en")
+ """
Get new requests to process. This will define the `main loop` of
the Sendmail service.
"""
@@ -117,18 +117,16 @@ class Sendmail(object):
status="ONHOLD", command="links", service="email"
)
+ """
+ Load strings for translations
+ """
+ # for now just english
+ strings.load_strings("en")
if help_requests:
try:
log.info("Got new help request.")
- # for now just english
- en = gettext.translation(
- 'email', localedir='locales', languages=['en']
- )
- en.install()
- _ = en.gettext
-
for request in help_requests:
id = request[0]
date = request[4]
@@ -142,8 +140,8 @@ class Sendmail(object):
yield self.sendmail(
email_addr=id,
- subject=_("help_subject"),
- body=_("help_body")
+ subject=strings._("help_subject"),
+ body=strings._("help_body")
)
yield self.conn.update_stats(
@@ -162,9 +160,6 @@ class Sendmail(object):
try:
log.info("Got new links request.")
- # for now just english
- strings.load_strings("en")
-
for request in link_requests:
id = request[0]
date = request[4]
diff --git a/gettor/services/twitter/twitter.py b/gettor/services/twitter/twitter.py
index b5e08ba..fcdc353 100644
--- a/gettor/services/twitter/twitter.py
+++ b/gettor/services/twitter/twitter.py
@@ -22,7 +22,10 @@ import core
import utils
import blacklist
-"""Twitter channel for distributing links to download Tor Browser."""
+"""
+Twitter channel for distributing links to download Tor Browser.
+Needs to be refactored to work with twisted and updates to twitter apis
+"""
OS = {
'osx': 'Mac OS X',
diff --git a/gettor/services/xmpp/xmpp.py b/gettor/services/xmpp/xmpp.py
index ff5e207..cfcf8bf 100644
--- a/gettor/services/xmpp/xmpp.py
+++ b/gettor/services/xmpp/xmpp.py
@@ -28,7 +28,10 @@ import utils
import blacklist
-"""XMPP module for processing requests."""
+"""
+XMPP module for processing requests.
+Needs to be refactored to work with twisted
+"""
OS = {
'osx': 'Mac OS X',
diff --git a/gettor/utils/db.py b/gettor/utils/db.py
index ee4c398..99ae081 100644
--- a/gettor/utils/db.py
+++ b/gettor/utils/db.py
@@ -16,7 +16,7 @@ from twisted.enterprise import adbapi
class SQLite3(object):
"""
-
+ This class handles the database connections and operations.
"""
def __init__(self, dbname):
"""Constructor."""
@@ -25,18 +25,26 @@ class SQLite3(object):
)
def query_callback(self, results=None):
- """ """
+ """
+ Query callback
+ Log that the database query has been executed and return results
+ """
log.msg("Database query executed successfully.")
return results
def query_errback(self, error=None):
- """ """
+ """
+ Query error callback
+ Logs database error
+ """
if error:
log.msg("Database error: {}".format(error))
return None
def new_request(self, id, command, service, platform, date, status):
- """ """
+ """
+ Perform a new request to the database
+ """
query = "INSERT INTO requests VALUES(?, ?, ?, ?, ?, ?)"
return self.dbpool.runQuery(
@@ -44,7 +52,9 @@ class SQLite3(object):
).addCallback(self.query_callback).addErrback(self.query_errback)
def get_requests(self, status, command, service):
- """ """
+ """
+ Perform a SELECT request to the database
+ """
query = "SELECT * FROM requests WHERE service=? AND command=? AND "\
"status = ?"
@@ -53,7 +63,9 @@ class SQLite3(object):
).addCallback(self.query_callback).addErrback(self.query_errback)
def get_num_requests(self, id, service):
- """ """
+ """
+ Get number of requests for statistics
+ """
query = "SELECT COUNT(rowid) FROM requests WHERE id=? AND service=?"
return self.dbpool.runQuery(
@@ -61,7 +73,9 @@ class SQLite3(object):
).addCallback(self.query_callback).addErrback(self.query_errback)
def update_request(self, id, hid, status, service, date):
- """ """
+ """
+ Update request record in the database
+ """
query = "UPDATE requests SET id=?, status=? WHERE id=? AND "\
"service=? AND date=?"
@@ -70,7 +84,9 @@ class SQLite3(object):
).addCallback(self.query_callback).addErrback(self.query_errback)
def update_stats(self, command, service, platform=None):
- """ """
+ """
+ Update statistics to the database
+ """
now_str = datetime.now().strftime("%Y%m%d")
query = "REPLACE INTO stats(num_requests, platform, "\
"command, service, date) VALUES(COALESCE((SELECT num_requests FROM stats "\
@@ -81,7 +97,9 @@ class SQLite3(object):
).addCallback(self.query_callback).addErrback(self.query_errback)
def get_links(self, platform, status):
- """ """
+ """
+ Get links from the database per platform
+ """
query = "SELECT * FROM links WHERE platform=? AND status=?"
return self.dbpool.runQuery(
query, (platform, status)
diff --git a/gettor/utils/options.py b/gettor/utils/options.py
index e8e2996..ef4b346 100644
--- a/gettor/utils/options.py
+++ b/gettor/utils/options.py
@@ -25,6 +25,10 @@ def load_settings(config=None):
return settings
def parse_settings():
+ """
+ Parse settings and loads strings in a given locale
+ This function needs to be rewritten considering passing a locale and
+ returing translated strings
+ """
strings.load_strings("en")
-
return load_settings(config=False)
diff --git a/gettor/utils/strings.py b/gettor/utils/strings.py
index 1cf0003..5231dc7 100644
--- a/gettor/utils/strings.py
+++ b/gettor/utils/strings.py
@@ -72,13 +72,20 @@ def get_resource_path(filename, path):
return os.path.join(prefix, filename)
def get_version():
- # The current version
+ """
+ The current version
+ """
+
version = ""
with open(get_resource_path('version.txt', '../share')) as f:
version = f.read().strip()
return version
def get_locales():
+ """
+ Get available_locales
+ """
+
filename = get_resource_path("available_locales.json", '../share/locale')
locales = {}
with open(filename, encoding='utf-8') as f:
diff --git a/gettor/web/http.py b/gettor/web/http.py
index 77120d6..067aef8 100644
--- a/gettor/web/http.py
+++ b/gettor/web/http.py
@@ -22,7 +22,10 @@ from time import gmtime, strftime
import core
import utils
-"""GetTor RESTful API"""
+"""
+GetTor RESTful API
+This part of GetTor has not been integrated into twisted-gettor just yet.
+"""
# currently supported locales for Tor Browser
LC = ['ar', 'de', 'en-US', 'es-ES', 'fa', 'fr', 'it', 'ko', 'nl', 'pl',
@@ -123,7 +126,7 @@ class HTTP(object):
except ValueError, e:
return False
return True
-
+
def _write_json(self, path, content):
"""
"""
@@ -376,9 +379,9 @@ class HTTP(object):
def build(self):
""" Build RESTful API. """
-
+
print "Building API"
-
+
# resources
self._write_json(
os.path.join(self.tree, 'api'),
@@ -388,7 +391,7 @@ class HTTP(object):
api_path = os.path.join(self.tree, 'api-content')
if not os.path.isdir(api_path):
os.mkdir(api_path)
-
+
# providers
self._write_json(
os.path.join(api_path, 'providers'),
@@ -415,7 +418,7 @@ class HTTP(object):
if not os.path.isdir(provider_path):
os.mkdir(provider_path)
-
+
for osys in self.links[provider]:
self._write_json(
os.path.join(provider_path, osys),
@@ -424,11 +427,11 @@ class HTTP(object):
provider_os_path = os.path.join(
provider_path, "%s-content" % osys
- )
+ )
if not os.path.isdir(provider_os_path):
os.mkdir(provider_os_path)
-
+
for lc in self.links[provider][osys]:
self._write_json(
os.path.join(provider_os_path, lc),
@@ -440,7 +443,7 @@ class HTTP(object):
os.path.join(api_path, 'latest'),
self.lv
)
-
+
lv_path = os.path.join(api_path, 'latest-content')
if not os.path.isdir(lv_path):
os.mkdir(lv_path)
@@ -453,7 +456,7 @@ class HTTP(object):
os.path.join(lv_path, release),
self.lv[release]
)
-
+
release_path = os.path.join(
lv_path,
"%s-content" % release
@@ -461,7 +464,7 @@ class HTTP(object):
if not os.path.isdir(release_path):
os.mkdir(release_path)
-
+
for osys in self.lv[release]['downloads']:
self._write_json(
os.path.join(release_path, osys),
@@ -475,7 +478,7 @@ class HTTP(object):
if not os.path.isdir(release_os_path):
os.mkdir(release_os_path)
-
+
for lc in self.lv[release]['downloads'][osys]:
self._write_json(
os.path.join(release_os_path, lc),
diff --git a/scripts/add_lins_to_db b/scripts/add_lins_to_db
new file mode 100644
index 0000000..5da92b3
--- /dev/null
+++ b/scripts/add_lins_to_db
@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# This file is part of GetTor, a Tor Browser distribution system.
+#
+# :authors: hiro <hiro at torproject.org>
+# see also AUTHORS file
+#
+# :license: This is Free Software. See LICENSE for license information.
+
+import os
+import sys
+import sqlite3
+import urllib import request
+
+def main():
+ args = parser.parse_args()
+ abs_filename = os.path.abspath(args.filename)
+
+ webFile = request.urlopen("https://lektor-staging.torproject.org/tpo/staging/projects/torbrowser/RecommendedTBBVersions/")
+ versions = webfile.read().decode('utf-8')
+ version = versions.split(""")[1]
+
+ gitlab = "https://gitlab.com/hiromipaw/torbrowser/raw/releases/"
+
+ prefixes = {
+ "osx": "TorBrowser-",
+ "windows": "torbrowser-install-",
+ "linux": "tor-browser-linux64-"
+ }
+
+ versions = {"windows": version, 'linux': version, 'osx': version}
+
+ suffixes = {
+ "osx": "-osx64_en-US.dmg",
+ "windows": "_en-US.exe",
+ "linux": "_en-US.tar.xz"
+ }
+
+ keys = set().union(suffixes, versions, prefixes)
+ releases = {k: "".join(dic.get(k, version) for dic in (prefixes, versions, suffixes)) for k in keys}
+
+
+ if not abs_filename:
+ print("Missing database filename.")
+ elif args.new and not args.overwrite and os.path.isfile(abs_filename):
+ print("Database file already exists.")
+ elif args.new:
+ conn = sqlite3.connect(abs_filename)
+ with conn:
+ c = conn.cursor()
+ """
+ Here we drop previous links TABLE but probably it would be better to just update old links to INACTIVE
+ """
+ c.execute("DROP TABLE IF EXISTS links")
+ c.execute(
+ "CREATE TABLE links(link TEXT, platform TEXT, arch TEXT,"
+ " version TEXT, provider TEXT, status TEXT)"
+ )
+ for k in keys:
+ c.execute(
+ "INSERT INTO links(link, platform, arch, version, provider, status)"
+ "VALUES ('%s', '%s' '64', '%s', 'gitlab', 'ACTIVE')" %(releases.get(key), k, version)
+ )
diff --git a/share/locale/en.json b/share/locale/en.json
index 0e9d47c..8824c2c 100644
--- a/share/locale/en.json
+++ b/share/locale/en.json
@@ -1,6 +1,8 @@
{
"links_body": "GetTor Test. Please be kind.",
"links_subject": "GetTor Email Test",
+ "help_body": "GetTor Help Test. Please be kind.",
+ "help_subject": "GetTor Help Email Test",
"help_debug": "Log application errors to stdout",
"help_config": "Custom config file location (optional)",
"smtp_links_subject": "[GetTor] Links for your request",
More information about the tor-commits
mailing list