tor-commits February 2020

tor-commits@lists.torproject.org

22 participants
1454 discussions

[torbutton/master] Bug 19757: Support on-disk storage of v3 client auth keys
by sysrqb＠torproject.org 21 Feb '20

21 Feb '20

commit dc1871601fad289645696bb8138e8b14c46dd60a Author: Kathy Brade <brade(a)pearlcrescent.com> Date: Wed Jan 15 08:24:27 2020 -0500 Bug 19757: Support on-disk storage of v3 client auth keys Add onionAuthViewKeys() and onionAuthRemove() Tor controller functions. --- modules/tor-control-port.js | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/modules/tor-control-port.js b/modules/tor-control-port.js index 815df473..2b0849e9 100644 --- a/modules/tor-control-port.js +++ b/modules/tor-control-port.js @@ -580,6 +580,26 @@ info.getConf = function (aControlSocket, key) { // A namespace for functions related to tor's ONION_CLIENT_AUTH_* commands. let onionAuth = {}; +onionAuth.keyInfoStringsFromMessage = utils.extractor(/^250-CLIENT\s+(.+)$/gmi); + +onionAuth.keyInfoObjectsFromMessage = function(message) { + let keyInfoStrings = onionAuth.keyInfoStringsFromMessage(message); + return keyInfoStrings.map(infoStr => utils.listMapData(infoStr, + ["hsAddress", "typeAndKey"])); +} + +// __onionAuth.viewKeys()__. +// Sends a ONION_CLIENT_AUTH_VIEW command to retrieve the list of private keys. +// Returns a promise that is fulfilled with an array of key info objects which +// contain the following properties: +// hsAddress +// typeAndKey +// Flags (e.g., "Permanent") +onionAuth.viewKeys = function (aControlSocket) { + let cmd = "onion_client_auth_view"; + return aControlSocket.sendCommand(cmd).then(onionAuth.keyInfoObjectsFromMessage); +}; + // __onionAuth.add(controlSocket, hsAddress, b64PrivateKey, isPermanent)__. // Sends a ONION_CLIENT_AUTH_ADD command to add a private key to the // Tor configuration. @@ -600,6 +620,19 @@ onionAuth.add = function (aControlSocket, hsAddress, b64PrivateKey, return aControlSocket.sendCommand(cmd); }; +// __onionAuth.remove(controlSocket, hsAddress)__. +// Sends a ONION_CLIENT_AUTH_REMOVE command to remove a private key from the +// Tor configuration. +onionAuth.remove = function (aControlSocket, hsAddress) { + if (!utils.isString(hsAddress)) { + return utils.rejectPromise("hsAddress argument should be a string"); + } + + let cmd = `onion_client_auth_remove ${hsAddress}`; + return aControlSocket.sendCommand(cmd); +}; + + // ## event // Handlers for events @@ -655,9 +688,12 @@ tor.controller = function (ipcFile, host, port, password, onError) { isOpen = true; return { getInfo : key => info.getInfo(socket, key), getConf : key => info.getConf(socket, key), + onionAuthViewKeys : () => onionAuth.viewKeys(socket), onionAuthAdd : (hsAddress, b64PrivateKey, isPermanent) => onionAuth.add(socket, hsAddress, b64PrivateKey, isPermanent), + onionAuthRemove : (hsAddress) => + onionAuth.remove(socket, hsAddress), watchEvent : (type, filter, onData) => event.watchEvent(socket, type, filter, onData), isOpen : () => isOpen,

1 0

[gettor/master] Revert to using twisted for sqlite3 db
by cohosh＠torproject.org 21 Feb '20

21 Feb '20

commit dac0ab6263ee157c1199e206a6b3f12fcd35fd8e Author: Cecylia Bocovich <cohosh(a)torproject.org> Date: Wed Feb 19 14:36:56 2020 -0500 Revert to using twisted for sqlite3 db --- README.md | 2 -- gettor/utils/db.py | 86 ++++++++++++++++++++++++++++-------------------------- tests/conftests.py | 1 - tests/test_db.py | 67 ------------------------------------------ 4 files changed, 45 insertions(+), 111 deletions(-) diff --git a/README.md b/README.md index 6ad9603..f31d113 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,5 @@ GetTor includes PyTest unit tests. To run the tests, first install the dependenc ``` -$ python3 scripts/create_db -n -c -o -f tests/gettor.db -$ python3 scripts/add_links_to_db -f tests/gettor.db $ pytest-3 tests/ ``` diff --git a/gettor/utils/db.py b/gettor/utils/db.py index 7c3853f..0ca11aa 100644 --- a/gettor/utils/db.py +++ b/gettor/utils/db.py @@ -9,10 +9,10 @@ from __future__ import absolute_import -import sqlite3 from datetime import datetime from twisted.python import log +from twisted.enterprise import adbapi class SQLite3(object): """ @@ -20,90 +20,94 @@ class SQLite3(object): """ def __init__(self, dbname): """Constructor.""" - self.conn = sqlite3.connect(dbname) + self.dbpool = adbapi.ConnectionPool( + "sqlite3", dbname, check_same_thread=False + ) + + def query_callback(self, results=None): + """ + Query callback + Log that the database query has been executed and return results + """ + log.msg("Database query executed successfully.") + return results + + def query_errback(self, error=None): + """ + Query error callback + Logs database error + """ + if error: + log.msg("Database error: {}".format(error)) + return None def new_request(self, id, command, service, platform, language, date, status): """ Perform a new request to the database """ - c = self.conn.cursor() query = "INSERT INTO requests VALUES(?, ?, ?, ?, ?, ?, ?)" - c.execute(query, (id, command, platform, language, service, - date, status)) - self.conn.commit() - return + return self.dbpool.runQuery( + query, (id, command, platform, language, service, date, status) + ).addCallback(self.query_callback).addErrback(self.query_errback) def get_requests(self, status, command, service): """ Perform a SELECT request to the database """ - c = self.conn.cursor() query = "SELECT * FROM requests WHERE service=? AND command=? AND "\ "status = ?" - c.execute(query, (service, command, status)) - - return c.fetchall() + return self.dbpool.runQuery( + query, (service, command, status) + ).addCallback(self.query_callback).addErrback(self.query_errback) def get_num_requests(self, id, service): """ Get number of requests for statistics """ - c = self.conn.cursor() - query = "SELECT COUNT(rowid) FROM requests WHERE id=? AND "\ - "service=?" + query = "SELECT COUNT(rowid) FROM requests WHERE id=? AND service=?" - c.execute(query, (id, service)) - return c.fetchone()[0] + return self.dbpool.runQuery( + query, (id, service) + ).addCallback(self.query_callback).addErrback(self.query_errback) def remove_request(self, id, service, date): """ Removes completed request record from the database """ - c = self.conn.cursor() - query = "DELETE FROM requests WHERE id=? AND service=? AND "\ - "date=?" + query = "DELETE FROM requests WHERE id=? AND service=? AND date=?" - c.execute(query, (id, service, date)) - self.conn.commit() - return + return self.dbpool.runQuery( + query, (id, service, date) + ).addCallback(self.query_callback).addErrback(self.query_errback) def update_stats(self, command, service, platform=None, language='en'): """ Update statistics to the database """ - c = self.conn.cursor() now_str = datetime.now().strftime("%Y%m%d") query = "INSERT INTO stats(num_requests, platform, language, command, "\ - "service, date) VALUES (1, ?, ?, ?, ?, ?) ON "\ - "CONFLICT(platform, language, command, service, date) "\ - "DO UPDATE SET num_requests=num_requests+1" + "service, date) VALUES (1, ?, ?, ?, ?, ?) ON CONFLICT(platform, "\ + "language, command, service, date) DO UPDATE SET num_requests=num_requests+1" - c.execute(query, (platform, language, command, service, - now_str)) - self.conn.commit() - return + return self.dbpool.runQuery( + query, (platform, language, command, service, now_str) + ).addCallback(self.query_callback).addErrback(self.query_errback) def get_links(self, platform, language, status): """ Get links from the database per platform """ - c = self.conn.cursor() query = "SELECT * FROM links WHERE platform=? AND language=? AND status=?" - c.execute(query, (platform, language, status)) - - return c.fetchall() + return self.dbpool.runQuery( + query, (platform, language, status) + ).addCallback(self.query_callback).addErrback(self.query_errback) def get_locales(self): """ Get a list of the supported tor browser binary locales """ - c = self.conn.cursor() query = "SELECT DISTINCT language FROM links" - c.execute(query) - - locales = [] - for locale in c.fetchall(): - locales.append(locale[0]) - return locales + return self.dbpool.runQuery(query + ).addCallback(self.query_callback).addErrback(self.query_errback) diff --git a/tests/conftests.py b/tests/conftests.py index d509776..cbb4d28 100644 --- a/tests/conftests.py +++ b/tests/conftests.py @@ -5,7 +5,6 @@ from __future__ import unicode_literals from gettor.utils import options from gettor.utils import strings from gettor.utils import twitter -from gettor.utils.db import SQLite3 from gettor.services.email.sendmail import Sendmail from gettor.services.twitter import twitterdm from gettor.parse.email import EmailParser, AddressError, DKIMError diff --git a/tests/test_db.py b/tests/test_db.py deleted file mode 100644 index d663d89..0000000 --- a/tests/test_db.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python3 -import pytest -from datetime import datetime -from twisted.trial import unittest - -from . import conftests - -class DatabaseTests(unittest.TestCase): - - # Fail any tests which take longer than 15 seconds. - timeout = 15 - def setUp(self): - self.settings = conftests.options.parse_settings("en","tests/test.conf.json") - print(self.settings.get("dbname")) - self.db = conftests.SQLite3(self.settings.get("dbname")) - - def tearDown(self): - print("tearDown()") - - def add_dummy_requests(self, num): - now_str = datetime.now().strftime("%Y%m%d") - for i in (0, num): - self.db.new_request( - id='testid', - command='links', - platform='linux', - language='en', - service='email', - date=now_str, - status="ONHOLD", - ) - - def test_stored_locales(self): - locales = self.db.get_locales() - self.assertIn('en-US', locales) - - def test_requests(self): - now_str = datetime.now().strftime("%Y%m%d") - self.add_dummy_requests(2) - num = self.db.get_num_requests("testid", "email") - self.assertEqual(num, 2) - - requests = self.db.get_requests("ONHOLD", "links", "email") - for request in requests: - print(request) - self.assertEqual(request[1], "links") - self.assertEqual(request[4], "email") - self.assertEqual(request[5], now_str) - self.assertEqual(request[6], "ONHOLD") - self.assertEqual(len(requests), 2) - - self.db.remove_request("testid", "email", now_str) - num = self.db.get_num_requests("testid", "email") - self.assertEqual(num, 0) - - def test_links(self): - links = self.db.get_links("linux", "en-US", "ACTIVE") - self.assertEqual(len(links), 2) # Right now we have github and gitlab - - for link in links: - self.assertEqual(link[1], "linux") - self.assertEqual(link[2], "en-US") - self.assertEqual(link[6], "ACTIVE") - self.assertIn(link[5], ["github", "gitlab"]) - -if __name__ == "__main__": - unittest.main()

1 0

[gettor/master] Add method to close dbpool and call it
by cohosh＠torproject.org 21 Feb '20

21 Feb '20

commit c413a8ebdaf1c70bc40255e35e7917165fa75552 Author: Cecylia Bocovich <cohosh(a)torproject.org> Date: Wed Feb 19 12:15:58 2020 -0500 Add method to close dbpool and call it We were leaving connections to the database open, which causes some calls to hang. This adds destructors to the SQLite3 class and the classes that use it, and refactors some code to make one database connection per class object. This also makes sure that the destructors for objects that use databases are actually called. --- gettor/parse/email.py | 13 +++++++------ gettor/parse/twitter.py | 11 ++++++----- gettor/services/email/sendmail.py | 2 ++ gettor/services/twitter/twitterdm.py | 3 +++ gettor/utils/db.py | 3 +++ scripts/process_email | 1 + tests/test_db.py | 4 +--- tests/test_email_service.py | 16 ++++++++++++++++ 8 files changed, 39 insertions(+), 14 deletions(-) diff --git a/gettor/parse/email.py b/gettor/parse/email.py index 9084cb7..a08f8ca 100644 --- a/gettor/parse/email.py +++ b/gettor/parse/email.py @@ -56,6 +56,10 @@ class EmailParser(object): self.to_addr = to_addr self.locales = [] self.platforms = self.settings.get("platforms") + self.conn = SQLite3(self.settings.get("dbname")) + + def __del__(self): + del self.conn def normalize(self, msg): # Normalization will convert <Alice Wonderland> alice(a)wonderland.net @@ -164,10 +168,8 @@ class EmailParser(object): @defer.inlineCallbacks def get_locales(self): - dbname = self.settings.get("dbname") - conn = SQLite3(dbname) - locales = yield conn.get_locales() + locales = yield self.conn.get_locales() for l in locales: self.locales.append(l[0]) @@ -235,7 +237,6 @@ class EmailParser(object): now_str = datetime.now().strftime("%Y%m%d%H%M%S") dbname = self.settings.get("dbname") test_hid = self.settings.get("test_hid") - conn = SQLite3(dbname) if request["command"]: @@ -247,7 +248,7 @@ class EmailParser(object): system="email parser" ) - num_requests = yield conn.get_num_requests( + num_requests = yield self.conn.get_num_requests( id=hid, service=request_service ) @@ -262,7 +263,7 @@ class EmailParser(object): ), system="email parser" ) else: - conn.new_request( + self.conn.new_request( id=request['id'], command=request['command'], platform=request['platform'], diff --git a/gettor/parse/twitter.py b/gettor/parse/twitter.py index 645bf47..fd197ac 100644 --- a/gettor/parse/twitter.py +++ b/gettor/parse/twitter.py @@ -35,7 +35,10 @@ class TwitterParser(object): """ self.settings = settings self.twitter_id = twitter_id + self.conn = SQLite3(self.settings.get("dbname")) + def __del__(self): + del self.conn def build_request(self, msg_text, twitter_id, languages, platforms): @@ -110,16 +113,14 @@ class TwitterParser(object): if request["command"]: now_str = datetime.now().strftime("%Y%m%d%H%M%S") - dbname = self.settings.get("dbname") - conn = SQLite3(dbname) hid = hashlib.sha256(str(request['id']).encode('utf-8')) # check limits first - num_requests = yield conn.get_num_requests( + num_requests = yield self.conn.get_num_requests( id=hid.hexdigest(), service=request['service'] ) - num_requests += yield conn.get_num_requests( + num_requests += yield self.conn.get_num_requests( id=str(request['id']), service=request['service'] ) @@ -130,7 +131,7 @@ class TwitterParser(object): ), system="twitter parser" ) else: - conn.new_request( + self.conn.new_request( id=str(request['id']), command=request['command'], platform=request['platform'], diff --git a/gettor/services/email/sendmail.py b/gettor/services/email/sendmail.py index 9d97567..754fb5b 100644 --- a/gettor/services/email/sendmail.py +++ b/gettor/services/email/sendmail.py @@ -45,6 +45,8 @@ class Sendmail(object): dbname = self.settings.get("dbname") self.conn = DB(dbname) + def __del__(self): + del self.conn def get_interval(self): """ diff --git a/gettor/services/twitter/twitterdm.py b/gettor/services/twitter/twitterdm.py index 5f143ca..b0a1a48 100644 --- a/gettor/services/twitter/twitterdm.py +++ b/gettor/services/twitter/twitterdm.py @@ -41,6 +41,8 @@ class Twitterdm(object): self.twitter = Twitter(settings) self.conn = DB(dbname) + def __del__(self): + del self.conn def get_interval(self): """ @@ -115,6 +117,7 @@ class Twitterdm(object): yield defer.maybeDeferred( tp.parse, e['message_create']['message_data']['text'], message_id ).addCallback(tp.parse_callback).addErrback(tp.parse_errback) + del tp # Manage help and links messages separately help_requests = yield self.conn.get_requests( diff --git a/gettor/utils/db.py b/gettor/utils/db.py index 0ca11aa..064049e 100644 --- a/gettor/utils/db.py +++ b/gettor/utils/db.py @@ -24,6 +24,9 @@ class SQLite3(object): "sqlite3", dbname, check_same_thread=False ) + def __del__(self): + self.dbpool.close() + def query_callback(self, results=None): """ Query callback diff --git a/scripts/process_email b/scripts/process_email index cce7bcc..1dd7dfd 100755 --- a/scripts/process_email +++ b/scripts/process_email @@ -41,6 +41,7 @@ def process_email(message): log.err("DKIM error: {}".format(e), system="process email") reactor.stop() + del ep reactor.stop() def main(): diff --git a/tests/test_db.py b/tests/test_db.py index e956850..e04ca39 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -2,8 +2,6 @@ import pytest import pytest_twisted from twisted.trial import unittest -from twisted.internet import defer, reactor -from twisted.internet import task from . import conftests @@ -19,7 +17,7 @@ class DatabaseTests(unittest.TestCase): def tearDown(self): print("tearDown()") - return self.conn.dbpool.close() + del self.conn @pytest_twisted.inlineCallbacks def test_stored_locales(self): diff --git a/tests/test_email_service.py b/tests/test_email_service.py index 00795c1..d064aa1 100644 --- a/tests/test_email_service.py +++ b/tests/test_email_service.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import pytest +import pytest_twisted import hashlib from datetime import datetime from twisted.trial import unittest @@ -31,6 +32,7 @@ class EmailServiceTests(unittest.TestCase): def tearDown(self): print("tearDown()") + del self.sm_client def test_get_interval(self): self.assertEqual(self.settings.get("sendmail_interval"), self.sm_client.get_interval()) @@ -39,6 +41,7 @@ class EmailServiceTests(unittest.TestCase): ep = conftests.EmailParser(self.settings, "gettor(a)torproject.org") request = ep.parse("From: \"silvia [hiro]\" <hiro(a)torproject.org>\n Subject: help\n Reply-To: hiro(a)torproject.org \nTo: gettor(a)torproject.org") self.assertEqual(request["command"], "help") + del ep def test_normalize_msg(self): ep = conftests.EmailParser(self.settings, "gettor(a)torproject.org") @@ -46,6 +49,7 @@ class EmailServiceTests(unittest.TestCase): msg = conftests.message_from_string(msg_str) request = ep.normalize(msg) self.assertEqual(request, ('silvia [hiro]', 'hiro(a)torproject.org', '', 'gettor(a)torproject.org')) + del ep def test_validate_msg(self): ep = conftests.EmailParser(self.settings, "gettor(a)torproject.org") @@ -53,6 +57,7 @@ class EmailServiceTests(unittest.TestCase): msg = conftests.message_from_string(msg_str) request = ep.validate("hiro(a)torproject.org", msg) assert request + del ep def test_dkim_verify(self): ep = conftests.EmailParser(self.settings, "gettor(a)torproject.org") @@ -60,6 +65,7 @@ class EmailServiceTests(unittest.TestCase): msg = conftests.message_from_string(msg_str) request = ep.dkim_verify(msg, "hiro(a)torproject.org") assert request + del ep def test_build_request(self): ep = conftests.EmailParser(self.settings, "gettor(a)torproject.org") @@ -70,6 +76,7 @@ class EmailServiceTests(unittest.TestCase): self.assertEqual(request["command"], "links") self.assertEqual(request["platform"], "osx") self.assertEqual(request["language"], "es") + del ep def test_too_many_request_exclude(self): ep = conftests.EmailParser(self.settings, "gettor(a)torproject.org") @@ -79,6 +86,7 @@ class EmailServiceTests(unittest.TestCase): check = ep.too_many_requests(hid, self.settings.get("test_hid"), num_requests, limit) self.assertEqual(hid, self.settings.get("test_hid")) self.assertEqual(check, False) + del ep def test_language_email_parser(self): ep = conftests.EmailParser(self.settings, "gettor(a)torproject.org") @@ -137,6 +145,7 @@ class EmailServiceTests(unittest.TestCase): "Subject: linux es-AR\r\n Reply-To: hiro(a)torproject.org \nTo:" "gettor(a)torproject.org\n linux es") self.assertEqual(request["language"], "es-AR") + del ep def test_sent_links_message(self): ep = self.sm_client @@ -159,6 +168,13 @@ class EmailServiceTests(unittest.TestCase): help_msg = ep.build_help_body_message() assert "This is how you can request a tor browser bundle link" in help_msg + @pytest_twisted.inlineCallbacks + def test_get_locales(self): + ep = conftests.EmailParser(self.settings, "gettor(a)torproject.org") + yield ep.get_locales().addErrback(ep.parse_errback) + assert "en-US" in ep.locales + del ep + if __name__ == "__main__": unittest.main()

1 0

[gettor/master] Add database tests
by cohosh＠torproject.org 21 Feb '20

21 Feb '20

commit c6cfd912410f673961db28e7c68dd413c67d5e7e Author: Cecylia Bocovich <cohosh(a)torproject.org> Date: Fri Feb 14 17:23:39 2020 -0500 Add database tests --- tests/conftests.py | 1 + tests/test_db.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/tests/conftests.py b/tests/conftests.py index cbb4d28..d509776 100644 --- a/tests/conftests.py +++ b/tests/conftests.py @@ -5,6 +5,7 @@ from __future__ import unicode_literals from gettor.utils import options from gettor.utils import strings from gettor.utils import twitter +from gettor.utils.db import SQLite3 from gettor.services.email.sendmail import Sendmail from gettor.services.twitter import twitterdm from gettor.parse.email import EmailParser, AddressError, DKIMError diff --git a/tests/test_db.py b/tests/test_db.py new file mode 100644 index 0000000..e956850 --- /dev/null +++ b/tests/test_db.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +import pytest +import pytest_twisted +from twisted.trial import unittest +from twisted.internet import defer, reactor +from twisted.internet import task + +from . import conftests + +class DatabaseTests(unittest.TestCase): + + # Fail any tests which take longer than 15 seconds. + timeout = 15 + def setUp(self): + self.settings = conftests.options.parse_settings("en","./gettor.conf.json") + self.locales = conftests.strings.get_locales() + + self.conn = conftests.SQLite3(self.settings.get("dbname")) + + def tearDown(self): + print("tearDown()") + return self.conn.dbpool.close() + + @pytest_twisted.inlineCallbacks + def test_stored_locales(self): + locales = yield self.conn.get_locales() + +if __name__ == "__main__": + unittest.main()

1 0

[gettor/master] Expand tests for database functions in gettor
by cohosh＠torproject.org 21 Feb '20

21 Feb '20

commit 64df8b27b36bca24f7612b83697e52becbe3dc32 Author: Cecylia Bocovich <cohosh(a)torproject.org> Date: Wed Feb 19 14:30:19 2020 -0500 Expand tests for database functions in gettor --- README.md | 4 +++- tests/test_db.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f31d113..498825c 100644 --- a/README.md +++ b/README.md @@ -82,5 +82,7 @@ GetTor includes PyTest unit tests. To run the tests, first install the dependenc ``` -$ pytest-3 tests/ +$ python3 scripts/create_db -n -c -o -f tests/gettor.db +$ python3 scripts/add_links_to_db -f tests/gettor.db +$ pytest-3 -s -v tests/ ``` diff --git a/tests/test_db.py b/tests/test_db.py index e04ca39..7d5634b 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import pytest import pytest_twisted +from datetime import datetime from twisted.trial import unittest from . import conftests @@ -10,7 +11,7 @@ class DatabaseTests(unittest.TestCase): # Fail any tests which take longer than 15 seconds. timeout = 15 def setUp(self): - self.settings = conftests.options.parse_settings("en","./gettor.conf.json") + self.settings = conftests.options.parse_settings("en","tests/test.conf.json") self.locales = conftests.strings.get_locales() self.conn = conftests.SQLite3(self.settings.get("dbname")) @@ -20,8 +21,55 @@ class DatabaseTests(unittest.TestCase): del self.conn @pytest_twisted.inlineCallbacks + def add_dummy_requests(self, num): + now_str = datetime.now().strftime("%Y%m%d") + for i in (0, num): + yield self.conn.new_request( + id='testid', + command='links', + platform='linux', + language='en', + service='email', + date=now_str, + status="ONHOLD", + ) + + @pytest_twisted.inlineCallbacks def test_stored_locales(self): - locales = yield self.conn.get_locales() + locales = [] + ls = yield self.conn.get_locales() + for l in ls: + locales.append(l[0]) + self.assertIn('en-US', locales) + + @pytest_twisted.inlineCallbacks + def test_requests(self): + now_str = datetime.now().strftime("%Y%m%d") + yield self.add_dummy_requests(2) + num = yield self.conn.get_num_requests("testid", "email") + self.assertEqual(num[0][0], 2) + + requests = yield self.conn.get_requests("ONHOLD", "links", "email") + for request in requests: + self.assertEqual(request[1], "links") + self.assertEqual(request[4], "email") + self.assertEqual(request[5], now_str) + self.assertEqual(request[6], "ONHOLD") + self.assertEqual(len(requests), 2) + + yield self.conn.remove_request("testid", "email", now_str) + num = yield self.conn.get_num_requests("testid", "email") + self.assertEqual(num[0][0], 0) + + @pytest_twisted.inlineCallbacks + def test_links(self): + links = yield self.conn.get_links("linux", "en-US", "ACTIVE") + + for link in links: + self.assertEqual(link[1], "linux") + self.assertEqual(link[2], "en-US") + self.assertEqual(link[6], "ACTIVE") + self.assertIn(link[5], ["github", "gitlab"]) if __name__ == "__main__": unittest.main()

1 0

[metrics-web/master] Set default time zone UTC.
by karsten＠torproject.org 21 Feb '20

21 Feb '20

commit efb86141eb58aafc652065faae1f6f870844ad15 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Jan 15 14:52:17 2020 +0000 Set default time zone UTC. --- CHANGELOG.md | 1 + .../bwhist/RelayDescriptorDatabaseImporter.java | 13 ++++------- .../torproject/metrics/stats/clients/Database.java | 8 ++----- .../org/torproject/metrics/stats/main/Main.java | 2 ++ .../torproject/metrics/stats/onionperf/Main.java | 27 +++++++--------------- .../torproject/metrics/stats/servers/Database.java | 25 +++++++------------- .../torproject/metrics/stats/totalcw/Database.java | 10 +++----- .../torproject/metrics/stats/webstats/Main.java | 6 +---- 8 files changed, 29 insertions(+), 63 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57694f1..5731bad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - Configure a base URL in order to turn ExoneraTor's permanent links into https:// links. - Set default locale `US` at the beginning of the execution. + - Set default time zone `UTC` at the beginning of the execution. # Changes in version 1.3.0 - 2019-11-09 diff --git a/src/main/java/org/torproject/metrics/stats/bwhist/RelayDescriptorDatabaseImporter.java b/src/main/java/org/torproject/metrics/stats/bwhist/RelayDescriptorDatabaseImporter.java index 2609188..2958bda 100644 --- a/src/main/java/org/torproject/metrics/stats/bwhist/RelayDescriptorDatabaseImporter.java +++ b/src/main/java/org/torproject/metrics/stats/bwhist/RelayDescriptorDatabaseImporter.java @@ -25,13 +25,11 @@ import java.sql.Timestamp; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; -import java.util.Calendar; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.SortedSet; -import java.util.TimeZone; import java.util.TreeSet; /** @@ -154,7 +152,6 @@ public final class RelayDescriptorDatabaseImporter { /* Initialize date format, so that we can format timestamps. */ this.dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - this.dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); } private void addDateToScheduledUpdates(long timestamp) @@ -189,11 +186,10 @@ public final class RelayDescriptorDatabaseImporter { if (this.importIntoDatabase) { try { this.addDateToScheduledUpdates(validAfter); - Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC")); Timestamp validAfterTimestamp = new Timestamp(validAfter); if (lastCheckedStatusEntries != validAfter) { insertedStatusEntries.clear(); - this.psSs.setTimestamp(1, validAfterTimestamp, cal); + this.psSs.setTimestamp(1, validAfterTimestamp); ResultSet rs = psSs.executeQuery(); while (rs.next()) { String insertedFingerprint = rs.getString(1); @@ -204,11 +200,11 @@ public final class RelayDescriptorDatabaseImporter { } if (!insertedStatusEntries.contains(fingerprint)) { this.psR.clearParameters(); - this.psR.setTimestamp(1, validAfterTimestamp, cal); + this.psR.setTimestamp(1, validAfterTimestamp); this.psR.setString(2, nickname); this.psR.setString(3, fingerprint); this.psR.setString(4, descriptor); - this.psR.setTimestamp(5, new Timestamp(published), cal); + this.psR.setTimestamp(5, new Timestamp(published)); this.psR.setString(6, address); this.psR.setLong(7, orPort); this.psR.setLong(8, dirPort); @@ -625,12 +621,11 @@ public final class RelayDescriptorDatabaseImporter { + "dirwrite"; statistics.add(columns.split(", ")); Statement st = this.conn.createStatement(); - Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); String queryString = "SELECT " + columns + " FROM stats_bandwidth"; try (ResultSet rs = st.executeQuery(queryString)) { while (rs.next()) { String[] outputLine = new String[7]; - outputLine[0] = rs.getDate("date", calendar).toLocalDate().toString(); + outputLine[0] = rs.getDate("date").toLocalDate().toString(); outputLine[1] = getBooleanFromResultSet(rs, "isexit"); outputLine[2] = getBooleanFromResultSet(rs, "isguard"); outputLine[3] = getLongFromResultSet(rs, "bwread"); diff --git a/src/main/java/org/torproject/metrics/stats/clients/Database.java b/src/main/java/org/torproject/metrics/stats/clients/Database.java index ac715fd..aa69318 100644 --- a/src/main/java/org/torproject/metrics/stats/clients/Database.java +++ b/src/main/java/org/torproject/metrics/stats/clients/Database.java @@ -12,9 +12,7 @@ import java.sql.Statement; import java.sql.Timestamp; import java.time.Instant; import java.util.ArrayList; -import java.util.Calendar; import java.util.List; -import java.util.TimeZone; /** Database wrapper to connect to the database, insert data, run the stored * procedure for aggregating data, and query aggregated data as output. */ @@ -94,12 +92,11 @@ class Database implements AutoCloseable { String columns = "date, node, country, transport, version, frac, users"; statistics.add(columns.split(", ")); Statement st = this.connection.createStatement(); - Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); String queryString = "SELECT " + columns + " FROM estimated"; try (ResultSet rs = st.executeQuery(queryString)) { while (rs.next()) { String[] outputLine = new String[7]; - outputLine[0] = rs.getDate("date", calendar).toLocalDate().toString(); + outputLine[0] = rs.getDate("date").toLocalDate().toString(); outputLine[1] = rs.getString("node"); outputLine[2] = rs.getString("country"); outputLine[3] = rs.getString("transport"); @@ -118,12 +115,11 @@ class Database implements AutoCloseable { String columns = "date, node, country, transport, version, frac, low, high"; statistics.add(columns.split(", ")); Statement st = this.connection.createStatement(); - Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); String queryString = "SELECT " + columns + " FROM combined"; try (ResultSet rs = st.executeQuery(queryString)) { while (rs.next()) { String[] outputLine = new String[8]; - outputLine[0] = rs.getDate("date", calendar).toLocalDate().toString(); + outputLine[0] = rs.getDate("date").toLocalDate().toString(); outputLine[1] = rs.getString("node"); outputLine[2] = rs.getString("country"); outputLine[3] = rs.getString("transport"); diff --git a/src/main/java/org/torproject/metrics/stats/main/Main.java b/src/main/java/org/torproject/metrics/stats/main/Main.java index 3a4baf7..6badd96 100644 --- a/src/main/java/org/torproject/metrics/stats/main/Main.java +++ b/src/main/java/org/torproject/metrics/stats/main/Main.java @@ -11,6 +11,7 @@ import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.List; import java.util.Locale; +import java.util.TimeZone; public class Main { @@ -31,6 +32,7 @@ public class Main { log.info("Starting metrics update run."); Locale.setDefault(Locale.US); + TimeZone.setDefault(TimeZone.getTimeZone("UTC")); File[] outputDirs = new File[] { modulesDir, statsDir }; for (File outputDir : outputDirs) { diff --git a/src/main/java/org/torproject/metrics/stats/onionperf/Main.java b/src/main/java/org/torproject/metrics/stats/onionperf/Main.java index c8e3a90..4325c01 100644 --- a/src/main/java/org/torproject/metrics/stats/onionperf/Main.java +++ b/src/main/java/org/torproject/metrics/stats/onionperf/Main.java @@ -24,11 +24,9 @@ import java.sql.Types; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; -import java.util.Calendar; import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.TimeZone; public class Main { @@ -96,7 +94,6 @@ public class Main { "INSERT INTO buildtimes (measurement_id, position, buildtime, delta) " + "VALUES (?, ?, ?, ?)"); - Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); DescriptorReader dr = DescriptorSourceFactory.createDescriptorReader(); for (Descriptor d : dr.readDescriptors( new File(org.torproject.metrics.stats.main.Main.descriptorsDir, @@ -113,7 +110,7 @@ public class Main { psMeasurementsSelect.setString(1, truncatedSource); psMeasurementsSelect.setInt(2, tr.getFileSize()); psMeasurementsSelect.setTimestamp(3, - new Timestamp(tr.getStartMillis()), calendar); + new Timestamp(tr.getStartMillis())); try (ResultSet rs = psMeasurementsSelect.executeQuery()) { if (rs.next()) { measurementId = rs.getInt(1); @@ -124,7 +121,7 @@ public class Main { psMeasurementsInsert.setString(1, truncatedSource); psMeasurementsInsert.setInt(2, tr.getFileSize()); psMeasurementsInsert.setTimestamp(3, - new Timestamp(tr.getStartMillis()), calendar); + new Timestamp(tr.getStartMillis())); long[] timestamps = new long[] { tr.getSocketMillis(), tr.getConnectMillis(), tr.getNegotiateMillis(), tr.getRequestMillis(), tr.getResponseMillis(), @@ -158,13 +155,13 @@ public class Main { psMeasurementsInsert.setNull(26, Types.TIMESTAMP); } else { psMeasurementsInsert.setTimestamp(26, - new Timestamp(tr.getLaunchMillis()), calendar); + new Timestamp(tr.getLaunchMillis())); } if (tr.getUsedAtMillis() < 0L) { psMeasurementsInsert.setNull(27, Types.TIMESTAMP); } else { psMeasurementsInsert.setTimestamp(27, - new Timestamp(tr.getUsedAtMillis()), calendar); + new Timestamp(tr.getUsedAtMillis())); } if (tr.getTimeout() < 0L) { psMeasurementsInsert.setNull(28, Types.INTEGER); @@ -251,12 +248,10 @@ public class Main { String queryString = "SELECT date, filesize, source, server, q1, md, q3, " + "timeouts, failures, requests FROM onionperf"; DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); try (ResultSet rs = st.executeQuery(queryString)) { while (rs.next()) { statistics.add(String.format("%s,%d,%s,%s,%.0f,%.0f,%.0f,%d,%d,%d", - dateFormat.format(rs.getDate("date", calendar)), + dateFormat.format(rs.getDate("date")), rs.getInt("filesize"), getStringFromResultSet(rs, "source"), getStringFromResultSet(rs, "server"), @@ -280,12 +275,10 @@ public class Main { String queryString = "SELECT date, source, position, q1, md, q3 " + "FROM buildtimes_stats"; DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); try (ResultSet rs = st.executeQuery(queryString)) { while (rs.next()) { statistics.add(String.format("%s,%s,%d,%d,%d,%d", - dateFormat.format(rs.getDate("date", calendar)), + dateFormat.format(rs.getDate("date")), getStringFromResultSet(rs, "source"), rs.getInt("position"), rs.getInt("q1"), @@ -305,12 +298,10 @@ public class Main { String queryString = "SELECT date, source, server, low, q1, md, q3, high " + "FROM latencies_stats"; DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); try (ResultSet rs = st.executeQuery(queryString)) { while (rs.next()) { statistics.add(String.format("%s,%s,%s,%d,%d,%d,%d,%d", - dateFormat.format(rs.getDate("date", calendar)), + dateFormat.format(rs.getDate("date")), getStringFromResultSet(rs, "source"), rs.getString("server"), rs.getInt("low"), @@ -332,12 +323,10 @@ public class Main { String queryString = "SELECT date, source, server, low, q1, md, q3, high " + "FROM throughput_stats"; DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); try (ResultSet rs = st.executeQuery(queryString)) { while (rs.next()) { statistics.add(String.format("%s,%s,%s,%d,%d,%d,%d,%d", - dateFormat.format(rs.getDate("date", calendar)), + dateFormat.format(rs.getDate("date")), getStringFromResultSet(rs, "source"), rs.getString("server"), rs.getInt("low"), diff --git a/src/main/java/org/torproject/metrics/stats/servers/Database.java b/src/main/java/org/torproject/metrics/stats/servers/Database.java index 8e5e31d..834d004 100644 --- a/src/main/java/org/torproject/metrics/stats/servers/Database.java +++ b/src/main/java/org/torproject/metrics/stats/servers/Database.java @@ -14,14 +14,12 @@ import java.sql.Types; import java.time.ZoneId; import java.time.ZonedDateTime; import java.util.ArrayList; -import java.util.Calendar; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.SortedSet; -import java.util.TimeZone; /** Database wrapper to connect to the database, insert data, run the stored * procedure for aggregating data, and query aggregated data as output. */ @@ -266,10 +264,9 @@ class Database implements AutoCloseable { this.psStatusesSelect.clearParameters(); this.psStatusesSelect.setString(1, networkStatus.isRelay ? "relay" : "bridge"); - Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); this.psStatusesSelect.setTimestamp(2, Timestamp.from(ZonedDateTime.of(networkStatus.timestamp, - ZoneId.of("UTC")).toInstant()), calendar); + ZoneId.of("UTC")).toInstant())); try (ResultSet rs = this.psStatusesSelect.executeQuery()) { if (rs.next()) { if (rs.getBoolean(1)) { @@ -284,7 +281,7 @@ class Database implements AutoCloseable { networkStatus.isRelay ? "relay" : "bridge"); this.psStatusesInsert.setTimestamp(2, Timestamp.from(ZonedDateTime.of(networkStatus.timestamp, - ZoneId.of("UTC")).toInstant()), calendar); + ZoneId.of("UTC")).toInstant())); this.psStatusesInsert.setInt(3, networkStatus.running); if (null != networkStatus.totalConsensusWeight) { this.psStatusesInsert.setFloat(4, networkStatus.totalConsensusWeight); @@ -412,12 +409,11 @@ class Database implements AutoCloseable { + "server_count_sum_avg, advertised_bandwidth_bytes_sum_avg"; statistics.add(columns.split(", ")); Statement st = this.connection.createStatement(); - Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); String queryString = "SELECT " + columns + " FROM ipv6servers"; try (ResultSet rs = st.executeQuery(queryString)) { while (rs.next()) { String[] outputLine = new String[9]; - outputLine[0] = rs.getDate("valid_after_date", calendar) + outputLine[0] = rs.getDate("valid_after_date") .toLocalDate().toString(); outputLine[1] = rs.getString("server"); outputLine[2] = rs.getString("guard_relay"); @@ -440,12 +436,11 @@ class Database implements AutoCloseable { String columns = "date, isexit, isguard, advbw"; statistics.add(columns.split(", ")); Statement st = this.connection.createStatement(); - Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); String queryString = "SELECT " + columns + " FROM bandwidth_advbw"; try (ResultSet rs = st.executeQuery(queryString)) { while (rs.next()) { String[] outputLine = new String[4]; - outputLine[0] = rs.getDate("date", calendar).toLocalDate().toString(); + outputLine[0] = rs.getDate("date").toLocalDate().toString(); outputLine[1] = rs.getString("isexit"); outputLine[2] = rs.getString("isguard"); outputLine[3] = getLongFromResultSet(rs, "advbw"); @@ -461,12 +456,11 @@ class Database implements AutoCloseable { String columns = "date, relays, bridges"; statistics.add(columns.split(", ")); Statement st = this.connection.createStatement(); - Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); String queryString = "SELECT " + columns + " FROM servers_networksize"; try (ResultSet rs = st.executeQuery(queryString)) { while (rs.next()) { String[] outputLine = new String[3]; - outputLine[0] = rs.getDate("date", calendar).toLocalDate().toString(); + outputLine[0] = rs.getDate("date").toLocalDate().toString(); outputLine[1] = getLongFromResultSet(rs, "relays"); outputLine[2] = getLongFromResultSet(rs, "bridges"); statistics.add(outputLine); @@ -481,12 +475,11 @@ class Database implements AutoCloseable { String columns = "date, flag, relays"; statistics.add(columns.split(", ")); Statement st = this.connection.createStatement(); - Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); String queryString = "SELECT " + columns + " FROM servers_relayflags"; try (ResultSet rs = st.executeQuery(queryString)) { while (rs.next()) { String[] outputLine = new String[3]; - outputLine[0] = rs.getDate("date", calendar).toLocalDate().toString(); + outputLine[0] = rs.getDate("date").toLocalDate().toString(); outputLine[1] = rs.getString("flag"); outputLine[2] = getLongFromResultSet(rs, "relays"); statistics.add(outputLine); @@ -501,12 +494,11 @@ class Database implements AutoCloseable { String columns = "date, version, relays"; statistics.add(columns.split(", ")); Statement st = this.connection.createStatement(); - Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); String queryString = "SELECT " + columns + " FROM servers_versions"; try (ResultSet rs = st.executeQuery(queryString)) { while (rs.next()) { String[] outputLine = new String[3]; - outputLine[0] = rs.getDate("date", calendar).toLocalDate().toString(); + outputLine[0] = rs.getDate("date").toLocalDate().toString(); outputLine[1] = rs.getString("version"); outputLine[2] = getLongFromResultSet(rs, "relays"); statistics.add(outputLine); @@ -521,12 +513,11 @@ class Database implements AutoCloseable { String columns = "date, platform, relays"; statistics.add(columns.split(", ")); Statement st = this.connection.createStatement(); - Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); String queryString = "SELECT " + columns + " FROM servers_platforms"; try (ResultSet rs = st.executeQuery(queryString)) { while (rs.next()) { String[] outputLine = new String[3]; - outputLine[0] = rs.getDate("date", calendar).toLocalDate().toString(); + outputLine[0] = rs.getDate("date").toLocalDate().toString(); outputLine[1] = rs.getString("platform"); outputLine[2] = getLongFromResultSet(rs, "relays"); statistics.add(outputLine); diff --git a/src/main/java/org/torproject/metrics/stats/totalcw/Database.java b/src/main/java/org/torproject/metrics/stats/totalcw/Database.java index 3d51604..74b84d1 100644 --- a/src/main/java/org/torproject/metrics/stats/totalcw/Database.java +++ b/src/main/java/org/torproject/metrics/stats/totalcw/Database.java @@ -15,9 +15,7 @@ import java.time.LocalDateTime; import java.time.ZoneId; import java.time.ZonedDateTime; import java.util.ArrayList; -import java.util.Calendar; import java.util.List; -import java.util.TimeZone; /** Database wrapper to connect to the database, insert data, run the stored * procedure for aggregating data, and query aggregated data as output. */ @@ -121,11 +119,10 @@ class Database implements AutoCloseable { private void insertStatusIfAbsent(LocalDateTime validAfter, Integer authorityId, long[] measuredSums) throws SQLException { - Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); this.psVoteSelect.clearParameters(); this.psVoteSelect.setTimestamp(1, Timestamp.from(ZonedDateTime.of(validAfter, - ZoneId.of("UTC")).toInstant()), calendar); + ZoneId.of("UTC")).toInstant())); if (null == authorityId) { this.psVoteSelect.setNull(2, Types.INTEGER); } else { @@ -144,7 +141,7 @@ class Database implements AutoCloseable { this.psVoteInsert.clearParameters(); this.psVoteInsert.setTimestamp(1, Timestamp.from(ZonedDateTime.of(validAfter, - ZoneId.of("UTC")).toInstant()), calendar); + ZoneId.of("UTC")).toInstant())); if (null == authorityId) { this.psVoteInsert.setNull(2, Types.INTEGER); } else { @@ -171,14 +168,13 @@ class Database implements AutoCloseable { Iterable<OutputLine> queryTotalcw() throws SQLException { List<OutputLine> statistics = new ArrayList<>(); Statement st = this.connection.createStatement(); - Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); String queryString = "SELECT " + OutputLine.columnHeadersDelimitedBy(", ") + " FROM totalcw"; try (ResultSet rs = st.executeQuery(queryString)) { while (rs.next()) { OutputLine outputLine = new OutputLine(); outputLine.validAfterDate = rs.getDate( - OutputLine.Column.VALID_AFTER_DATE.name(), calendar).toLocalDate(); + OutputLine.Column.VALID_AFTER_DATE.name()).toLocalDate(); outputLine.nickname = rs.getString(OutputLine.Column.NICKNAME.name()); outputLine.haveGuardFlag = rs.getBoolean( OutputLine.Column.HAVE_GUARD_FLAG.name()); diff --git a/src/main/java/org/torproject/metrics/stats/webstats/Main.java b/src/main/java/org/torproject/metrics/stats/webstats/Main.java index 17e21e9..bca86c5 100644 --- a/src/main/java/org/torproject/metrics/stats/webstats/Main.java +++ b/src/main/java/org/torproject/metrics/stats/webstats/Main.java @@ -31,11 +31,9 @@ import java.text.SimpleDateFormat; import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.ArrayList; -import java.util.Calendar; import java.util.List; import java.util.Map; import java.util.SortedSet; -import java.util.TimeZone; import java.util.TreeSet; /** Main class of the webstats module that downloads log files from the server, @@ -272,12 +270,10 @@ public class Main { Statement st = connection.createStatement(); String queryString = "SELECT " + ALL_COLUMNS + " FROM webstats"; DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); try (ResultSet rs = st.executeQuery(queryString)) { while (rs.next()) { statistics.add(String.format("%s,%s,%s,%s,%s,%s,%d", - dateFormat.format(rs.getDate(LOG_DATE, calendar)), + dateFormat.format(rs.getDate(LOG_DATE)), emptyNull(rs.getString(REQUEST_TYPE)), emptyNull(rs.getString(PLATFORM)), emptyNull(rs.getString(CHANNEL)),

1 0

[metrics-web/master] Set default locale US.
by karsten＠torproject.org 21 Feb '20

21 Feb '20

commit 9cd3ac5aa2cfccfa7c93c84be6f98059cafa181d Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Jan 15 09:22:10 2020 +0100 Set default locale US. Fixes #24532. --- CHANGELOG.md | 1 + .../bwhist/RelayDescriptorDatabaseImporter.java | 4 +--- .../torproject/metrics/stats/clients/Database.java | 7 ++----- .../metrics/stats/hidserv/Aggregator.java | 3 +-- .../stats/hidserv/ComputedNetworkFractions.java | 5 ++--- .../stats/hidserv/ExtrapolatedHidServStats.java | 6 ++---- .../torproject/metrics/stats/hidserv/Simulate.java | 5 ++--- .../org/torproject/metrics/stats/main/Main.java | 3 +++ .../torproject/metrics/stats/onionperf/Main.java | 9 ++++----- .../torproject/metrics/stats/servers/Database.java | 22 +++++++--------------- .../torproject/metrics/stats/totalcw/Database.java | 7 ++----- .../torproject/metrics/stats/webstats/Main.java | 6 ++---- .../torproject/metrics/web/DirectoryListing.java | 3 +-- .../org/torproject/metrics/web/NewsServlet.java | 3 +-- .../org/torproject/metrics/web/ServerMain.java | 3 +++ .../org/torproject/metrics/web/UpdateNews.java | 2 ++ 16 files changed, 36 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d5e6188..57694f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - Make Jetty host configurable. - Configure a base URL in order to turn ExoneraTor's permanent links into https:// links. + - Set default locale `US` at the beginning of the execution. # Changes in version 1.3.0 - 2019-11-09 diff --git a/src/main/java/org/torproject/metrics/stats/bwhist/RelayDescriptorDatabaseImporter.java b/src/main/java/org/torproject/metrics/stats/bwhist/RelayDescriptorDatabaseImporter.java index 09e7b67..2609188 100644 --- a/src/main/java/org/torproject/metrics/stats/bwhist/RelayDescriptorDatabaseImporter.java +++ b/src/main/java/org/torproject/metrics/stats/bwhist/RelayDescriptorDatabaseImporter.java @@ -28,7 +28,6 @@ import java.util.ArrayList; import java.util.Calendar; import java.util.HashSet; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.SortedSet; @@ -626,8 +625,7 @@ public final class RelayDescriptorDatabaseImporter { + "dirwrite"; statistics.add(columns.split(", ")); Statement st = this.conn.createStatement(); - Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), - Locale.US); + Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); String queryString = "SELECT " + columns + " FROM stats_bandwidth"; try (ResultSet rs = st.executeQuery(queryString)) { while (rs.next()) { diff --git a/src/main/java/org/torproject/metrics/stats/clients/Database.java b/src/main/java/org/torproject/metrics/stats/clients/Database.java index 3d8d63d..ac715fd 100644 --- a/src/main/java/org/torproject/metrics/stats/clients/Database.java +++ b/src/main/java/org/torproject/metrics/stats/clients/Database.java @@ -14,7 +14,6 @@ import java.time.Instant; import java.util.ArrayList; import java.util.Calendar; import java.util.List; -import java.util.Locale; import java.util.TimeZone; /** Database wrapper to connect to the database, insert data, run the stored @@ -95,8 +94,7 @@ class Database implements AutoCloseable { String columns = "date, node, country, transport, version, frac, users"; statistics.add(columns.split(", ")); Statement st = this.connection.createStatement(); - Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), - Locale.US); + Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); String queryString = "SELECT " + columns + " FROM estimated"; try (ResultSet rs = st.executeQuery(queryString)) { while (rs.next()) { @@ -120,8 +118,7 @@ class Database implements AutoCloseable { String columns = "date, node, country, transport, version, frac, low, high"; statistics.add(columns.split(", ")); Statement st = this.connection.createStatement(); - Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), - Locale.US); + Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); String queryString = "SELECT " + columns + " FROM combined"; try (ResultSet rs = st.executeQuery(queryString)) { while (rs.next()) { diff --git a/src/main/java/org/torproject/metrics/stats/hidserv/Aggregator.java b/src/main/java/org/torproject/metrics/stats/hidserv/Aggregator.java index 3ab1c1d..cb52598 100644 --- a/src/main/java/org/torproject/metrics/stats/hidserv/Aggregator.java +++ b/src/main/java/org/torproject/metrics/stats/hidserv/Aggregator.java @@ -13,7 +13,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Comparator; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.SortedMap; @@ -177,7 +176,7 @@ public class Aggregator { /* Put together all aggregated values in a single line. */ String date = e.getKey(); int numStats = weightedValues.size(); - sb.append(String.format(Locale.US, "%s,%s,%.0f,%.0f,%.0f,%.8f,%d%n", + sb.append(String.format("%s,%s,%.0f,%.0f,%.0f,%.8f,%d%n", date, type, weightedMean, weightedMedian, weightedInterquartileMean, sumFraction, numStats)); } diff --git a/src/main/java/org/torproject/metrics/stats/hidserv/ComputedNetworkFractions.java b/src/main/java/org/torproject/metrics/stats/hidserv/ComputedNetworkFractions.java index 4858588..3f3f12d 100644 --- a/src/main/java/org/torproject/metrics/stats/hidserv/ComputedNetworkFractions.java +++ b/src/main/java/org/torproject/metrics/stats/hidserv/ComputedNetworkFractions.java @@ -8,7 +8,6 @@ import org.slf4j.LoggerFactory; import java.util.Collections; import java.util.HashMap; -import java.util.Locale; import java.util.Map; /** Computed fraction of hidden-service activity that a single relay is @@ -117,9 +116,9 @@ public class ComputedNetworkFractions implements Document { validAfterDate); String second = validAfterHour + (this.fractionRendRelayedCells == 0.0 ? "," - : String.format(Locale.US, ",%f", this.fractionRendRelayedCells)) + : String.format(",%f", this.fractionRendRelayedCells)) + (this.fractionDirOnionsSeen == 0.0 ? "," - : String.format(Locale.US, ",%f", this.fractionDirOnionsSeen)); + : String.format(",%f", this.fractionDirOnionsSeen)); return new String[] { first, second }; } diff --git a/src/main/java/org/torproject/metrics/stats/hidserv/ExtrapolatedHidServStats.java b/src/main/java/org/torproject/metrics/stats/hidserv/ExtrapolatedHidServStats.java index ba42317..71048f3 100644 --- a/src/main/java/org/torproject/metrics/stats/hidserv/ExtrapolatedHidServStats.java +++ b/src/main/java/org/torproject/metrics/stats/hidserv/ExtrapolatedHidServStats.java @@ -6,8 +6,6 @@ package org.torproject.metrics.stats.hidserv; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Locale; - /** Extrapolated network totals of hidden-service statistics reported by a * single relay. Extrapolated values are based on reported statistics and * computed network fractions in the statistics interval. */ @@ -118,10 +116,10 @@ public class ExtrapolatedHidServStats implements Document { DateTimeHelper.ISO_DATE_FORMAT); String second = this.fingerprint + (this.fractionRendRelayedCells == 0.0 ? ",," - : String.format(Locale.US, ",%.0f,%f", + : String.format(",%.0f,%f", this.extrapolatedRendRelayedCells, this.fractionRendRelayedCells)) + (this.fractionDirOnionsSeen == 0.0 ? ",," - : String.format(Locale.US, ",%.0f,%f", + : String.format(",%.0f,%f", this.extrapolatedDirOnionsSeen, this.fractionDirOnionsSeen)); return new String[] { first, second }; } diff --git a/src/main/java/org/torproject/metrics/stats/hidserv/Simulate.java b/src/main/java/org/torproject/metrics/stats/hidserv/Simulate.java index 8374514..696fc1d 100644 --- a/src/main/java/org/torproject/metrics/stats/hidserv/Simulate.java +++ b/src/main/java/org/torproject/metrics/stats/hidserv/Simulate.java @@ -13,7 +13,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Random; import java.util.SortedMap; @@ -197,7 +196,7 @@ public class Simulate { totalInterquartileProbability += extrapolation[2]; } } - sb.append(String.format(Locale.US, "%d,%.2f,%.0f,%.0f,%.0f%n", run, + sb.append(String.format("%d,%.2f,%.0f,%.0f,%.0f%n", run, fraction, totalValues / totalProbability, weightedMedian, totalInterquartileValues / totalInterquartileProbability)); } @@ -352,7 +351,7 @@ public class Simulate { totalInterquartileProbability += extrapolation[2]; } } - sb.append(String.format(Locale.US, "%d,%.2f,%.0f,%.0f,%.0f%n", run, + sb.append(String.format("%d,%.2f,%.0f,%.0f,%.0f%n", run, fraction, totalValues / totalProbability, weightedMedian, totalInterquartileValues / totalInterquartileProbability)); } diff --git a/src/main/java/org/torproject/metrics/stats/main/Main.java b/src/main/java/org/torproject/metrics/stats/main/Main.java index 4ea15b4..3a4baf7 100644 --- a/src/main/java/org/torproject/metrics/stats/main/Main.java +++ b/src/main/java/org/torproject/metrics/stats/main/Main.java @@ -10,6 +10,7 @@ import java.nio.file.Files; import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.List; +import java.util.Locale; public class Main { @@ -29,6 +30,8 @@ public class Main { log.info("Starting metrics update run."); + Locale.setDefault(Locale.US); + File[] outputDirs = new File[] { modulesDir, statsDir }; for (File outputDir : outputDirs) { if (outputDir.exists()) { diff --git a/src/main/java/org/torproject/metrics/stats/onionperf/Main.java b/src/main/java/org/torproject/metrics/stats/onionperf/Main.java index 454e3b2..c8e3a90 100644 --- a/src/main/java/org/torproject/metrics/stats/onionperf/Main.java +++ b/src/main/java/org/torproject/metrics/stats/onionperf/Main.java @@ -27,7 +27,6 @@ import java.util.ArrayList; import java.util.Calendar; import java.util.HashSet; import java.util.List; -import java.util.Locale; import java.util.Set; import java.util.TimeZone; @@ -251,7 +250,7 @@ public class Main { Statement st = connection.createStatement(); String queryString = "SELECT date, filesize, source, server, q1, md, q3, " + "timeouts, failures, requests FROM onionperf"; - DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd", Locale.US); + DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); try (ResultSet rs = st.executeQuery(queryString)) { @@ -280,7 +279,7 @@ public class Main { Statement st = connection.createStatement(); String queryString = "SELECT date, source, position, q1, md, q3 " + "FROM buildtimes_stats"; - DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd", Locale.US); + DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); try (ResultSet rs = st.executeQuery(queryString)) { @@ -305,7 +304,7 @@ public class Main { Statement st = connection.createStatement(); String queryString = "SELECT date, source, server, low, q1, md, q3, high " + "FROM latencies_stats"; - DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd", Locale.US); + DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); try (ResultSet rs = st.executeQuery(queryString)) { @@ -332,7 +331,7 @@ public class Main { Statement st = connection.createStatement(); String queryString = "SELECT date, source, server, low, q1, md, q3, high " + "FROM throughput_stats"; - DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd", Locale.US); + DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); try (ResultSet rs = st.executeQuery(queryString)) { diff --git a/src/main/java/org/torproject/metrics/stats/servers/Database.java b/src/main/java/org/torproject/metrics/stats/servers/Database.java index fb3fdf5..8e5e31d 100644 --- a/src/main/java/org/torproject/metrics/stats/servers/Database.java +++ b/src/main/java/org/torproject/metrics/stats/servers/Database.java @@ -18,7 +18,6 @@ import java.util.Calendar; import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.SortedSet; @@ -267,8 +266,7 @@ class Database implements AutoCloseable { this.psStatusesSelect.clearParameters(); this.psStatusesSelect.setString(1, networkStatus.isRelay ? "relay" : "bridge"); - Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), - Locale.US); + Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); this.psStatusesSelect.setTimestamp(2, Timestamp.from(ZonedDateTime.of(networkStatus.timestamp, ZoneId.of("UTC")).toInstant()), calendar); @@ -414,8 +412,7 @@ class Database implements AutoCloseable { + "server_count_sum_avg, advertised_bandwidth_bytes_sum_avg"; statistics.add(columns.split(", ")); Statement st = this.connection.createStatement(); - Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), - Locale.US); + Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); String queryString = "SELECT " + columns + " FROM ipv6servers"; try (ResultSet rs = st.executeQuery(queryString)) { while (rs.next()) { @@ -443,8 +440,7 @@ class Database implements AutoCloseable { String columns = "date, isexit, isguard, advbw"; statistics.add(columns.split(", ")); Statement st = this.connection.createStatement(); - Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), - Locale.US); + Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); String queryString = "SELECT " + columns + " FROM bandwidth_advbw"; try (ResultSet rs = st.executeQuery(queryString)) { while (rs.next()) { @@ -465,8 +461,7 @@ class Database implements AutoCloseable { String columns = "date, relays, bridges"; statistics.add(columns.split(", ")); Statement st = this.connection.createStatement(); - Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), - Locale.US); + Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); String queryString = "SELECT " + columns + " FROM servers_networksize"; try (ResultSet rs = st.executeQuery(queryString)) { while (rs.next()) { @@ -486,8 +481,7 @@ class Database implements AutoCloseable { String columns = "date, flag, relays"; statistics.add(columns.split(", ")); Statement st = this.connection.createStatement(); - Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), - Locale.US); + Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); String queryString = "SELECT " + columns + " FROM servers_relayflags"; try (ResultSet rs = st.executeQuery(queryString)) { while (rs.next()) { @@ -507,8 +501,7 @@ class Database implements AutoCloseable { String columns = "date, version, relays"; statistics.add(columns.split(", ")); Statement st = this.connection.createStatement(); - Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), - Locale.US); + Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); String queryString = "SELECT " + columns + " FROM servers_versions"; try (ResultSet rs = st.executeQuery(queryString)) { while (rs.next()) { @@ -528,8 +521,7 @@ class Database implements AutoCloseable { String columns = "date, platform, relays"; statistics.add(columns.split(", ")); Statement st = this.connection.createStatement(); - Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), - Locale.US); + Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); String queryString = "SELECT " + columns + " FROM servers_platforms"; try (ResultSet rs = st.executeQuery(queryString)) { while (rs.next()) { diff --git a/src/main/java/org/torproject/metrics/stats/totalcw/Database.java b/src/main/java/org/torproject/metrics/stats/totalcw/Database.java index a35099f..3d51604 100644 --- a/src/main/java/org/torproject/metrics/stats/totalcw/Database.java +++ b/src/main/java/org/torproject/metrics/stats/totalcw/Database.java @@ -17,7 +17,6 @@ import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Calendar; import java.util.List; -import java.util.Locale; import java.util.TimeZone; /** Database wrapper to connect to the database, insert data, run the stored @@ -122,8 +121,7 @@ class Database implements AutoCloseable { private void insertStatusIfAbsent(LocalDateTime validAfter, Integer authorityId, long[] measuredSums) throws SQLException { - Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), - Locale.US); + Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); this.psVoteSelect.clearParameters(); this.psVoteSelect.setTimestamp(1, Timestamp.from(ZonedDateTime.of(validAfter, @@ -173,8 +171,7 @@ class Database implements AutoCloseable { Iterable<OutputLine> queryTotalcw() throws SQLException { List<OutputLine> statistics = new ArrayList<>(); Statement st = this.connection.createStatement(); - Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), - Locale.US); + Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); String queryString = "SELECT " + OutputLine.columnHeadersDelimitedBy(", ") + " FROM totalcw"; try (ResultSet rs = st.executeQuery(queryString)) { diff --git a/src/main/java/org/torproject/metrics/stats/webstats/Main.java b/src/main/java/org/torproject/metrics/stats/webstats/Main.java index f581c8a..17e21e9 100644 --- a/src/main/java/org/torproject/metrics/stats/webstats/Main.java +++ b/src/main/java/org/torproject/metrics/stats/webstats/Main.java @@ -33,7 +33,6 @@ import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Calendar; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.SortedSet; import java.util.TimeZone; @@ -272,10 +271,9 @@ public class Main { SortedSet<String> statistics = new TreeSet<>(); Statement st = connection.createStatement(); String queryString = "SELECT " + ALL_COLUMNS + " FROM webstats"; - DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd", Locale.US); + DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), - Locale.US); + Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); try (ResultSet rs = st.executeQuery(queryString)) { while (rs.next()) { statistics.add(String.format("%s,%s,%s,%s,%s,%s,%d", diff --git a/src/main/java/org/torproject/metrics/web/DirectoryListing.java b/src/main/java/org/torproject/metrics/web/DirectoryListing.java index 09f2aaa..aceec59 100644 --- a/src/main/java/org/torproject/metrics/web/DirectoryListing.java +++ b/src/main/java/org/torproject/metrics/web/DirectoryListing.java @@ -18,7 +18,6 @@ import java.net.URLConnection; import java.util.ArrayList; import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.SortedSet; import java.util.TreeSet; @@ -201,7 +200,7 @@ public class DirectoryListing extends HashMap<String, List<String[]>> int exp = (int) (Math.log(bytes) / Math.log(1024)); char pre = "KMGTPE".charAt(exp - 1); return String - .format(Locale.US, "%.1f %siB", bytes / Math.pow(1024, exp), pre); + .format("%.1f %siB", bytes / Math.pow(1024, exp), pre); } } diff --git a/src/main/java/org/torproject/metrics/web/NewsServlet.java b/src/main/java/org/torproject/metrics/web/NewsServlet.java index c91233a..b5dddc1 100644 --- a/src/main/java/org/torproject/metrics/web/NewsServlet.java +++ b/src/main/java/org/torproject/metrics/web/NewsServlet.java @@ -8,7 +8,6 @@ import java.util.ArrayList; import java.util.Calendar; import java.util.LinkedHashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.TimeZone; @@ -49,7 +48,7 @@ public class NewsServlet extends AnyServlet { HttpServletResponse response) throws IOException, ServletException { /* Create categories based on current system time. */ Map<String, String[]> cutOffDates = new LinkedHashMap<>(); - Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.US); + Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC")); cal.set(Calendar.DAY_OF_WEEK, 1); cutOffDates.put(String.format("%tF", cal), new String[] { "This week", "week" }); diff --git a/src/main/java/org/torproject/metrics/web/ServerMain.java b/src/main/java/org/torproject/metrics/web/ServerMain.java index db01bc5..bb03086 100644 --- a/src/main/java/org/torproject/metrics/web/ServerMain.java +++ b/src/main/java/org/torproject/metrics/web/ServerMain.java @@ -9,12 +9,15 @@ import org.eclipse.jetty.xml.XmlConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Locale; + public class ServerMain { private static final Logger log = LoggerFactory.getLogger(ServerMain.class); /** Starts the web server listening for incoming client connections. */ public static void main(String[] args) { + Locale.setDefault(Locale.US); try { Resource jettyXml = Resource.newSystemResource("jetty.xml"); log.info("Reading configuration from '{}'.", jettyXml); diff --git a/src/main/java/org/torproject/metrics/web/UpdateNews.java b/src/main/java/org/torproject/metrics/web/UpdateNews.java index 92103fe..8f4440e 100644 --- a/src/main/java/org/torproject/metrics/web/UpdateNews.java +++ b/src/main/java/org/torproject/metrics/web/UpdateNews.java @@ -14,6 +14,7 @@ import java.io.InputStreamReader; import java.net.URL; import java.util.ArrayList; import java.util.List; +import java.util.Locale; public class UpdateNews { @@ -21,6 +22,7 @@ public class UpdateNews { /** Update news. */ public static void main(String[] args) throws Exception { + Locale.setDefault(Locale.US); URL textFile = new URL( "https://trac.torproject.org/projects/tor/wiki/doc/" + "MetricsTimeline?format=txt");

1 0

[tpo/master] Makes /download min-header work in all locales
by antonela＠torproject.org 21 Feb '20

21 Feb '20

commit 04f067300ae8dc89e155d8e786f683b611a3f8dc Author: Antonela <hola(a)antonela.me> Date: Fri Feb 21 13:48:14 2020 -0300 Makes /download min-header work in all locales https://dip.torproject.org/torproject/web/tpo/issues/59 --- templates/meta.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/meta.html b/templates/meta.html index 5a899f2..9e9c26b 100644 --- a/templates/meta.html +++ b/templates/meta.html @@ -20,7 +20,7 @@ <title>{{ _("Tor Project") }} | {% block title %}{{ this.title }}{% endblock %}</title> <body class="no-gutters"> <header> - {% if this.section == "Download" %} + {% if this.path == '/download' %} {% include 'navbar-min.html' %} {% else %} {% include 'navbar.html' %}

1 0

[tpo/master] Donate button is back
by antonela＠torproject.org 21 Feb '20

21 Feb '20

commit dd1e1ff689b5af8924e35af2da14dbaec4366089 Author: Antonela <hola(a)antonela.me> Date: Fri Feb 21 13:23:36 2020 -0300 Donate button is back --- templates/navbar-min.html | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/templates/navbar-min.html b/templates/navbar-min.html index f9d2bc7..3eb4ba1 100644 --- a/templates/navbar-min.html +++ b/templates/navbar-min.html @@ -14,6 +14,13 @@ <span class="sr-only">Tor Logo</span> </a> + {% set link = bag('links', this.alt, 'donate') %} + {% if link %} + <h4 class="pl-2 pr-2"><a href="{{ link }}" title="{{ _("Donate") }}"><span class="badge badge-warning p-2">{{ _("Donate Now") }}</span></a></h4> + {% else %} + <h4 class="pl-2 pr-2"><a href="https://www.torproject.org/donate/{{ this.alt }}" title="{{ _("Donate") }}"><span class="badge badge-warning p-2">{{ _("Donate Now") }}</span></a></h4> + {% endif %} + </div> </nav> </div>

1 0

[tpo/master] Add minimal footer
by antonela＠torproject.org 21 Feb '20

21 Feb '20

commit 308e94ceb25c4511daaea8bea99e6b58c7086678 Author: Antonela <hola(a)antonela.me> Date: Fri Feb 21 13:24:08 2020 -0300 Add minimal footer to avoid duplicated flows in the download page --- templates/footer-min.html | 92 +++++++++++++++++++++++++++++++++++++++++++++++ templates/layout.html | 6 +++- 2 files changed, 97 insertions(+), 1 deletion(-) diff --git a/templates/footer-min.html b/templates/footer-min.html new file mode 100644 index 0000000..cd0a025 --- /dev/null +++ b/templates/footer-min.html @@ -0,0 +1,92 @@ +{% if this.path == '/download' %} + {% include 'download-android.html' %} +{% endif %} +{% set locale = bag('alternatives', this.alt) %} +<div class="container-fluid bg-dark footer onion-bg {{ locale.order }} {{ locale.direction }}"> + <div class="row"> + <div class="col-12 d-flex justify-content-center"> + <div class="col-sm-6 offset-lx-1 offset-sm-0 mt-5"> + <h5 class="font-weight-bold mt-5 text-white">{{ _('Our mission:') }}</h5> + <p class="text-white mb-5" >{{ _('To advance human rights and freedoms by creating and deploying free and open source anonymity and privacy technologies, supporting their unrestricted availability and use, and furthering their scientific and popular understanding.') }}</p> + </div> + <div class="col-sm-2 mt-5 d-none d-sm-block"> + <ul class="nav flex-column mt-sm-5"> + {% set menubag = 'menu_footer' %} + {% for id, item in bag(menubag).items() %} + <li{% if this.is_child_of(item.path) %} class="nav-item active" {% else %} class="nav-item"{% endif %}> + {% set link = site.get(item.path) %} + {% if link %} + <a class="nav-link text-light" href="{{ item.path|url(alt=this.alt) }}"> + {% else %} + {% set local = bag('links', this.alt, id) %} + {% if local %} + <a class="nav-link text-light" href="{{ local }}"> + {% else %} + <a class="nav-link text-light" href="{{ item.path }}"> + {% endif %} + {% endif %} + {{ _(item.label) }} + {% if this.is_child_of(item.path) %} + <span class="sr-only">(current)</span> + {% endif %} + </a> + </li> + {% endfor %} + </ul> + {% set link = bag('links', this.alt, 'donate') %} + {% if link %} + <a href="{{ link }}" title="{{ _("Donate") }}"><h5><span class="badge badge-warning p-2 mt-2">{{ _("Donate Now") }}</span></h5></a> + {% else %} + <a href="https://www.torproject.org/donate/{{ this.alt }}" title="{{ _("Donate") }}"><h4><span class="badge badge-warning p-2">{{ _("Donate Now") }}</span></h4></a> + {% endif %} + </div> + </div> + <div class="col-12 d-flex justify-content-center border"> + <div class="col-sm-6 offset-lx-1 offset-sm-0 mt-5"> + <p class="text-primary-light text-uppercase">{{ _('Subscribe to our Newsletter') }}</p> + <p class="text-light">{{ _('Get monthly updates and opportunities from the Tor Project:') }}</p> + <p class="w"><a class="btn btn-small btn-dark mt-2 text-white text-uppercase active" role="button" aria-pressed="true" href="https://newsletter.torproject.org/">{{ _('Sign up') }}</a></p> + </div> + <div class="col-sm-2 mt-5 d-none d-sm-block"> + <div class="row"> + <h4><a class="text-white px-3" target="_blank" href="https://www.facebook.com/TorProject/"><i class="fab fa-facebook-png"></i></a></h4> + <h4><a class="mastodon text-white px-3" target="_blank" href="https://mastodon.social/@torproject"><i class="fab fa-mastodon-png"></i></a></h4> + <h4><a class="twitter text-white px-3" target="_blank" href="https://twitter.com/torproject"><i class="fab fa-twitter-png"></i></a></h4> + </div> + <div class="row"> + <h4><a class="instagram text-white px-3" target="_blank" href="https://www.instagram.com/torproject"><i class="fab fa-instagram-png"></i></a></h4> + <h4><a class="linkedin text-white px-3" target="_blank" href="https://www.linkedin.com/company/tor-project"><i class="fab fa-linkedin-png"></i></a></h4> + <h4><a class="github text-white px-3" target="_blank" href="https://github.com/torproject"><i class="fab fa-github-png"></i></a></h4> + </div> + </div> + </div> + + <div class="col-12 d-flex justify-content-center"> + <div class="col-sm-6 offset-lx-1 offset-sm-0 mt-5"> + {% from "macros/footer.html" import render_faq %} + <p class="text-primary text-light">{{ _('Trademark, copyright notices, and rules for use by third parties can be found in our %(link_to_faq)s')|format(link_to_faq=render_faq())|safe }}.</p> + </div> + <div class="col-sm-2 mt-5 d-none d-sm-block"> + <div class="btn-group dropdown float-right"> + <button type="button" class="btn btn-primary dropdown-toggle" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false"> + {{ bag('alternatives', this.alt, 'language') }} + </button> + <div class="dropdown-menu"> + {% for id, item in bag('alternatives').items() %} + {% if this.alt != id %} + <a class="dropdown-item" href="{{ this.path|url(alt=id) }}">{{ item.language }}</a> + {% endif %} + {% endfor %} + </div> + </div> + </div> + </div> + <script src="{{ '/static/js/jquery-3.2.1.min.js'|asseturl }}" ></script> + <script src="{{ '/static/js/popper.min.js'|asseturl }}" ></script> + <script src="{{ '/static/js/bootstrap.bundle.min.js'|asseturl }}" ></script> + <script src="{{ '/static/js/scrollspy.min.js'|asseturl }}" ></script> + <script src="{{ '/static/js/modernizr.js'|asseturl }}" ></script> + <script src="{{ '/static/js/download.js'|asseturl }}" ></script> + <script src="{{ '/static/js/fallback.js'|asseturl }}" ></script> + </div> +</div> diff --git a/templates/layout.html b/templates/layout.html index ab604b8..86001dd 100644 --- a/templates/layout.html +++ b/templates/layout.html @@ -13,6 +13,10 @@ </div> </div> </div> - {% include 'footer.html' %} + {% if this.section == "Download" %} + {% include 'footer-min.html' %} + {% else %} + {% include 'footer.html' %} + {% endif %} </body> </html>

1 0

Jump to page:

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits February 2020