September 2019 - tor-commits - lists.torproject.org

[collector/master] Tweak URL-based unit tests.
by karsten＠torproject.org 27 Sep '19

27 Sep '19

commit 834ef7c5e76778e31f9ca8569e69423b32700acf Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Sep 27 12:23:19 2019 +0200 Tweak URL-based unit tests. Turns out that one of the tests that we disabled earlier was only slow, because creating a URL instance to https://something.dummy.org/ triggered an actual DNS resolve. There's absolutely no need to do this, so we can as well use localhost in all our test URLs. --- .../org/torproject/metrics/collector/conf/ConfigurationTest.java | 5 ++--- .../torproject/metrics/collector/downloader/DownloaderTest.java | 8 ++++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java b/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java index 67a9082..7845909 100644 --- a/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java +++ b/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java @@ -128,13 +128,12 @@ public class ConfigurationTest { } } - @Ignore("This test takes 40 seconds, which is too long.") @Test() public void testUrlArrayValues() throws Exception { URL[] array = new URL[randomSource.nextInt(30) + 1]; for (int i = 0; i < array.length; i++) { - array[i] = new URL("https://" - + Integer.toBinaryString(randomSource.nextInt(100)) + ".dummy.org"); + array[i] = new URL("http://localhost/" + + Integer.toBinaryString(randomSource.nextInt(100))); } String input = Arrays.toString(array).replace("[", "").replace("]", "") diff --git a/src/test/java/org/torproject/metrics/collector/downloader/DownloaderTest.java b/src/test/java/org/torproject/metrics/collector/downloader/DownloaderTest.java index 19db782..0617940 100644 --- a/src/test/java/org/torproject/metrics/collector/downloader/DownloaderTest.java +++ b/src/test/java/org/torproject/metrics/collector/downloader/DownloaderTest.java @@ -85,7 +85,7 @@ public class DownloaderTest { @Test public void testExistingResource() throws Exception { - URL requestedUrl = new URL("http://example.org/exists"); + URL requestedUrl = new URL("http://localhost/exists"); byte[] expectedDownloadedBytes = "content".getBytes(); HttpURLConnection urlConnection = mock(HttpURLConnection.class); httpUrlStreamHandler.addConnection(requestedUrl, urlConnection); @@ -98,7 +98,7 @@ public class DownloaderTest { @Test public void testNonExistingResource() throws Exception { - URL requestedUrl = new URL("http://example.org/notfound"); + URL requestedUrl = new URL("http://localhost/notfound"); HttpURLConnection urlConnection = mock(HttpURLConnection.class); httpUrlStreamHandler.addConnection(requestedUrl, urlConnection); given(urlConnection.getResponseCode()).willReturn(404); @@ -108,7 +108,7 @@ public class DownloaderTest { @Test public void testEmptyResource() throws Exception { - URL requestedUrl = new URL("http://example.org/empty"); + URL requestedUrl = new URL("http://localhost/empty"); byte[] expectedDownloadedBytes = new byte[0]; HttpURLConnection urlConnection = mock(HttpURLConnection.class); httpUrlStreamHandler.addConnection(requestedUrl, urlConnection); @@ -121,7 +121,7 @@ public class DownloaderTest { @Test(expected = SocketTimeoutException.class) public void testTimeout() throws Exception { - URL requestedUrl = new URL("http://example.org/timeout"); + URL requestedUrl = new URL("http://localhost/timeout"); SocketTimeoutException expectedException = new SocketTimeoutException(); HttpURLConnection urlConnection = mock(HttpURLConnection.class); httpUrlStreamHandler.addConnection(requestedUrl, urlConnection);

1 0

[translation/tpo-web] https://gitweb.torproject.org/translation.git/commit/?h=tpo-web
by translation＠torproject.org 27 Sep '19

27 Sep '19

commit ba56477c599e17f417cfe203a1e34a549f3b94ee Author: Translation commit bot <translation(a)torproject.org> Date: Fri Sep 27 10:22:49 2019 +0000 https://gitweb.torproject.org/translation.git/commit/?h=tpo-web --- contents+ca.po | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/contents+ca.po b/contents+ca.po index a2cb03d1b..bfcfcb809 100644 --- a/contents+ca.po +++ b/contents+ca.po @@ -2,8 +2,8 @@ # Jaime Muñoz Martín <jmmartin_5(a)outlook.com>, 2019 # erinm, 2019 # jmontane, 2019 -# Ecron <ecron_89(a)hotmail.com>, 2019 # Emma Peel, 2019 +# Ecron <ecron_89(a)hotmail.com>, 2019 # msgid "" msgstr "" @@ -11,7 +11,7 @@ msgstr "" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2019-09-17 15:46+CET\n" "PO-Revision-Date: 2019-03-09 10:41+0000\n" -"Last-Translator: Emma Peel, 2019\n" +"Last-Translator: Ecron <ecron_89(a)hotmail.com>, 2019\n" "Language-Team: Catalan (https://www.transifex.com/otf/teams/1519/ca/)\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -21,7 +21,7 @@ msgstr "" #: https//www.torproject.org/ (content/contents+en.lrpage.title) msgid "Anonymity Online" -msgstr "Anonimat a la xarxa" +msgstr "Anonimat en la xarxa" #: https//www.torproject.org/ (content/contents+en.lrpage.section) msgid "home" @@ -45,7 +45,7 @@ msgid "" "We advance human rights and defend your privacy online through free software" " and open networks. [Meet our team](about/people)." msgstr "" -"Fem avançar els drets humans i defenem la vostra privadesa a la xarxa " +"Fem avançar els drets humans i defenem la vostra privadesa en la xarxa " "mitjançant programari lliure i xarxes obertes. [Coneixeu el nostre " "equip](about/people)." @@ -203,6 +203,10 @@ msgid "" "bugs](https://support.torproject.org/misc/bug-or-feedback/), and are not " "putting yourself at risk." msgstr "" +"Baixeu la versió alfa només si no us importa que algunes coses no funcionin " +"correctament, si voleu ajudar-nos a trobar i [notificar " +"errors](https://support.torproject.org/misc/bug-or-feedback/) i si no us " +"esteu posant en risc." #: https//www.torproject.org/download/languages/ #: (content/download/languages/contents+en.lrpage.title) @@ -630,7 +634,7 @@ msgstr "" #: https//www.torproject.org/about/trademark/trademark-faq/ #: (content/about/trademark/trademark-faq/contents+en.lrtopic.title) msgid "Tor Trademark Frequently Asked Questions" -msgstr "" +msgstr "Preguntes Més Freqüents sobre la marca Tor" #: lego/templates/footer.html:9 lego/templates/footer.html:18 #: lego/templates/navbar.html:73 templates/footer.html:9 @@ -933,7 +937,7 @@ msgstr "" #: templates/hero-download.html:45 msgid "Download in another language or platform" -msgstr "Baixeu-lo en altra llengua o plataforma" +msgstr "Baixeu-lo en una altra llengua o plataforma" #: templates/hero-download.html:46 msgid "Download the latest alpha build" @@ -1059,7 +1063,7 @@ msgstr "" #: templates/meta.html:17 msgid "Tor Project" -msgstr "Projecte Tor" +msgstr "Tor Project" #: templates/people.html:2 msgid "Board of Directors"

1 0

[gettor/master] Script update_files should delete branches and prune repository before uploading files
by hiro＠torproject.org 27 Sep '19

27 Sep '19

commit 3bcd5eaca4e05dd6bae3062c03ed80c4b4eb5324 Author: hiro <hiro(a)torproject.org> Date: Fri Sep 6 12:56:40 2019 +0200 Script update_files should delete branches and prune repository before uploading files --- scripts/update_files | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/update_files b/scripts/update_files index ce8fb88..924db45 100755 --- a/scripts/update_files +++ b/scripts/update_files @@ -25,7 +25,7 @@ for row in $( if [[ $r = *[!\ ]* ]]; then wget $r git fetch --all - git checkout releases + git checkout -b releases git add . git commit -m '[dist ci] commit from CI runner - update with new torbrowser downloads' diffs=$(git diff origin/releases) @@ -34,8 +34,7 @@ for row in $( else git push -f --follow-tags origin releases fi - git checkout torbrowser-releases - git merge releases + git checkout -b torbrowser-releases diffs=$(git diff github/torbrowser-releases) if [ -z "$diffs" ]; then echo "No new releases"

1 0

[gettor/master] Remove previous provider folder
by hiro＠torproject.org 27 Sep '19

27 Sep '19

1 0

[gettor/master] Use file name to help people locate file in gdrive
by hiro＠torproject.org 27 Sep '19

27 Sep '19

commit 1c2e45107d18dcc24a3f214095dd8483a8570c31 Author: hiro <hiro(a)torproject.org> Date: Fri Sep 6 13:55:32 2019 +0200 Use file name to help people locate file in gdrive --- gettor/services/email/sendmail.py | 3 ++- scripts/add_links_to_db | 2 +- scripts/create_db | 4 ++-- share/locale/en.json | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/gettor/services/email/sendmail.py b/gettor/services/email/sendmail.py index fd01257..15cb728 100644 --- a/gettor/services/email/sendmail.py +++ b/gettor/services/email/sendmail.py @@ -182,6 +182,7 @@ class Sendmail(object): version = link[4] arch = link[3] url = link[0] + file = link[6] link_str = "Tor Browser {} for {}-{}-{} ({}): {}".format( version, platform, locale, arch, provider, url @@ -192,7 +193,7 @@ class Sendmail(object): else: link_msg = link_str - body_msg = strings._("links_body").format(platform, link_msg) + body_msg = strings._("links_body").format(platform, link_msg, file) subject_msg = strings._("links_subject") hid = hashlib.sha256(id.encode('utf-8')) diff --git a/scripts/add_links_to_db b/scripts/add_links_to_db index a0014ed..74d4b69 100755 --- a/scripts/add_links_to_db +++ b/scripts/add_links_to_db @@ -142,7 +142,7 @@ def main(): release_link = releases.get(k).replace("en-US", l) c.execute( "INSERT INTO links(link, platform, language, arch, version, provider, status)" - "VALUES ('%s', '%s', '%s', '64', '%s', '%s', 'ACTIVE')" % (providers.get(p) + release_link, k, l, version, p)) + "VALUES ('%s', '%s', '%s', '64', '%s', '%s', 'ACTIVE', '%s')" % (providers.get(p) + release_link, k, l, version, p, release_link)) if __name__ == "__main__": diff --git a/scripts/create_db b/scripts/create_db index 32121df..ed4ee34 100755 --- a/scripts/create_db +++ b/scripts/create_db @@ -87,7 +87,7 @@ def main(): ) c.execute( "CREATE TABLE links(link TEXT, platform TEXT, language TEXT," - " arch TEXT, version TEXT, provider TEXT, status TEXT)" + " arch TEXT, version TEXT, provider TEXT, status TEXT, file TEXT)" ) c.execute( "CREATE TABLE stats(num_requests NUMBER, platform TEXT," @@ -122,7 +122,7 @@ def main(): ) c.execute( "CREATE TABLE links(link TEXT, platform TEXT, language TEXT," - "arch TEXT, version TEXT, provider TEXT, status TEXT," + "arch TEXT, version TEXT, provider TEXT, status TEXT, file TEXT" "PRIMARY KEY(platform, arch, version, provider, status))" ) c.execute( diff --git a/share/locale/en.json b/share/locale/en.json index 3006c7c..8e99ef1 100644 --- a/share/locale/en.json +++ b/share/locale/en.json @@ -10,7 +10,7 @@ "smtp_help_subject": "[GetTor] Help", "smtp_unsupported_locale_subject": "[GetTor] Unsupported locale", "smtp_unsupported_locale_msg": "The locale you requested '{}' is not supported.", - "smtp_vlinks_msg": "You requested Tor Browser for {}.\n\nYou will need only one of the links below to download the bundle. If a link does not work for you, try the next one.\n\n{}\n\n \n--\nGetTor", + "smtp_vlinks_msg": "You requested Tor Browser for {}.\n\nYou will need only one of the links below to download the bundle. If a link does not work for you, try the next one.\n\n{}\n\nShould you have issues with any of the links above you can access the following Google Drive folder: https://drive.google.com/open?id=13CADQTsCwrGsIID09YQbNz2DfRMUoxUU\n\n Download the file: {}\n\n \n--\nGetTor", "smtp_mirrors_msg": "Hi! this is the GetTor robot.\n\nThank you for your request. Attached to this email you will find\nan updated list of mirrors of Tor Project's website.", "smtp_help_msg": "Hi! This is the GetTor robot. I am here to help you download the\nlatest version of Tor Browser.\n\nPlease reply to this message with one of the options below:\n\nwindows\nlinux\nosx\nmirrors\n\nI will then send you the download instructions.\n\nIf you are unsure, just send a blank reply to this message." }

1 0

[gettor/master] Script update_files should update to gdrive too
by hiro＠torproject.org 27 Sep '19

27 Sep '19

commit e4ccf5f44a473068a28d159a1e405b63edde55fd Author: hiro <hiro(a)torproject.org> Date: Fri Sep 6 13:43:30 2019 +0200 Script update_files should update to gdrive too --- scripts/update_files | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/scripts/update_files b/scripts/update_files index 924db45..b1253c1 100755 --- a/scripts/update_files +++ b/scripts/update_files @@ -10,8 +10,13 @@ # :license: This is Free Software. See LICENSE for license information. cd ~/releases -git checkout releases -rm -f torbrowser-* TorBrowser-* tor-browser-* +git checkout naster +git branch -D releases +git push origin --delete releases +git branch -D torbrowser-releases +git push github --delete torbrowser-releases + +git fetch --all --prune for row in $( curl -s 'https://aus1.torproject.org/torbrowser/update_3/release/downloads.json' | @@ -43,3 +48,9 @@ for row in $( fi fi; done + +rclone delete gdrive:releases + +for f in $(ls); do + rclone copy $f gdrive:releases +done

1 0

[gettor/master] Gettor should include signatures
by hiro＠torproject.org 27 Sep '19

27 Sep '19

commit 3d75cb6fe8eb4c04a3944a73806af7b4f4c976bc Author: hiro <hiro(a)torproject.org> Date: Fri Sep 6 14:03:17 2019 +0200 Gettor should include signatures --- gettor/services/email/sendmail.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gettor/services/email/sendmail.py b/gettor/services/email/sendmail.py index 15cb728..7951e9d 100644 --- a/gettor/services/email/sendmail.py +++ b/gettor/services/email/sendmail.py @@ -183,11 +183,14 @@ class Sendmail(object): arch = link[3] url = link[0] file = link[6] + sig_url = url + ".asc" - link_str = "Tor Browser {} for {}-{}-{} ({}): {}".format( + link_str = "Tor Browser {} for {}-{}-{} ({}): {}\n".format( version, platform, locale, arch, provider, url ) + link_str += "Signature file: {}\n".format(sig_url) + if link_msg: link_msg = "{}\n{}".format(link_msg, link_str) else:

1 0

[gettor/master] Add checks for email service
by hiro＠torproject.org 27 Sep '19

27 Sep '19

commit 98862a5f9e423a4b65a550620858342f72c5dd2c Author: hiro <hiro(a)torproject.org> Date: Thu Sep 26 13:23:35 2019 +0200 Add checks for email service --- bin/gettor_check | 24 +++++++++ scripts/check_service | 135 ++++++++++++++++++++++++++++++++++++++++++++++++++ scripts/update_files | 4 +- 3 files changed, 162 insertions(+), 1 deletion(-) diff --git a/bin/gettor_check b/bin/gettor_check new file mode 100755 index 0000000..55fd1a3 --- /dev/null +++ b/bin/gettor_check @@ -0,0 +1,24 @@ +#!/bin/bash +# +# This file is part of GetTor, a Tor Browser distribution system. +# +# :authors: Hiro <hiro(a)torproject.org> +# see also AUTHORS file +# +# :copyright: (c) 2008-2019, The Tor Project, Inc. +# +# :license: This is Free Software. See LICENSE for license information. + +cd ; +virtualenv gettor/venv +source gettor/venv/bin/activate + +python3 gettor/scripts/check_service $1 + +# store exit status +status=$? + +if test $status -ne 0 +then + ./bin/gettor_service start +fi diff --git a/scripts/check_service b/scripts/check_service new file mode 100644 index 0000000..38bb1a2 --- /dev/null +++ b/scripts/check_service @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# This file is part of GetTor, a Tor Browser distribution system. +# +# :authors: hiro <hiro(a)torproject.org> +# see also AUTHORS file +# +# :license: This is Free Software. See LICENSE for license information. + +import sys +import smtplib +import time +import imaplib +import email +import time + +# Standard Nagios return codes +OK, WARNING, CRITICAL, UNKNOWN = range(4) + +ORG_EMAIL = "@gmail.com" +FROM_EMAIL = "test.gettor.browser" + ORG_EMAIL +SMTP_SERVER = "imap.gmail.com" +SMTP_PORT = 993 + +MESSAGE_FROM = "gettor(a)torproject.org" +MESSAGE_SUBJECT = "[GetTor] Links for your request" +MESSAGE_BODY = "https://gitlab.com/thetorproject/" + +STATUS_FILE = "/srv/gettor.torproject.org/check/status" + +# ------------------------------------------------- +# +# Utility to read email from Gmail Using Python +# +# ------------------------------------------------ + +def test_email_from_gmail(password): + try: + mail = imaplib.IMAP4_SSL(SMTP_SERVER) + mail.login(FROM_EMAIL, password) + mail.select('INBOX') + + type, data = mail.search(None, 'ALL') + mail_ids = data[0] + + id_list = mail_ids.split() + first_email_id = int(str(id_list[0], 'utf-8')) + latest_email_id = int(str(id_list[-1], 'utf-8')) + + for i in range(int(latest_email_id), int(first_email_id), -1): + typ, data = mail.fetch(str(i), '(RFC822)') + + for response_part in data: + if isinstance(response_part, tuple): + m = str(response_part[1], 'utf-8') + msg = email.message_from_string(m) + email_subject = "{}".format(msg['subject']) + email_from = "{}".format(msg['from']) + email_body = "{}".format(msg.as_string()) + + if (MESSAGE_FROM == email_from) and (MESSAGE_SUBJECT == email_subject) and (MESSAGE_BODY in email_body): + mail.store(str(i), '+FLAGS', '\\Deleted') + return OK, "Found correct gettor email." + else: + print('if not working') + mail.store(str(i), '+FLAGS', '\\Deleted') + + + + + return WARNING, "No emails from gettor found" + + except Exception as e: + return CRITICAL, str(e) + +def send_email_from_gmail(password): + sent_from = FROM_EMAIL + sent_to = [MESSAGE_FROM] + subject = 'windows en' + body = 'windows en' + + email_text = """From: %s\nTo: %s\nSubject: %s\n\n%s + """ % (sent_from, ", ".join(sent_to), subject, body) + + try: + server = smtplib.SMTP_SSL('smtp.gmail.com', 465) + server.ehlo() + server.login(sent_from, password) + server.sendmail(sent_from, to, email_text) + server.close() + except Exception as e: + return UNKNOWN, str(e) + +if __name__ == "__main__": + status, message = None, None + + host = None + use_https = False + + if len(sys.argv) == 2: + password = sys.argv[1] + else: + password = "yourPassword" + + status_file = open(STATUS_FILE,'w') + + try: + status, message = send_email_from_gmail(password) + except Exception as e: + status = CRITICAL + message = repr(e) + sys.exit(status) + + time.sleep(180) + + try: + status, message = test_email_from_gmail(password) + except KeyboardInterrupt: + status, message = CRITICAL, "Caught Control-C..." + except Exception as e: + status = CRITICAL + message = repr(e) + finally: + if status == OK: + status_file.write("OK: %s" % message) + elif status == WARNING: + status_file.write("WARNING: %s" % message) + elif status == CRITICAL: + status_file.write("CRITICAL: %s" % message) + else: + status_file.write("UNKNOWN: %s" % message) + status = UNKNOWN + + sys.exit(status) diff --git a/scripts/update_files b/scripts/update_files index df800d6..dac62e4 100755 --- a/scripts/update_files +++ b/scripts/update_files @@ -10,7 +10,7 @@ # :license: This is Free Software. See LICENSE for license information. cd ~/releases -git checkout naster +git checkout master git branch -D releases git push origin --delete releases git branch -D torbrowser-releases @@ -52,6 +52,8 @@ done rclone delete gdrive:releases for f in $(ls); do + # Update Google Drive rclone copy $f gdrive:releases + # Update Internet Archive ia upload <identifier> - --remote-name=$f --metadata="title:New Tor Browser release ${f}." done

1 0

[gettor/master] Add upload to archive.org
by hiro＠torproject.org 27 Sep '19

27 Sep '19

commit 8bd76e1060c9ce917d8d8a400318bcb7fe69c4a1 Author: hiro <hiro(a)torproject.org> Date: Thu Sep 12 13:58:28 2019 +0200 Add upload to archive.org --- scripts/update_files | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/update_files b/scripts/update_files index b1253c1..df800d6 100755 --- a/scripts/update_files +++ b/scripts/update_files @@ -53,4 +53,5 @@ rclone delete gdrive:releases for f in $(ls); do rclone copy $f gdrive:releases + ia upload <identifier> - --remote-name=$f --metadata="title:New Tor Browser release ${f}." done

1 0

[gettor/master] Merge refactoring work from Hiro
by hiro＠torproject.org 27 Sep '19

27 Sep '19

commit 9b76dabab78f67776ad0a34ed8380be83bd80e1d Merge: ece5dfe 98862a5 Author: hiro <hiro(a)torproject.org> Date: Fri Sep 27 11:20:38 2019 +0200 Merge refactoring work from Hiro bin/gettor_check | 24 ++++++ gettor/providers/drive.links | 28 ------- gettor/providers/dropbox.links | 28 ------- gettor/providers/dropbox.links.backup | 25 ------- gettor/providers/github.links | 28 ------- gettor/services/email/sendmail.py | 8 +- scripts/add_links_to_db | 2 +- scripts/check_service | 135 ++++++++++++++++++++++++++++++++++ scripts/create_db | 4 +- scripts/update_files | 23 ++++-- share/locale/en.json | 2 +- 11 files changed, 187 insertions(+), 120 deletions(-)

1 0