March 2020 - tor-commits - lists.torproject.org

[gettor/master] Add more test cases for different email styles
by cohosh＠torproject.org 12 Mar '20

12 Mar '20

commit c8e385babeee9e46b04261e12d777107de38a16d Author: Cecylia Bocovich <cohosh(a)torproject.org> Date: Fri Feb 14 16:15:33 2020 -0500 Add more test cases for different email styles teor mentioned in ticket #23225 that some clients will top-post replies without quoting the the text. This commit adds a test case for that. Since the command detector is greedy, gettor parses the reply first. Also added a test for bottom-posting. --- tests/test_email_service.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/test_email_service.py b/tests/test_email_service.py index 995ba55..71c3d4f 100644 --- a/tests/test_email_service.py +++ b/tests/test_email_service.py @@ -239,6 +239,39 @@ class EmailServiceTests(unittest.TestCase): request = ep.parse("From: \"silvia [hiro]\" <hiro(a)torproject.org>\n" "Subject: Re: [GetTor] Help Email\r\n Reply-To: hiro(a)torproject.org \nTo:" + "gettor(a)torproject.org\nlinux fa\n\n" + "On 2020-02-10 11:54 a.m., gettor(a)torproject.org wrote:\n" + "This is how you can request a tor browser bundle link.\n" + "\n" + "Send an email to: gettor(a)torproject.org\n" + "\n" + "In the body of the email only write: <operating system> <language>.\n" + "\n" + "We only support windows, osx and linux as operating systems.\n" + "\n") + self.assertEqual(request["command"], "links") + self.assertEqual(request["language"], "fa") + self.assertEqual(request["platform"], "linux") + + request = ep.parse("From: \"silvia [hiro]\" <hiro(a)torproject.org>\n" + "Subject: Re: [GetTor] Help Email\r\n Reply-To: hiro(a)torproject.org \nTo:" + "gettor(a)torproject.org\n" + "On 2020-02-10 11:54 a.m., gettor(a)torproject.org wrote:\n" + "> This is how you can request a tor browser bundle link.\n" + ">\n" + "> Send an email to: gettor(a)torproject.org\n" + ">\n" + "> In the body of the email only write: <operating system> <language>.\n" + ">\n" + "> We only support windows, osx and linux as operating systems.\n" + ">\n" + "linux fa\n\n") + self.assertEqual(request["command"], "links") + self.assertEqual(request["language"], "fa") + self.assertEqual(request["platform"], "linux") + + request = ep.parse("From: \"silvia [hiro]\" <hiro(a)torproject.org>\n" + "Subject: Re: [GetTor] Help Email\r\n Reply-To: hiro(a)torproject.org \nTo:" "gettor(a)torproject.org\n" "On 2020-02-10 11:54 a.m., gettor(a)torproject.org wrote:\n" "> This is how you can request a tor browser bundle link.\n"

1 0

[gettor/master] Make locale parsing less eager
by cohosh＠torproject.org 12 Mar '20

12 Mar '20

commit 829fa9fbaabd9c196701085c66621efac9d07b8f Author: Cecylia Bocovich <cohosh(a)torproject.org> Date: Fri Feb 14 16:36:29 2020 -0500 Make locale parsing less eager This will make the locale parsing more restrictive so that, for example, sending "i like fries" will not result in the locale "fr". --- gettor/parse/email.py | 16 +++++++++------- tests/test_email_service.py | 7 ++++++- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/gettor/parse/email.py b/gettor/parse/email.py index 9409a99..5a3e45e 100644 --- a/gettor/parse/email.py +++ b/gettor/parse/email.py @@ -123,16 +123,18 @@ class EmailParser(object): if len(line.strip()) > 0 and line.strip()[0] == ">": continue for word in re.split(r"\s+", line.strip()): + word = word.lower() for locale in self.locales: - if word.lower() == locale.lower(): + if word == locale.lower(): request["language"] = locale - elif (not request["language"]) and (word.lower()[:2] == - locale.lower()[:2]): - request["language"] = locale - if word.lower() in self.platforms: + elif not request["language"]: + parts = re.split(r"[-_]", word) + if parts[0] == locale.lower()[:2]: + request["language"] = locale + if word in self.platforms: request["command"] = "links" - request["platform"] = word.lower() - if (not request["command"]) and word.lower() == "help": + request["platform"] = word + if (not request["command"]) and word == "help": request["command"] = "help" return request diff --git a/tests/test_email_service.py b/tests/test_email_service.py index 71c3d4f..4326b9c 100644 --- a/tests/test_email_service.py +++ b/tests/test_email_service.py @@ -213,7 +213,7 @@ class EmailServiceTests(unittest.TestCase): def test_help_reply(self): #Replying to GetTor Help with a valid links request should get you links ep = conftests.EmailParser(self.settings, "gettor(a)torproject.org") - ep.locales = ["en-US", "es-ES", "es-AR", "pt-BR", "fa"] + ep.locales = ["en-US", "es-ES", "es-AR", "pt-BR", "fa", "fr"] request = ep.parse("From: \"silvia [hiro]\" <hiro(a)torproject.org>\n" "Subject: Re: [GetTor] Help Email\r\n Reply-To: hiro(a)torproject.org \nTo:" "gettor(a)torproject.org\n osx en\n") @@ -223,6 +223,11 @@ class EmailServiceTests(unittest.TestCase): request = ep.parse("From: \"silvia [hiro]\" <hiro(a)torproject.org>\n" "Subject: Re: [GetTor] Help Email\r\n Reply-To: hiro(a)torproject.org \nTo:" + "gettor(a)torproject.org\n i like french fries\n") + self.assertEqual(request["command"], "help") + + request = ep.parse("From: \"silvia [hiro]\" <hiro(a)torproject.org>\n" + "Subject: Re: [GetTor] Help Email\r\n Reply-To: hiro(a)torproject.org \nTo:" "gettor(a)torproject.org\nlinux fa\n\n" "On 2020-02-10 11:54 a.m., gettor(a)torproject.org wrote:\n" "> This is how you can request a tor browser bundle link.\n"

1 0

[torspec/master] bandwidth-file: Add tor_version header
by nickm＠torproject.org 12 Mar '20

12 Mar '20

commit 7d8b4bcffbbeb11a6651b3d042cab20d49da45ef Author: juga0 <juga(a)riseup.net> Date: Sun Feb 9 18:21:35 2020 +0000 bandwidth-file: Add tor_version header Part of #30196. --- bandwidth-file-spec.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/bandwidth-file-spec.txt b/bandwidth-file-spec.txt index 5a2954c..71ed21f 100644 --- a/bandwidth-file-spec.txt +++ b/bandwidth-file-spec.txt @@ -70,6 +70,9 @@ Diagnostic relay lines SHOULD be marked with vote=0, and Tor SHOULD NOT use their bandwidths in its votes. + 1.5.0 - Add system information headers, as operating system, OpenSSL and + Tor versions. + All Tor versions can consume format version 1.0.0. All Tor versions can consume format version 1.1.0 and later, @@ -479,6 +482,14 @@ This Line was added in version 1.4.0 of this specification. + "tor_version=" version_number NL + + [Zero or one time.] + + The Tor version of the Tor process controlled by the generator. + + This Line was added in version 1.5.0 of this specification. + KeyValue NL [Zero or more times.]

1 0

[torspec/master] bandwidth-file: Correct KeyValues names
by nickm＠torproject.org 12 Mar '20

12 Mar '20

commit 4ec93f2d8b2add1da598d927785ed7d8e60f0ec3 Author: juga0 <juga(a)riseup.net> Date: Tue Mar 3 14:58:04 2020 +0000 bandwidth-file: Correct KeyValues names The KeyValues never appeared in a bandwidth file generated by the bwauths with those names, it was a mistake when writing this specification. The replacements done are: - s/desc_bw_average/desc_bw_avg/ - s/desc_obs_bw_last/desc_bw_obs_last/ - s/desc_obs_bw_mean/desc_bw_obs_mean/ And in the examples: - s/desc_avg_bw/desc_bw_avg/ - s/desc_obs_bw_last/desc_bw_obs_last/ - s/desc_obs_bw_mean/desc_bw_obs_mean/ Part of #30726. --- bandwidth-file-spec.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bandwidth-file-spec.txt b/bandwidth-file-spec.txt index 5a2954c..0345c96 100644 --- a/bandwidth-file-spec.txt +++ b/bandwidth-file-spec.txt @@ -771,7 +771,7 @@ This KeyValue was added in version 1.2.0 of this specification. - "desc_bw_average=" Int + "desc_bw_avg=" Int [Zero or one time.] @@ -779,7 +779,7 @@ This KeyValue was added in version 1.2.0 of this specification. - "desc_obs_bw_last=" Int + "desc_bw_obs_last=" Int [Zero or one time.] @@ -788,7 +788,7 @@ This KeyValue was added in version 1.2.0 of this specification. - "desc_obs_bw_mean=" Int + "desc_bw_obs_mean=" Int [Zero or one time.] @@ -972,8 +972,8 @@ percent_eligible_relays=93 software=sbws software_version=1.0.3 ===== -bw=38000 bw_mean=1127824 bw_median=1180062 desc_avg_bw=1073741824 desc_obs_bw_last=17230879 desc_obs_bw_mean=14732306 error_circ=0 error_misc=0 error_stream=1 master_key_ed25519=YaqV4vbvPYKucElk297eVdNArDz9HtIwUoIeo0+cVIpQ nick=Test node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 rtt=380 success=1 time=2018-05-08T16:13:26 -bw=1 bw_mean=199162 bw_median=185675 desc_avg_bw=409600 desc_obs_bw_last=836165 desc_obs_bw_mean=858030 error_circ=0 error_misc=0 error_stream=0 master_key_ed25519=a6a+dZadrQBtfSbmQkP7j2ardCmLnm5NJ4ZzkvDxbo0I nick=Test2 node_id=$96C15995F30895689291F455587BD94CA427B6FC rtt=378 success=1 time=2018-05-08T16:13:36 +bw=38000 bw_mean=1127824 bw_median=1180062 desc_bw_avg=1073741824 desc_bw_obs_last=17230879 desc_bw_obs_mean=14732306 error_circ=0 error_misc=0 error_stream=1 master_key_ed25519=YaqV4vbvPYKucElk297eVdNArDz9HtIwUoIeo0+cVIpQ nick=Test node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 rtt=380 success=1 time=2018-05-08T16:13:26 +bw=1 bw_mean=199162 bw_median=185675 desc_bw_avg=409600 desc_bw_obs_last=836165 desc_bw_obs_mean=858030 error_circ=0 error_misc=0 error_stream=0 master_key_ed25519=a6a+dZadrQBtfSbmQkP7j2ardCmLnm5NJ4ZzkvDxbo0I nick=Test2 node_id=$96C15995F30895689291F455587BD94CA427B6FC rtt=378 success=1 time=2018-05-08T16:13:36 A.3.1. When there are not enough eligible measured relays:

1 0

[torspec/master] bandwidth-file: Remove specification version 1.3.0.
by nickm＠torproject.org 12 Mar '20

12 Mar '20

commit 9cf0d3fbcb0ef73d3a850e4476f50f4eff3da090 Author: juga0 <juga(a)riseup.net> Date: Tue Mar 3 15:34:47 2020 +0000 bandwidth-file: Remove specification version 1.3.0. since there was never a bandwidth file published by bwauths using that nor sbws ever generated that version because it was forgotten to increment it when adding country related KeyValues. --- bandwidth-file-spec.txt | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/bandwidth-file-spec.txt b/bandwidth-file-spec.txt index 6dd90c2..42bfe37 100644 --- a/bandwidth-file-spec.txt +++ b/bandwidth-file-spec.txt @@ -54,13 +54,12 @@ SHOULD contain a header, but no relays. (To match Torflow's existing behaviour.) + Adds scanner and destination countries to the header. Adds new KeyValue Lines to the Header List section with statistics about the number of relays included in the file. Adds new KeyValues to Relay Bandwidth Lines, with different bandwidth values (averages and descriptor bandwidths). - 1.3.0 - Adds scanner and destination countries to the header. - 1.4.0 - Adds monitoring KeyValues to the header and relay lines. RelayLines for excluded relays MAY be present in the bandwidth @@ -327,7 +326,7 @@ The country, as in political geolocation, where the generator is run. - This Line was added in version 1.3.0 of this specification. + This Line was added in version 1.2.0 of this specification. "destinations_countries=" CountryCodeList NL @@ -338,7 +337,7 @@ The destination Web Servers serve the data that the generator retrieves to measure the bandwidth. - This Line was added in version 1.3.0 of this specification. + This Line was added in version 1.2.0 of this specification. "recent_consensus_count=" Int NL @@ -1060,7 +1059,7 @@ software_version=1.0.3 A.4. Headers generated by sbws version 1.0.4 1523911758 -version=1.3.0 +version=1.2.0 latest_bandwidth=2018-04-16T20:49:18 destinations_countries=TH,ZZ file_created=2018-04-16T21:49:18

1 0

[torspec/master] bandwidth-file: Replace `Add` by `Adds`
by nickm＠torproject.org 12 Mar '20

12 Mar '20

commit 1621056cf738d7f7c7082a46795f0ee5068dfa41 Author: juga0 <juga(a)riseup.net> Date: Wed Mar 4 15:33:00 2020 +0000 bandwidth-file: Replace `Add` by `Adds` --- bandwidth-file-spec.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bandwidth-file-spec.txt b/bandwidth-file-spec.txt index 42bfe37..fe1bd89 100644 --- a/bandwidth-file-spec.txt +++ b/bandwidth-file-spec.txt @@ -47,7 +47,7 @@ 1.0.0 - The legacy Bandwidth File format - 1.1.0 - Add a header containing information about the bandwidth + 1.1.0 - Adds a header containing information about the bandwidth file. Document the sbws and Torflow relay line keys. 1.2.0 - If there are not enough eligible relays, the bandwidth file

1 0

[torspec/master] bandwidth-file: Add forgotten KeyValues
by nickm＠torproject.org 12 Mar '20

12 Mar '20

commit 7b12eb7cee099deef2b74dc8cfc3d2505c2013d9 Author: juga0 <juga(a)riseup.net> Date: Tue Mar 3 15:16:45 2020 +0000 bandwidth-file: Add forgotten KeyValues Some bandwidth related KeyValues were added to the version 1.2.0, but they were forgotten when writing this specification. The same happened with some monitoring KeyValues in the version 1.4.0. They have not been added to the examples, since they don't contain all the KeyValues. Closes #30726. --- bandwidth-file-spec.txt | 67 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/bandwidth-file-spec.txt b/bandwidth-file-spec.txt index 0345c96..6dd90c2 100644 --- a/bandwidth-file-spec.txt +++ b/bandwidth-file-spec.txt @@ -56,7 +56,7 @@ Adds new KeyValue Lines to the Header List section with statistics about the number of relays included in the file. - Add new KeyValues to Relay Bandwidth Lines, with different + Adds new KeyValues to Relay Bandwidth Lines, with different bandwidth values (averages and descriptor bandwidths). 1.3.0 - Adds scanner and destination countries to the header. @@ -797,6 +797,71 @@ This KeyValue was added in version 1.2.0 of this specification. + "desc_bw_bur=" Int + + [Zero or one time.] + + The descriptor burst bandwidth for this relay in bytes per + second. + + This KeyValue was added in version 1.2.0 of this specification. + + "consensus_bandwidth" Int + + [Zero or one time.] + + The consensus bandwidth for this relay in bytes per second. + + This KeyValue was added in version 1.2.0 of this specification. + + "consensus_bandwidth_is_unmeasured" Bool + + [Zero or one time.] + + If the consensus bandwidth for this relay was not obtained from + three or more bandwidth authorities, this KeyValue is True or + False otherwise. + + This KeyValue was added in version 1.2.0 of this specification. + + "relay_in_recent_consensus_count" Int + + [Zero or one time.] + + The number of times this relay was found in a consensus in the + last data_period days. (Unless otherwise stated, data_period is + 5 by default.) + + This KeyValue was added in version 1.4.0 of this specification. + + "relay_recent_priority_list_count" Int + + [Zero or one time.] + + The number of times this relay has been prioritized to be measured + in the last data_period days. + + This KeyValue was added in version 1.4.0 of this specification. + + "relay_recent_measurement_attempt_count" Int + + [Zero or one time.] + + The number of times this relay was tried to be measured in the + last data_period days. + + This KeyValue was added in version 1.4.0 of this specification. + + "relay_recent_measurement_failure_count" Int + + [Zero or one time.] + + The number of times this relay was tried to be measured in the + last data_period days, but it was not possible to obtain a + measurement. + + This KeyValue was added in version 1.4.0 of this specification. + "relay_recent_measurements_excluded_error_count=" Int [Zero or one time.]

1 0

[torspec/master] Merge remote-tracking branch 'tor-github/pr/107'
by nickm＠torproject.org 12 Mar '20

12 Mar '20

commit 739c26d0f66c4f47d3232d03492714abc212a427 Merge: 0c42e77 7d8b4bc Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Mar 12 12:47:52 2020 -0400 Merge remote-tracking branch 'tor-github/pr/107' bandwidth-file-spec.txt | 11 +++++++++++ 1 file changed, 11 insertions(+)

1 0

[torspec/master] Merge branch 'master' of git-rw.torproject.org:/torspec
by nickm＠torproject.org 12 Mar '20

12 Mar '20

commit 2692b1c94ca5e2898eef8eefa2fc06545d80fe3d Merge: 2896446 0bfbfad Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Mar 12 12:48:22 2020 -0400 Merge branch 'master' of git-rw.torproject.org:/torspec control-spec.txt | 10 +++++----- proposals/313-relay-ipv6-stats.txt | 11 +---------- rend-spec-v3.txt | 31 ++++++++++++++++++------------- 3 files changed, 24 insertions(+), 28 deletions(-)

1 0

[torspec/master] Merge remote-tracking branch 'tor-github/pr/113'
by nickm＠torproject.org 12 Mar '20

12 Mar '20

commit 2896446cea4fe05790d7805ea6f654673069134c Merge: 739c26d 1621056 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Mar 12 12:48:12 2020 -0400 Merge remote-tracking branch 'tor-github/pr/113' bandwidth-file-spec.txt | 88 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 76 insertions(+), 12 deletions(-)

1 0