July 2020 - tor-commits - lists.torproject.org

[onionperf/develop] Remove now unused imports.
by karsten＠torproject.org 14 Jul '20

14 Jul '20

commit 0189c3a73339c7b2554aadb1c60b1dc92d534572 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sun Jul 12 22:57:11 2020 +0200 Remove now unused imports. --- onionperf/analysis.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/onionperf/analysis.py b/onionperf/analysis.py index 8506756..19c0192 100644 --- a/onionperf/analysis.py +++ b/onionperf/analysis.py @@ -4,11 +4,8 @@ See LICENSE for licensing information ''' -import sys, os, re, json, datetime, logging +import os, re, json, datetime, logging -from multiprocessing import Pool, cpu_count -from signal import signal, SIGINT, SIG_IGN -from socket import gethostname from abc import ABCMeta, abstractmethod # stem imports

1 0

[onionperf/develop] Fix visualizations.
by karsten＠torproject.org 14 Jul '20

14 Jul '20

commit b9dbc158666940bc1bb2c7ffccee2b73873c7368 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sun Jul 12 23:18:29 2020 +0200 Fix visualizations. Turns out that stream_info/recvsize contains a string, not an int. --- onionperf/visualization.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/onionperf/visualization.py b/onionperf/visualization.py index 52d08fc..80c0781 100644 --- a/onionperf/visualization.py +++ b/onionperf/visualization.py @@ -70,12 +70,12 @@ class TGenVisualization(Visualization): stream["time_to_last_byte"] = float(s["usecs-to-last-byte-recv"])/1000000 if "elapsed_seconds" in stream_data: s = stream_data["elapsed_seconds"] - # Explanation of the math below for computing Mbps: From filesize_bytes - # and payload_progress fields we can compute the number of seconds that + # Explanation of the math below for computing Mbps: From stream_info/recvsize + # and payload_progress_recv fields we can compute the number of seconds that # have elapsed between receiving bytes 524,288 and 1,048,576, which is a # total amount of 524,288 bytes or 4,194,304 bits or 4.194304 megabits. # We want the reciprocal of that value with unit megabits per second. - if stream_data["stream_info"]["recvsize"] == 5242880 and "0.2" in s["payload_progress_recv"]: + if stream_data["stream_info"]["recvsize"] == "5242880" and "0.2" in s["payload_progress_recv"]: stream["mbps"] = 4.194304 / (s["payload_progress_recv"]["0.2"] - s["payload_progress_recv"]["0.1"]) if "error" in stream_data["transport_info"] and stream_data["transport_info"]["error"] != "NONE": stream["error_code"] = stream_data["transport_info"]["error"]

1 0

[onionperf/develop] Fix reprocessing submode.
by karsten＠torproject.org 14 Jul '20

14 Jul '20

commit aea46c56a8bcd3d6f54580868c3b8e72cbcb2d9b Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Jul 14 09:01:08 2020 +0200 Fix reprocessing submode. --- onionperf/reprocessing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onionperf/reprocessing.py b/onionperf/reprocessing.py index ad0308f..8d8dcd4 100644 --- a/onionperf/reprocessing.py +++ b/onionperf/reprocessing.py @@ -1,4 +1,4 @@ -from onionperf.analysis import Analysis +from onionperf.analysis import OPAnalysis from onionperf import util from functools import partial from multiprocessing import Pool, cpu_count @@ -47,7 +47,7 @@ def match(tgen_logs, tor_logs, date_filter): def analyze_func(prefix, nick, do_complete, pair): - analysis = Analysis(nickname=nick) + analysis = OPAnalysis(nickname=nick) logging.info('Analysing pair for date {0}'.format(pair[2])) analysis.add_tgen_file(pair[0]) analysis.add_torctl_file(pair[1])

1 0

[onionperf/develop] Fix unit tests.
by karsten＠torproject.org 14 Jul '20

14 Jul '20

commit 3e67cb32257ecbaec632b782603c162aa10a0af6 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Jul 13 16:55:10 2020 +0200 Fix unit tests. --- onionperf/tests/test_analysis.py | 265 ++++++++++++++++----------------------- 1 file changed, 110 insertions(+), 155 deletions(-) diff --git a/onionperf/tests/test_analysis.py b/onionperf/tests/test_analysis.py index 6463458..82e2043 100644 --- a/onionperf/tests/test_analysis.py +++ b/onionperf/tests/test_analysis.py @@ -1,7 +1,8 @@ import os import pkg_resources from nose.tools import * -from onionperf import analysis, util +from onionperf import util +from tgentools import analysis def absolute_data_path(relative_path=""): @@ -12,188 +13,142 @@ def absolute_data_path(relative_path=""): "tests/data/" + relative_path) DATA_DIR = absolute_data_path() -LINE_ERROR = '2019-04-22 14:41:20 1555940480.647663 [message] [shd-tgen-transfer.c:1504] [_tgentransfer_log] [transfer-error] transport TCP,12,localhost:127.0.0.1:46878,localhost:127.0.0.1:43735,dc34og3c3aqdqntblnxkstzfvh7iy7llojd4fi5j23y2po32ock2k7ad.onion:0.0.0.0:8080,state=ERROR,error=READ transfer transfer5m,4,cyan,GET,5242880,(null),0,state=ERROR,error=PROXY total-bytes-read=0 total-bytes-write=0 payload-bytes-read=0/5242880 (0.00%) usecs-to-socket-create=11 usecs-to-socket-connect=210 usecs-to-proxy-init=283 usecs-to-proxy-choice=348 usecs-to-proxy-request=412 usecs-to-proxy-response=-1 usecs-to-command=-1 usecs-to-response=-1 usecs-to-first-byte=-1 usecs-to-last-byte=-1 usecs-to-checksum=-1' - -NO_PARSE_LINE = '2018-04-14 21:10:04 1523740204.809894 [message] [shd-tgen-transfer.c:803] [_tgentransfer_log] [transfer-error] transport TCP,17,NULL:37.218.247.40:26006,NULL:0.0.0.0:0,146.0.73.4:146.0.73.4:1313,state=SUCCESS,error=NONE transfer (null),26847,op-nl,NONE,0,(null),0,state=ERROR,error=AUTH total-bytes-read=1 total-bytes-write=0 payload-bytes-write=0/0 (-nan%) usecs-to-socket-create=0 usecs-to-socket-connect=8053676879205 usecs-to-proxy-init=-1 usecs-to-proxy-choice=-1 usecs-to-proxy-request=-1 usecs-to-proxy-response=-1 usecs-to-command=-1 usecs-to-response=-1 usecs-to-first-byte=-1 usecs-to-last-byte=-1 usecs-to-checksum=-1' - -def test_transfer_status_event(): - transfer = analysis.TransferStatusEvent(LINE_ERROR) - assert_equals(transfer.is_success, False) - assert_equals(transfer.is_error, False) - assert_equals(transfer.is_complete, False) - assert_equals(transfer.unix_ts_end, 1555940480.647663) - assert_equals(transfer.endpoint_local, 'localhost:127.0.0.1:46878') - assert_equals(transfer.endpoint_proxy, 'localhost:127.0.0.1:43735') - assert_equals( - transfer.endpoint_remote, - 'dc34og3c3aqdqntblnxkstzfvh7iy7llojd4fi5j23y2po32ock2k7ad.onion:0.0.0.0:8080' - ) +LINE_ERROR = '2019-04-22 14:41:20 1555940480.647663 [message] [tgen-stream.c:1618] [_tgenstream_log] [stream-error] transport [fd=12,local=localhost:127.0.0.1:46878,proxy=localhost:127.0.0.1:43735,remote=dc34og3c3aqdqntblnxkstzfvh7iy7llojd4fi5j23y2po32ock2k7ad.onion:0.0.0.0:8080,state=ERROR,error=READ] stream [id=4,vertexid=stream5m,name=cyan,peername=(null),sendsize=0,recvsize=5242880,sendstate=SEND_COMMAND,recvstate=RECV_NONE,error=PROXY] bytes [total-bytes-recv=0,total-bytes-send=0,payload-bytes-recv=0,payload-bytes-send=0,payload-progress-recv=0.00%,payload-progress-send=100.00%] times [created-ts=5948325159988,usecs-to-socket-create=11,usecs-to-socket-connect=210,usecs-to-proxy-init=283,usecs-to-proxy-choice=348,usecs-to-proxy-request=412,usecs-to-proxy-response=-1,usecs-to-command=-1,usecs-to-response=-1,usecs-to-first-byte-recv=-1,usecs-to-last-byte-recv=-1,usecs-to-checksum-recv=-1,usecs-to-first-byte-send=-1,usecs-to-last-byte-send=-1,usecs-to-checksum-send=-1,now-ts=594844 6579043]' + +NO_PARSE_LINE = '2018-04-14 21:10:04 1523740204.809894 [message] [tgen-stream.c:1618] [_tgenstream_log] [stream-error] transport [fd=17,local=localhost:127.0.0.1:46878,proxy=localhost:127.0.0.1:43735,remote=dc34og3c3aqdqntblnxkstzfvh7iy7llojd4fi5j23y2po32ock2k7ad.onion:0.0.0.0:8080,state=SUCCESS,error=NONE] stream [id=4,vertexid=stream5m,name=cyan,peername=(null),sendsize=0,recvsize=5242880,sendstate=SEND_COMMAND,recvstate=RECV_NONE,error=PROXY] bytes [total-bytes-recv=1,total-bytes-send=0,payload-bytes-recv=0,payload-bytes-send=0,payload-progress-recv=0.00%,payload-progress-send=100.00%] times [created-ts=5948325159988,usecs-to-socket-create=0,usecs-to-socket-connect=210,usecs-to-proxy-init=-1,usecs-to-proxy-choice=-1,usecs-to-proxy-request=-1,usecs-to-proxy-response=-1,usecs-to-command=-1,usecs-to-response=-1,usecs-to-first-byte-recv=-1,usecs-to-last-byte-recv=-1,usecs-to-checksum-recv=-1,usecs-to-first-byte-send=-1,usecs-to-last-byte-send=-1,usecs-to-checksum-send=-1,now-ts=59484 46579043]' + + +def test_stream_status_event(): + stream = analysis.StreamStatusEvent(LINE_ERROR) + assert_equals(stream.is_success, False) + assert_equals(stream.is_error, False) + assert_equals(stream.is_complete, False) + assert_equals(stream.unix_ts_end, 1555940480.647663) + assert_equals(stream.transport_info['local'], 'localhost:127.0.0.1:46878') + assert_equals(stream.transport_info['proxy'], 'localhost:127.0.0.1:43735') assert_equals( - transfer.endpoint_remote, + stream.transport_info['remote'], 'dc34og3c3aqdqntblnxkstzfvh7iy7llojd4fi5j23y2po32ock2k7ad.onion:0.0.0.0:8080' ) - assert_equals(transfer.transfer_id, 'transfer5m:4') - assert_equals(transfer.hostname_local, 'cyan') - assert_equals(transfer.method, 'GET') - assert_equals(transfer.filesize_bytes, 5242880) - assert_equals(transfer.hostname_remote, '(null)') - assert_equals(transfer.error_code, 'PROXY') - assert_equals(transfer.total_bytes_read, 0) - assert_equals(transfer.total_bytes_write, 0) - assert_equals(transfer.is_commander, True) - assert_equals(transfer.payload_bytes_status, 0) - assert_equals(transfer.unconsumed_parts, [ - 'usecs-to-socket-create=11', 'usecs-to-socket-connect=210', - 'usecs-to-proxy-init=283', 'usecs-to-proxy-choice=348', - 'usecs-to-proxy-request=412', 'usecs-to-proxy-response=-1', - 'usecs-to-command=-1', 'usecs-to-response=-1', - 'usecs-to-first-byte=-1', 'usecs-to-last-byte=-1', - 'usecs-to-checksum=-1' - ]) - assert_equals(transfer.elapsed_seconds, {}) - - -def test_transfer_complete_event_init(): - complete = analysis.TransferCompleteEvent(LINE_ERROR) + assert_equals(stream.stream_id, '4:12:localhost:127.0.0.1:46878:dc34og3c3aqdqntblnxkstzfvh7iy7llojd4fi5j23y2po32ock2k7ad.onion:0.0.0.0:8080') + assert_equals(stream.stream_info['name'], 'cyan') + assert_equals(stream.stream_info['recvsize'], '5242880') + assert_equals(stream.stream_info['peername'], '(null)') + assert_equals(stream.stream_info['error'], 'PROXY') + assert_equals(stream.byte_info['total-bytes-recv'], '0') + assert_equals(stream.byte_info['total-bytes-send'], '0') + assert_equals(stream.byte_info['payload-bytes-recv'], '0') + + +def test_stream_complete_event_init(): + complete = analysis.StreamCompleteEvent(LINE_ERROR) assert_equals(complete.is_complete, True) - assert_equals( - complete.elapsed_seconds, { - 'proxy_init': 0.000283, - 'proxy_request': 0.000412, - 'proxy_choice': 0.000348, - 'socket_connect': 0.00021, - 'socket_create': 1.1e-05 - }) + assert_equals(complete.time_info['usecs-to-proxy-init'], '283') + assert_equals(complete.time_info['usecs-to-proxy-request'], '412') + assert_equals(complete.time_info['usecs-to-proxy-choice'], '348') + assert_equals(complete.time_info['usecs-to-socket-connect'], '210') + assert_equals(complete.time_info['usecs-to-socket-create'], '11') assert_equals(complete.unix_ts_start, 1555940480.6472511) -def test_transfer_error_event(): - error = analysis.TransferErrorEvent(LINE_ERROR) +def test_stream_error_event(): + error = analysis.StreamErrorEvent(LINE_ERROR) assert_equals(error.is_error, True) assert_equals(error.is_success, False) -def test_transfer_success_event_init(): - success = analysis.TransferSuccessEvent(LINE_ERROR) +def test_stream_success_event_init(): + success = analysis.StreamSuccessEvent(LINE_ERROR) assert_equals(success.is_success, True) -def test_transfer_object_init(): - error = analysis.TransferErrorEvent(LINE_ERROR) - t = analysis.Transfer(error.transfer_id) - assert_equals(t.id, 'transfer5m:4') - assert_equals(t.last_event, None) - assert_equals( - t.payload_progress, { - 0.0: None, - 0.1: None, - 0.2: None, - 0.3: None, - 0.4: None, - 0.5: None, - 0.6: None, - 0.7: None, - 0.8: None, - 0.9: None, - 1.0: None - }) +def test_stream_object_init(): + error = analysis.StreamErrorEvent(LINE_ERROR) + s = analysis.Stream(error.stream_id) + assert_equals(s.id, '4:12:localhost:127.0.0.1:46878:dc34og3c3aqdqntblnxkstzfvh7iy7llojd4fi5j23y2po32ock2k7ad.onion:0.0.0.0:8080') + assert_equals(s.last_event, None) -def test_transfer_object_add_event(): - error = analysis.TransferErrorEvent(LINE_ERROR) - t = analysis.Transfer(error.transfer_id) - t.add_event(error) - assert_equals(t.last_event, error) - assert_equals( - t.payload_progress, { - 0.0: 1555940480.647663, - 0.1: None, - 0.2: None, - 0.3: None, - 0.4: None, - 0.5: None, - 0.6: None, - 0.7: None, - 0.8: None, - 0.9: None, - 1.0: None - }) +def test_stream_object_add_event(): + error = analysis.StreamErrorEvent(LINE_ERROR) + s = analysis.Stream(error.stream_id) + s.add_event(error) + assert_equals(s.last_event, error) @raises(KeyError) -def test_transfer_object_get_data_error(): - error = analysis.TransferErrorEvent(LINE_ERROR) - t = analysis.Transfer(error.transfer_id) - t.add_event(error) - t.get_data()['elapsed_seconds']['payload_progress'] +def test_stream_object_get_data_error(): + error = analysis.StreamErrorEvent(LINE_ERROR) + s = analysis.Stream(error.stream_id) + s.add_event(error) + s.get_data()['elapsed_seconds']['payload_progress_recv'] -def test_transfer_object_get_data_no_error(): - success = analysis.TransferSuccessEvent(LINE_ERROR) - t = analysis.Transfer(success.transfer_id) - t.add_event(success) +def test_stream_object_get_data_no_error(): + success = analysis.StreamSuccessEvent(LINE_ERROR) + s = analysis.Stream(success.stream_id) + s.add_event(success) assert_true( - t.get_data()['elapsed_seconds']['payload_progress'] is not None) + s.get_data()['elapsed_seconds']['payload_progress_recv'] is not None) -def test_transfer_object_end_to_end(): - error = analysis.TransferErrorEvent(LINE_ERROR) - t = analysis.Transfer(error.transfer_id) - t.add_event(error) +def test_stream_object_end_to_end(): + error = analysis.StreamErrorEvent(LINE_ERROR) + s = analysis.Stream(error.stream_id) + s.add_event(error) assert_equals( - t.get_data(), { - 'is_error': - True, - 'endpoint_local': - 'localhost:127.0.0.1:46878', - 'total_bytes_read': - 0, - 'error_code': - 'PROXY', - 'unix_ts_end': - 1555940480.647663, - 'hostname_local': - 'cyan', - 'endpoint_remote': - 'dc34og3c3aqdqntblnxkstzfvh7iy7llojd4fi5j23y2po32ock2k7ad.onion:0.0.0.0:8080', - 'elapsed_seconds': { - 'proxy_init': 0.000283, - 'proxy_request': 0.000412, - 'proxy_choice': 0.000348, - 'socket_connect': 0.00021, - 'socket_create': 1.1e-05 + s.get_data(), { + 'is_success': False, + 'is_error': True, + 'is_complete': True, + 'unix_ts_end': 1555940480.647663, + 'transport_info': {'fd': '12', + 'local': 'localhost:127.0.0.1:46878', + 'proxy': 'localhost:127.0.0.1:43735', + 'remote': 'dc34og3c3aqdqntblnxkstzfvh7iy7llojd4fi5j23y2po32ock2k7ad.onion:0.0.0.0:8080', + 'state': 'ERROR', + 'error': 'READ'}, + 'stream_info': { + 'id': '4', + 'vertexid': 'stream5m', + 'name': 'cyan', + 'peername': '(null)', + 'sendsize': '0', + 'recvsize': '5242880', + 'sendstate': 'SEND_COMMAND', + 'recvstate': 'RECV_NONE', + 'error': 'PROXY' }, - 'method': - 'GET', - 'is_commander': - True, - 'total_bytes_write': - 0, - 'unix_ts_start': - 1555940480.6472511, - 'hostname_remote': - '(null)', - 'transfer_id': - 'transfer5m:4', - 'is_success': - False, - 'payload_bytes_status': - 0, - 'endpoint_proxy': - 'localhost:127.0.0.1:43735', - 'is_complete': - True, - 'filesize_bytes': - 5242880 + 'byte_info': { + 'total-bytes-recv': '0', + 'total-bytes-send': '0', + 'payload-bytes-recv': '0', + 'payload-bytes-send': '0', + 'payload-progress-recv': '0.00%', + 'payload-progress-send': '100.00%' + }, + 'time_info': { + 'created-ts': '5948325159988', + 'usecs-to-socket-create': '11', + 'usecs-to-socket-connect': '210', + 'usecs-to-proxy-init': '283', + 'usecs-to-proxy-choice': '348', + 'usecs-to-proxy-request': '412', + 'usecs-to-proxy-response': '-1', + 'usecs-to-command': '-1', + 'usecs-to-response': '-1', + 'usecs-to-first-byte-recv': '-1', + 'usecs-to-last-byte-recv': '-1', + 'usecs-to-checksum-recv': '-1', + 'usecs-to-first-byte-send': '-1', + 'usecs-to-last-byte-send': '-1', + 'usecs-to-checksum-send': '-1', + 'now-ts': '5948446579043' + }, + 'stream_id': '4:12:localhost:127.0.0.1:46878:dc34og3c3aqdqntblnxkstzfvh7iy7llojd4fi5j23y2po32ock2k7ad.onion:0.0.0.0:8080', + 'unix_ts_start': 1555940480.6472511 }) - -@raises(ZeroDivisionError) -def test_transfer_status_parse_error(): - transfer = analysis.TransferStatusEvent(NO_PARSE_LINE) - t = analysis.Transfer(transfer.transfer_id) - t.add_event(transfer) - def test_parsing_parse_error(): parser = analysis.TGenParser() parser.parse(util.DataSource(DATA_DIR + 'parse_error'))

1 0

[onionperf/develop] Remove TGen 0.0.1 reference from README.md.
by karsten＠torproject.org 14 Jul '20

14 Jul '20

commit 272cd2558ad352936260522de8d49a417e40b501 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Jul 13 16:53:27 2020 +0200 Remove TGen 0.0.1 reference from README.md. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e3df3bf..ecd7c0c 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ In this case the resulting `tor` binary can be found in `~/tor/src/app/tor` and ### TGen -OnionPerf uses TGen to generate traffic on client and server side for its measurements. Installing dependencies, cloning TGen to a subdirectory in the user's home directory, checking out version 0.0.1, and building TGen is done as follows: +OnionPerf uses TGen to generate traffic on client and server side for its measurements. Installing dependencies, cloning TGen to a subdirectory in the user's home directory, and building TGen is done as follows: ```shell sudo apt install cmake libglib2.0-dev libigraph0-dev make

1 0

[onionperf/develop] Merge branch 'tgen-1.0.0' into develop
by karsten＠torproject.org 14 Jul '20

14 Jul '20

commit a11531b53fb4443aa3c49eed0bf78ea9d577a500 Merge: adece5d aea46c5 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Jul 14 09:03:24 2020 +0200 Merge branch 'tgen-1.0.0' into develop README.md | 7 +- onionperf/analysis.py | 332 +++++---------------------------------- onionperf/model.py | 10 +- onionperf/onionperf | 16 +- onionperf/reprocessing.py | 12 +- onionperf/tests/test_analysis.py | 265 +++++++++++++------------------ onionperf/visualization.py | 90 +++++++---- requirements.txt | 1 + 8 files changed, 231 insertions(+), 502 deletions(-)

1 0

[bridgedb/develop] Only process text/plain part of emails.
by phw＠torproject.org 14 Jul '20

14 Jul '20

commit 61e63879157343571f89ec02e0e21dd19e270e64 Author: Philipp Winter <phw(a)nymity.ch> Date: Mon Jun 1 15:12:34 2020 -0700 Only process text/plain part of emails. When our autoresponder receives a multipart email from a user, it may get confused by the parts that are not text/plain. Instead of bending over backwards to parse all sorts email encodings, this patch discard the parts of a multipart email that aren't text/plain. This fixes tpo/anti-censorship/bridgedb#33835. --- CHANGELOG | 4 +++ bridgedb/distributors/email/server.py | 18 +++++++++- bridgedb/test/test_email_server.py | 63 +++++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index 860cbda..ad20b91 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,7 @@ + * FIXES https://gitlab.torproject.org/tpo/anti-censorship/bridgedb/-/issues/33835 + If a user sends a multipart email to our autoresponder, we now only parse + the part that has a text/plain encoding. + * FIXES https://bugs.torproject.org/33647 Fix broken link in the README and fix our Makefile's pylint target. diff --git a/bridgedb/distributors/email/server.py b/bridgedb/distributors/email/server.py index 54a8231..b2baf86 100644 --- a/bridgedb/distributors/email/server.py +++ b/bridgedb/distributors/email/server.py @@ -50,6 +50,7 @@ Servers which interface with clients and distribute bridges over SMTP. from __future__ import unicode_literals import email.message +import email.policy import logging import io import socket @@ -228,6 +229,20 @@ class SMTPMessage(object): if not self.ignoring: self.message = self.getIncomingMessage() self.responder.reply() + + # If a user sends a multipart email, we only consider the part whose + # content type is text/plain. + if self.message.is_multipart(): + has_plaintext = False + for part in self.message.get_payload(): + if part.get_content_type() != "text/plain": + continue + self.lines = part.get_payload().split("\n") + has_plaintext = True + + if not has_plaintext: + logging.warning("User email had no text/plain content type.") + return defer.succeed(None) def connectionLost(self): @@ -242,7 +257,8 @@ class SMTPMessage(object): :returns: A ``Message`` comprised of all lines received thus far. """ - return email.message_from_string('\n'.join(self.lines)) + return email.message_from_string('\n'.join(self.lines), + policy=email.policy.compat32) @implementer(smtp.IMessageDelivery) diff --git a/bridgedb/test/test_email_server.py b/bridgedb/test/test_email_server.py index fb82cc0..e0aee18 100644 --- a/bridgedb/test/test_email_server.py +++ b/bridgedb/test/test_email_server.py @@ -100,6 +100,69 @@ class SMTPMessageTests(unittest.TestCase): self.assertIsInstance(self.message.getIncomingMessage(), email.message.Message) + def test_SMTPMessage_multipart1(self): + """`eomReceived` should get rid of HTML multiparts.""" + + # Gmail's web interface would send a message like this when replying to + # one of BridgeDB's emails. + multipartEmail = [ + 'MIME-Version: 1.0', + 'Date: Mon, 1 Jun 2020 15:55:33 -0700', + 'Subject: Re: test', + 'From: Foo Bar <foo(a)bar.com>', + 'To: bridges(a)bridges.torproject.org', + 'Content-Type: multipart/alternative; boundary="00000000000041b34105a70db186"', + '', + '--00000000000041b34105a70db186', + 'Content-Type: text/plain; charset="UTF-8"', + '', + 'This is plaintext.', + '', + '--00000000000041b34105a70db186', + 'Content-Type: text/html; charset="UTF-8"', + 'Content-Transfer-Encoding: quoted-printable', + '', + 'This is HTML.', + '', + '--00000000000041b34105a70db186--'] + + for line in multipartEmail: + self.message.lineReceived(line) + self.message.eomReceived() + content = "".join(self.message.lines) + self.assertTrue("This is plaintext." in content) + self.assertFalse("This is HTML." in content) + + def test_SMTPMessage_multipart2(self): + """`eomReceived` should get rid of HTML multiparts.""" + + # Outlook would send a message like this when replying to one of + # BridgeDB's emails. + multipartEmail = [ + 'Content-Type: multipart/alternative;', + ' boundary="_000_VI1PR08MB4351A21D6C4A31C2B0FDA34F8CC10VI1PR08MB4351eurp_"', + '', + '--_000_VI1PR08MB4351A21D6C4A31C2B0FDA34F8CC10VI1PR08MB4351eurp_', + 'Content-Type: text/plain; charset="us-ascii"', + 'Content-Transfer-Encoding: quoted-printable', + '', + 'This is plaintext.', + '', + '--_000_VI1PR08MB4351A21D6C4A31C2B0FDA34F8CC10VI1PR08MB4351eurp_', + 'Content-Type: text/html; charset="us-ascii"', + 'Content-Transfer-Encoding: quoted-printable', + '', + 'This is HTML.', + '', + '--_000_VI1PR08MB4351A21D6C4A31C2B0FDA34F8CC10VI1PR08MB4351eurp_--'] + + for line in multipartEmail: + self.message.lineReceived(line) + self.message.eomReceived() + content = "".join(self.message.lines) + self.assertTrue("This is plaintext." in content) + self.assertFalse("This is HTML." in content) + class SMTPIncomingDeliveryTests(unittest.TestCase): """Unittests for :class:`email.server.SMTPIncomingDelivery`."""

1 0

[bridgedb/develop] Merge branch 'defect/33835' into 'develop'
by phw＠torproject.org 14 Jul '20

14 Jul '20

commit bd8b3796d9622101630687690ce672ec147ed21f Merge: 95ee763 61e6387 Author: Philipp Winter <phw(a)torproject.org> Date: Tue Jul 14 02:22:18 2020 +0000 Merge branch 'defect/33835' into 'develop' Only process text/plain part of emails. See merge request tpo/anti-censorship/bridgedb!10 CHANGELOG | 4 +++ bridgedb/distributors/email/server.py | 18 +++++++++- bridgedb/test/test_email_server.py | 63 +++++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 1 deletion(-)

1 0

[bridgedb/develop] Merge branch 'issue-33647' into 'develop'
by phw＠torproject.org 14 Jul '20

14 Jul '20

commit 95ee763cd5cb7761dd639b388541661c93e690d0 Merge: fb3cfa7 04fd047 Author: Philipp Winter <phw(a)torproject.org> Date: Tue Jul 14 01:55:53 2020 +0000 Merge branch 'issue-33647' into 'develop' Issue 33647 See merge request tpo/anti-censorship/bridgedb!5 .pylintrc | 2 +- CHANGELOG | 3 +++ README.rst | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-)

1 0

[bridgedb/develop] Make init hook compatible with Python 3.
by phw＠torproject.org 14 Jul '20

14 Jul '20

commit 04fd047378022592306b76b5a68ddf8fd2df39a0 Author: Philipp Winter <phw(a)nymity.ch> Date: Tue Jun 16 10:00:34 2020 -0700 Make init hook compatible with Python 3. Running `make pylint` now works again. This fixes issue #33647. --- .pylintrc | 2 +- CHANGELOG | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.pylintrc b/.pylintrc index 8bf60a0..5532cd7 100644 --- a/.pylintrc +++ b/.pylintrc @@ -6,7 +6,7 @@ # Python code to execute, usually for sys.path manipulation such as # pygtk.require(). #init-hook='import sys,os; venv = os.environ.get("VIRTUAL_ENV"); v = os.path.join(venv, "lib", "python" + str(sys.version_info.major) + "." + str(sys.version_info.micro), "site-packages") if venv is not None else None; sys.path.insert(0, v);' -init-hook="import sys,os;a=os.getcwd();print a;sys.path.insert(0, a));" +init-hook="import sys,os;a=os.getcwd();print(a);sys.path.insert(0, a);" # Profiled execution. profile=no diff --git a/CHANGELOG b/CHANGELOG index 1e6e730..860cbda 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,6 @@ + * FIXES https://bugs.torproject.org/33647 + Fix broken link in the README and fix our Makefile's pylint target. + Changes in version 0.11.0 - 2020-07-08 * FIXES https://bugs.torproject.org/31422

1 0