August 2020 - tor-commits - lists.torproject.org

[onionperf/master] Fix reprocessing submode.
by karsten＠torproject.org 10 Aug '20

10 Aug '20

commit aea46c56a8bcd3d6f54580868c3b8e72cbcb2d9b Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Jul 14 09:01:08 2020 +0200 Fix reprocessing submode. --- onionperf/reprocessing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onionperf/reprocessing.py b/onionperf/reprocessing.py index ad0308f..8d8dcd4 100644 --- a/onionperf/reprocessing.py +++ b/onionperf/reprocessing.py @@ -1,4 +1,4 @@ -from onionperf.analysis import Analysis +from onionperf.analysis import OPAnalysis from onionperf import util from functools import partial from multiprocessing import Pool, cpu_count @@ -47,7 +47,7 @@ def match(tgen_logs, tor_logs, date_filter): def analyze_func(prefix, nick, do_complete, pair): - analysis = Analysis(nickname=nick) + analysis = OPAnalysis(nickname=nick) logging.info('Analysing pair for date {0}'.format(pair[2])) analysis.add_tgen_file(pair[0]) analysis.add_torctl_file(pair[1])

1 0

[onionperf/master] Fix unit tests.
by karsten＠torproject.org 10 Aug '20

10 Aug '20

commit 3e67cb32257ecbaec632b782603c162aa10a0af6 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Jul 13 16:55:10 2020 +0200 Fix unit tests. --- onionperf/tests/test_analysis.py | 265 ++++++++++++++++----------------------- 1 file changed, 110 insertions(+), 155 deletions(-) diff --git a/onionperf/tests/test_analysis.py b/onionperf/tests/test_analysis.py index 6463458..82e2043 100644 --- a/onionperf/tests/test_analysis.py +++ b/onionperf/tests/test_analysis.py @@ -1,7 +1,8 @@ import os import pkg_resources from nose.tools import * -from onionperf import analysis, util +from onionperf import util +from tgentools import analysis def absolute_data_path(relative_path=""): @@ -12,188 +13,142 @@ def absolute_data_path(relative_path=""): "tests/data/" + relative_path) DATA_DIR = absolute_data_path() -LINE_ERROR = '2019-04-22 14:41:20 1555940480.647663 [message] [shd-tgen-transfer.c:1504] [_tgentransfer_log] [transfer-error] transport TCP,12,localhost:127.0.0.1:46878,localhost:127.0.0.1:43735,dc34og3c3aqdqntblnxkstzfvh7iy7llojd4fi5j23y2po32ock2k7ad.onion:0.0.0.0:8080,state=ERROR,error=READ transfer transfer5m,4,cyan,GET,5242880,(null),0,state=ERROR,error=PROXY total-bytes-read=0 total-bytes-write=0 payload-bytes-read=0/5242880 (0.00%) usecs-to-socket-create=11 usecs-to-socket-connect=210 usecs-to-proxy-init=283 usecs-to-proxy-choice=348 usecs-to-proxy-request=412 usecs-to-proxy-response=-1 usecs-to-command=-1 usecs-to-response=-1 usecs-to-first-byte=-1 usecs-to-last-byte=-1 usecs-to-checksum=-1' - -NO_PARSE_LINE = '2018-04-14 21:10:04 1523740204.809894 [message] [shd-tgen-transfer.c:803] [_tgentransfer_log] [transfer-error] transport TCP,17,NULL:37.218.247.40:26006,NULL:0.0.0.0:0,146.0.73.4:146.0.73.4:1313,state=SUCCESS,error=NONE transfer (null),26847,op-nl,NONE,0,(null),0,state=ERROR,error=AUTH total-bytes-read=1 total-bytes-write=0 payload-bytes-write=0/0 (-nan%) usecs-to-socket-create=0 usecs-to-socket-connect=8053676879205 usecs-to-proxy-init=-1 usecs-to-proxy-choice=-1 usecs-to-proxy-request=-1 usecs-to-proxy-response=-1 usecs-to-command=-1 usecs-to-response=-1 usecs-to-first-byte=-1 usecs-to-last-byte=-1 usecs-to-checksum=-1' - -def test_transfer_status_event(): - transfer = analysis.TransferStatusEvent(LINE_ERROR) - assert_equals(transfer.is_success, False) - assert_equals(transfer.is_error, False) - assert_equals(transfer.is_complete, False) - assert_equals(transfer.unix_ts_end, 1555940480.647663) - assert_equals(transfer.endpoint_local, 'localhost:127.0.0.1:46878') - assert_equals(transfer.endpoint_proxy, 'localhost:127.0.0.1:43735') - assert_equals( - transfer.endpoint_remote, - 'dc34og3c3aqdqntblnxkstzfvh7iy7llojd4fi5j23y2po32ock2k7ad.onion:0.0.0.0:8080' - ) +LINE_ERROR = '2019-04-22 14:41:20 1555940480.647663 [message] [tgen-stream.c:1618] [_tgenstream_log] [stream-error] transport [fd=12,local=localhost:127.0.0.1:46878,proxy=localhost:127.0.0.1:43735,remote=dc34og3c3aqdqntblnxkstzfvh7iy7llojd4fi5j23y2po32ock2k7ad.onion:0.0.0.0:8080,state=ERROR,error=READ] stream [id=4,vertexid=stream5m,name=cyan,peername=(null),sendsize=0,recvsize=5242880,sendstate=SEND_COMMAND,recvstate=RECV_NONE,error=PROXY] bytes [total-bytes-recv=0,total-bytes-send=0,payload-bytes-recv=0,payload-bytes-send=0,payload-progress-recv=0.00%,payload-progress-send=100.00%] times [created-ts=5948325159988,usecs-to-socket-create=11,usecs-to-socket-connect=210,usecs-to-proxy-init=283,usecs-to-proxy-choice=348,usecs-to-proxy-request=412,usecs-to-proxy-response=-1,usecs-to-command=-1,usecs-to-response=-1,usecs-to-first-byte-recv=-1,usecs-to-last-byte-recv=-1,usecs-to-checksum-recv=-1,usecs-to-first-byte-send=-1,usecs-to-last-byte-send=-1,usecs-to-checksum-send=-1,now-ts=594844 6579043]' + +NO_PARSE_LINE = '2018-04-14 21:10:04 1523740204.809894 [message] [tgen-stream.c:1618] [_tgenstream_log] [stream-error] transport [fd=17,local=localhost:127.0.0.1:46878,proxy=localhost:127.0.0.1:43735,remote=dc34og3c3aqdqntblnxkstzfvh7iy7llojd4fi5j23y2po32ock2k7ad.onion:0.0.0.0:8080,state=SUCCESS,error=NONE] stream [id=4,vertexid=stream5m,name=cyan,peername=(null),sendsize=0,recvsize=5242880,sendstate=SEND_COMMAND,recvstate=RECV_NONE,error=PROXY] bytes [total-bytes-recv=1,total-bytes-send=0,payload-bytes-recv=0,payload-bytes-send=0,payload-progress-recv=0.00%,payload-progress-send=100.00%] times [created-ts=5948325159988,usecs-to-socket-create=0,usecs-to-socket-connect=210,usecs-to-proxy-init=-1,usecs-to-proxy-choice=-1,usecs-to-proxy-request=-1,usecs-to-proxy-response=-1,usecs-to-command=-1,usecs-to-response=-1,usecs-to-first-byte-recv=-1,usecs-to-last-byte-recv=-1,usecs-to-checksum-recv=-1,usecs-to-first-byte-send=-1,usecs-to-last-byte-send=-1,usecs-to-checksum-send=-1,now-ts=59484 46579043]' + + +def test_stream_status_event(): + stream = analysis.StreamStatusEvent(LINE_ERROR) + assert_equals(stream.is_success, False) + assert_equals(stream.is_error, False) + assert_equals(stream.is_complete, False) + assert_equals(stream.unix_ts_end, 1555940480.647663) + assert_equals(stream.transport_info['local'], 'localhost:127.0.0.1:46878') + assert_equals(stream.transport_info['proxy'], 'localhost:127.0.0.1:43735') assert_equals( - transfer.endpoint_remote, + stream.transport_info['remote'], 'dc34og3c3aqdqntblnxkstzfvh7iy7llojd4fi5j23y2po32ock2k7ad.onion:0.0.0.0:8080' ) - assert_equals(transfer.transfer_id, 'transfer5m:4') - assert_equals(transfer.hostname_local, 'cyan') - assert_equals(transfer.method, 'GET') - assert_equals(transfer.filesize_bytes, 5242880) - assert_equals(transfer.hostname_remote, '(null)') - assert_equals(transfer.error_code, 'PROXY') - assert_equals(transfer.total_bytes_read, 0) - assert_equals(transfer.total_bytes_write, 0) - assert_equals(transfer.is_commander, True) - assert_equals(transfer.payload_bytes_status, 0) - assert_equals(transfer.unconsumed_parts, [ - 'usecs-to-socket-create=11', 'usecs-to-socket-connect=210', - 'usecs-to-proxy-init=283', 'usecs-to-proxy-choice=348', - 'usecs-to-proxy-request=412', 'usecs-to-proxy-response=-1', - 'usecs-to-command=-1', 'usecs-to-response=-1', - 'usecs-to-first-byte=-1', 'usecs-to-last-byte=-1', - 'usecs-to-checksum=-1' - ]) - assert_equals(transfer.elapsed_seconds, {}) - - -def test_transfer_complete_event_init(): - complete = analysis.TransferCompleteEvent(LINE_ERROR) + assert_equals(stream.stream_id, '4:12:localhost:127.0.0.1:46878:dc34og3c3aqdqntblnxkstzfvh7iy7llojd4fi5j23y2po32ock2k7ad.onion:0.0.0.0:8080') + assert_equals(stream.stream_info['name'], 'cyan') + assert_equals(stream.stream_info['recvsize'], '5242880') + assert_equals(stream.stream_info['peername'], '(null)') + assert_equals(stream.stream_info['error'], 'PROXY') + assert_equals(stream.byte_info['total-bytes-recv'], '0') + assert_equals(stream.byte_info['total-bytes-send'], '0') + assert_equals(stream.byte_info['payload-bytes-recv'], '0') + + +def test_stream_complete_event_init(): + complete = analysis.StreamCompleteEvent(LINE_ERROR) assert_equals(complete.is_complete, True) - assert_equals( - complete.elapsed_seconds, { - 'proxy_init': 0.000283, - 'proxy_request': 0.000412, - 'proxy_choice': 0.000348, - 'socket_connect': 0.00021, - 'socket_create': 1.1e-05 - }) + assert_equals(complete.time_info['usecs-to-proxy-init'], '283') + assert_equals(complete.time_info['usecs-to-proxy-request'], '412') + assert_equals(complete.time_info['usecs-to-proxy-choice'], '348') + assert_equals(complete.time_info['usecs-to-socket-connect'], '210') + assert_equals(complete.time_info['usecs-to-socket-create'], '11') assert_equals(complete.unix_ts_start, 1555940480.6472511) -def test_transfer_error_event(): - error = analysis.TransferErrorEvent(LINE_ERROR) +def test_stream_error_event(): + error = analysis.StreamErrorEvent(LINE_ERROR) assert_equals(error.is_error, True) assert_equals(error.is_success, False) -def test_transfer_success_event_init(): - success = analysis.TransferSuccessEvent(LINE_ERROR) +def test_stream_success_event_init(): + success = analysis.StreamSuccessEvent(LINE_ERROR) assert_equals(success.is_success, True) -def test_transfer_object_init(): - error = analysis.TransferErrorEvent(LINE_ERROR) - t = analysis.Transfer(error.transfer_id) - assert_equals(t.id, 'transfer5m:4') - assert_equals(t.last_event, None) - assert_equals( - t.payload_progress, { - 0.0: None, - 0.1: None, - 0.2: None, - 0.3: None, - 0.4: None, - 0.5: None, - 0.6: None, - 0.7: None, - 0.8: None, - 0.9: None, - 1.0: None - }) +def test_stream_object_init(): + error = analysis.StreamErrorEvent(LINE_ERROR) + s = analysis.Stream(error.stream_id) + assert_equals(s.id, '4:12:localhost:127.0.0.1:46878:dc34og3c3aqdqntblnxkstzfvh7iy7llojd4fi5j23y2po32ock2k7ad.onion:0.0.0.0:8080') + assert_equals(s.last_event, None) -def test_transfer_object_add_event(): - error = analysis.TransferErrorEvent(LINE_ERROR) - t = analysis.Transfer(error.transfer_id) - t.add_event(error) - assert_equals(t.last_event, error) - assert_equals( - t.payload_progress, { - 0.0: 1555940480.647663, - 0.1: None, - 0.2: None, - 0.3: None, - 0.4: None, - 0.5: None, - 0.6: None, - 0.7: None, - 0.8: None, - 0.9: None, - 1.0: None - }) +def test_stream_object_add_event(): + error = analysis.StreamErrorEvent(LINE_ERROR) + s = analysis.Stream(error.stream_id) + s.add_event(error) + assert_equals(s.last_event, error) @raises(KeyError) -def test_transfer_object_get_data_error(): - error = analysis.TransferErrorEvent(LINE_ERROR) - t = analysis.Transfer(error.transfer_id) - t.add_event(error) - t.get_data()['elapsed_seconds']['payload_progress'] +def test_stream_object_get_data_error(): + error = analysis.StreamErrorEvent(LINE_ERROR) + s = analysis.Stream(error.stream_id) + s.add_event(error) + s.get_data()['elapsed_seconds']['payload_progress_recv'] -def test_transfer_object_get_data_no_error(): - success = analysis.TransferSuccessEvent(LINE_ERROR) - t = analysis.Transfer(success.transfer_id) - t.add_event(success) +def test_stream_object_get_data_no_error(): + success = analysis.StreamSuccessEvent(LINE_ERROR) + s = analysis.Stream(success.stream_id) + s.add_event(success) assert_true( - t.get_data()['elapsed_seconds']['payload_progress'] is not None) + s.get_data()['elapsed_seconds']['payload_progress_recv'] is not None) -def test_transfer_object_end_to_end(): - error = analysis.TransferErrorEvent(LINE_ERROR) - t = analysis.Transfer(error.transfer_id) - t.add_event(error) +def test_stream_object_end_to_end(): + error = analysis.StreamErrorEvent(LINE_ERROR) + s = analysis.Stream(error.stream_id) + s.add_event(error) assert_equals( - t.get_data(), { - 'is_error': - True, - 'endpoint_local': - 'localhost:127.0.0.1:46878', - 'total_bytes_read': - 0, - 'error_code': - 'PROXY', - 'unix_ts_end': - 1555940480.647663, - 'hostname_local': - 'cyan', - 'endpoint_remote': - 'dc34og3c3aqdqntblnxkstzfvh7iy7llojd4fi5j23y2po32ock2k7ad.onion:0.0.0.0:8080', - 'elapsed_seconds': { - 'proxy_init': 0.000283, - 'proxy_request': 0.000412, - 'proxy_choice': 0.000348, - 'socket_connect': 0.00021, - 'socket_create': 1.1e-05 + s.get_data(), { + 'is_success': False, + 'is_error': True, + 'is_complete': True, + 'unix_ts_end': 1555940480.647663, + 'transport_info': {'fd': '12', + 'local': 'localhost:127.0.0.1:46878', + 'proxy': 'localhost:127.0.0.1:43735', + 'remote': 'dc34og3c3aqdqntblnxkstzfvh7iy7llojd4fi5j23y2po32ock2k7ad.onion:0.0.0.0:8080', + 'state': 'ERROR', + 'error': 'READ'}, + 'stream_info': { + 'id': '4', + 'vertexid': 'stream5m', + 'name': 'cyan', + 'peername': '(null)', + 'sendsize': '0', + 'recvsize': '5242880', + 'sendstate': 'SEND_COMMAND', + 'recvstate': 'RECV_NONE', + 'error': 'PROXY' }, - 'method': - 'GET', - 'is_commander': - True, - 'total_bytes_write': - 0, - 'unix_ts_start': - 1555940480.6472511, - 'hostname_remote': - '(null)', - 'transfer_id': - 'transfer5m:4', - 'is_success': - False, - 'payload_bytes_status': - 0, - 'endpoint_proxy': - 'localhost:127.0.0.1:43735', - 'is_complete': - True, - 'filesize_bytes': - 5242880 + 'byte_info': { + 'total-bytes-recv': '0', + 'total-bytes-send': '0', + 'payload-bytes-recv': '0', + 'payload-bytes-send': '0', + 'payload-progress-recv': '0.00%', + 'payload-progress-send': '100.00%' + }, + 'time_info': { + 'created-ts': '5948325159988', + 'usecs-to-socket-create': '11', + 'usecs-to-socket-connect': '210', + 'usecs-to-proxy-init': '283', + 'usecs-to-proxy-choice': '348', + 'usecs-to-proxy-request': '412', + 'usecs-to-proxy-response': '-1', + 'usecs-to-command': '-1', + 'usecs-to-response': '-1', + 'usecs-to-first-byte-recv': '-1', + 'usecs-to-last-byte-recv': '-1', + 'usecs-to-checksum-recv': '-1', + 'usecs-to-first-byte-send': '-1', + 'usecs-to-last-byte-send': '-1', + 'usecs-to-checksum-send': '-1', + 'now-ts': '5948446579043' + }, + 'stream_id': '4:12:localhost:127.0.0.1:46878:dc34og3c3aqdqntblnxkstzfvh7iy7llojd4fi5j23y2po32ock2k7ad.onion:0.0.0.0:8080', + 'unix_ts_start': 1555940480.6472511 }) - -@raises(ZeroDivisionError) -def test_transfer_status_parse_error(): - transfer = analysis.TransferStatusEvent(NO_PARSE_LINE) - t = analysis.Transfer(transfer.transfer_id) - t.add_event(transfer) - def test_parsing_parse_error(): parser = analysis.TGenParser() parser.parse(util.DataSource(DATA_DIR + 'parse_error'))

1 0

[onionperf/master] Merge branch 'tgen-1.0.0' into develop
by karsten＠torproject.org 10 Aug '20

10 Aug '20

commit a11531b53fb4443aa3c49eed0bf78ea9d577a500 Merge: adece5d aea46c5 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Jul 14 09:03:24 2020 +0200 Merge branch 'tgen-1.0.0' into develop README.md | 7 +- onionperf/analysis.py | 332 +++++---------------------------------- onionperf/model.py | 10 +- onionperf/onionperf | 16 +- onionperf/reprocessing.py | 12 +- onionperf/tests/test_analysis.py | 265 +++++++++++++------------------ onionperf/visualization.py | 90 +++++++---- requirements.txt | 1 + 8 files changed, 231 insertions(+), 502 deletions(-)

1 0

[onionperf/master] Remove now unused imports.
by karsten＠torproject.org 10 Aug '20

10 Aug '20

commit 0189c3a73339c7b2554aadb1c60b1dc92d534572 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sun Jul 12 22:57:11 2020 +0200 Remove now unused imports. --- onionperf/analysis.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/onionperf/analysis.py b/onionperf/analysis.py index 8506756..19c0192 100644 --- a/onionperf/analysis.py +++ b/onionperf/analysis.py @@ -4,11 +4,8 @@ See LICENSE for licensing information ''' -import sys, os, re, json, datetime, logging +import os, re, json, datetime, logging -from multiprocessing import Pool, cpu_count -from signal import signal, SIGINT, SIG_IGN -from socket import gethostname from abc import ABCMeta, abstractmethod # stem imports

1 0

[onionperf/master] Update TGen traffic model to use TGen version 1.0.0
by karsten＠torproject.org 10 Aug '20

10 Aug '20

commit 25bae0dc902e3b486c357e29ea0368cb85fad262 Author: Ana Custura <ana(a)netstat.org.uk> Date: Fri Jun 26 10:40:24 2020 +0100 Update TGen traffic model to use TGen version 1.0.0 --- onionperf/model.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/onionperf/model.py b/onionperf/model.py index 1f4d9fe..f06f8e0 100644 --- a/onionperf/model.py +++ b/onionperf/model.py @@ -74,7 +74,7 @@ class TorperfModel(GeneratableTGenModel): else: g.add_node("start", serverport=self.tgen_port, peers=server_str, loglevel="info", heartbeat="1 minute") g.add_node("pause", time="5 minutes") - g.add_node("transfer5m", type="get", protocol="tcp", size="5 MiB", timeout="270 seconds", stallout="0 seconds") + g.add_node("stream5m", sendsize="0", recvsize="5 mib", timeout="270 seconds", stallout="0 seconds") g.add_edge("start", "pause") @@ -83,7 +83,7 @@ class TorperfModel(GeneratableTGenModel): g.add_edge("pause", "pause") # these are chosen with weighted probability, change edge 'weight' attributes to adjust probability - g.add_edge("pause", "transfer5m") + g.add_edge("pause", "stream5m") return g @@ -103,10 +103,10 @@ class OneshotModel(GeneratableTGenModel): g.add_node("start", serverport=self.tgen_port, peers=server_str, loglevel="info", heartbeat="1 minute", socksproxy=self.socksproxy) else: g.add_node("start", serverport=self.tgen_port, peers=server_str, loglevel="info", heartbeat="1 minute") - g.add_node("transfer5m", type="get", protocol="tcp", size="5 MiB", timeout="15 seconds", stallout="10 seconds") + g.add_node("stream5m", sendsize="0", recvsize="5 mib", timeout="270 seconds", stallout="0 seconds") - g.add_edge("start", "transfer5m") - g.add_edge("transfer5m", "start") + g.add_edge("start", "stream5m") + g.add_edge("stream5m", "start") return g

1 0

[onionperf/master] Remove TGen 0.0.1 reference from README.md.
by karsten＠torproject.org 10 Aug '20

10 Aug '20

commit 272cd2558ad352936260522de8d49a417e40b501 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Jul 13 16:53:27 2020 +0200 Remove TGen 0.0.1 reference from README.md. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e3df3bf..ecd7c0c 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ In this case the resulting `tor` binary can be found in `~/tor/src/app/tor` and ### TGen -OnionPerf uses TGen to generate traffic on client and server side for its measurements. Installing dependencies, cloning TGen to a subdirectory in the user's home directory, checking out version 0.0.1, and building TGen is done as follows: +OnionPerf uses TGen to generate traffic on client and server side for its measurements. Installing dependencies, cloning TGen to a subdirectory in the user's home directory, and building TGen is done as follows: ```shell sudo apt install cmake libglib2.0-dev libigraph0-dev make

1 0

[onionperf/master] Pass date_filter argument to parent analysis
by karsten＠torproject.org 10 Aug '20

10 Aug '20

commit e32deb2994576cd1bfb83477da5d36b7991ef4d1 Author: Ana Custura <ana(a)netstat.org.uk> Date: Wed Jul 22 15:10:06 2020 +0100 Pass date_filter argument to parent analysis --- onionperf/analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onionperf/analysis.py b/onionperf/analysis.py index e269f6a..898b165 100644 --- a/onionperf/analysis.py +++ b/onionperf/analysis.py @@ -34,7 +34,7 @@ class OPAnalysis(Analysis): return self.date_filter = date_filter - super().analyze(do_complete=True) + super().analyze(do_complete=True, date_filter=self.date_filter) torctl_parser = TorCtlParser(date_filter=self.date_filter) for (filepaths, parser, json_db_key) in [(self.torctl_filepaths, torctl_parser, 'tor')]:

1 0

[onionperf/master] Merge branch 'task-40005' into develop
by karsten＠torproject.org 10 Aug '20

10 Aug '20

commit a2ef62c2b7d1406f72ac87769ce0396f543146d8 Merge: c761ac6 c1cd60e Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Jul 16 09:55:36 2020 +0200 Merge branch 'task-40005' into develop onionperf/analysis.py | 76 ++++++++++++------------------------ onionperf/measurement.py | 2 +- onionperf/onionperf | 9 +---- onionperf/reprocessing.py | 8 ++-- onionperf/tests/test_reprocessing.py | 2 +- 5 files changed, 33 insertions(+), 64 deletions(-)

1 0

[onionperf/master] Take out Tor summaries and the do_complete switch.
by karsten＠torproject.org 10 Aug '20

10 Aug '20

commit 95a2f8b482ee882b4ec898a7bbc76018a6d98659 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Jul 14 10:14:22 2020 +0200 Take out Tor summaries and the do_complete switch. Implements tpo/metrics/onionperf#40005. --- onionperf/analysis.py | 70 ++++++++++++------------------------ onionperf/measurement.py | 2 +- onionperf/onionperf | 9 ++--- onionperf/reprocessing.py | 8 ++--- onionperf/tests/test_reprocessing.py | 2 +- 5 files changed, 30 insertions(+), 61 deletions(-) diff --git a/onionperf/analysis.py b/onionperf/analysis.py index 19c0192..26b00b2 100644 --- a/onionperf/analysis.py +++ b/onionperf/analysis.py @@ -29,13 +29,7 @@ class OPAnalysis(Analysis): def add_torctl_file(self, filepath): self.torctl_filepaths.append(filepath) - def get_tor_bandwidth_summary(self, node, direction): - try: - return self.json_db['data'][node]['tor']['bandwidth_summary'][direction] - except: - return None - - def analyze(self, do_complete=False, date_filter=None): + def analyze(self, date_filter=None): if self.did_analysis: return @@ -47,7 +41,7 @@ class OPAnalysis(Analysis): if len(filepaths) > 0: for filepath in filepaths: logging.info("parsing log file at {0}".format(filepath)) - parser.parse(util.DataSource(filepath), do_complete=do_complete) + parser.parse(util.DataSource(filepath), do_complete=True) if self.nickname is None: parsed_name = parser.get_name() @@ -134,7 +128,7 @@ class OPAnalysis(Analysis): class Parser(object, metaclass=ABCMeta): @abstractmethod - def parse(self, source, do_complete): + def parse(self, source, do_complete=True): pass @abstractmethod def get_data(self): @@ -270,14 +264,10 @@ class TorCtlParser(Parser): def __init__(self, date_filter=None): ''' date_filter should be given in UTC ''' - self.do_complete = False - self.bandwidth_summary = {'bytes_read':{}, 'bytes_written':{}} self.circuits_state = {} self.circuits = {} - self.circuits_summary = {'buildtimes':[], 'lifetimes':[]} self.streams_state = {} self.streams = {} - self.streams_summary = {'lifetimes':{}} self.name = None self.boot_succeeded = False self.build_timeout_last = None @@ -320,15 +310,10 @@ class TorCtlParser(Parser): data = circ.get_data() if data is not None: - if built is not None and started is not None and len(data['path']) == 3: - self.circuits_summary['buildtimes'].append(built - started) - if ended is not None and started is not None: - self.circuits_summary['lifetimes'].append(ended - started) - if self.do_complete: - self.circuits[cid] = data + self.circuits[cid] = data self.circuits_state.pop(cid) - elif self.do_complete and isinstance(event, CircMinorEvent): + elif isinstance(event, CircMinorEvent): if event.purpose != event.old_purpose or event.event != CircEvent.PURPOSE_CHANGED: key = "{0}:{1}".format(event.event, event.purpose) circ.add_event(key, arrival_dt) @@ -364,15 +349,9 @@ class TorCtlParser(Parser): data = strm.get_data() if data is not None: - if self.do_complete: - self.streams[sid] = data - self.streams_summary['lifetimes'].setdefault(stream_type, []).append(ended - started) + self.streams[sid] = data self.streams_state.pop(sid) - def __handle_bw(self, event, arrival_dt): - self.bandwidth_summary['bytes_read'][int(arrival_dt)] = event.read - self.bandwidth_summary['bytes_written'][int(arrival_dt)] = event.written - def __handle_buildtimeout(self, event, arrival_dt): self.build_timeout_last = event.timeout self.build_quantile_last = event.quantile @@ -382,8 +361,6 @@ class TorCtlParser(Parser): self.__handle_circuit(event, arrival_dt) elif isinstance(event, StreamEvent): self.__handle_stream(event, arrival_dt) - elif isinstance(event, BandwidthEvent): - self.__handle_bw(event, arrival_dt) elif isinstance(event, BuildTimeoutSetEvent): self.__handle_buildtimeout(event, arrival_dt) @@ -408,27 +385,26 @@ class TorCtlParser(Parser): elif re.search("BOOTSTRAP", line) is not None and re.search("PROGRESS=100", line) is not None: self.boot_succeeded = True - if self.do_complete or (self.do_complete is False and re.search("650\sBW", line) is not None): - # parse with stem - timestamps, sep, raw_event_str = line.partition(" 650 ") - if sep == '': - return True + # parse with stem + timestamps, sep, raw_event_str = line.partition(" 650 ") + if sep == '': + return True + + # event.arrived_at is also available but at worse granularity + unix_ts = float(timestamps.strip().split()[2]) - # event.arrived_at is also available but at worse granularity - unix_ts = float(timestamps.strip().split()[2]) + # check if we should ignore the line + line_date = datetime.datetime.utcfromtimestamp(unix_ts).date() + if not self.__is_date_valid(line_date): + return True - # check if we should ignore the line - line_date = datetime.datetime.utcfromtimestamp(unix_ts).date() - if not self.__is_date_valid(line_date): - return True + event = ControlMessage.from_str("{0} {1}".format(sep.strip(), raw_event_str)) + convert('EVENT', event) + self.__handle_event(event, unix_ts) - event = ControlMessage.from_str("{0} {1}".format(sep.strip(), raw_event_str)) - convert('EVENT', event) - self.__handle_event(event, unix_ts) return True - def parse(self, source, do_complete=False): - self.do_complete = do_complete + def parse(self, source, do_complete=True): source.open(newline='\r\n') for line in source: # ignore line parsing errors @@ -442,9 +418,7 @@ class TorCtlParser(Parser): source.close() def get_data(self): - return {'circuits': self.circuits, 'circuits_summary': self.circuits_summary, - 'streams':self.streams, 'streams_summary': self.streams_summary, - 'bandwidth_summary': self.bandwidth_summary} + return {'circuits': self.circuits, 'streams': self.streams} def get_name(self): return self.name diff --git a/onionperf/measurement.py b/onionperf/measurement.py index 9d84d4e..823f37a 100644 --- a/onionperf/measurement.py +++ b/onionperf/measurement.py @@ -157,7 +157,7 @@ def logrotate_thread_task(writables, tgen_writable, torctl_writable, docroot, ni anal.add_torctl_file(torctl_writable.rotate_file(filename_datetime=next_midnight)) # run the analysis, i.e. parse the files - anal.analyze(do_simple=False) + anal.analyze() # save the results in onionperf json format in the www docroot anal.save(output_prefix=docroot, do_compress=True, date_prefix=next_midnight.date()) diff --git a/onionperf/onionperf b/onionperf/onionperf index 5e2c2fb..36520ad 100755 --- a/onionperf/onionperf +++ b/onionperf/onionperf @@ -279,11 +279,6 @@ files generated by this script will be written""", action="store", dest="date_prefix", default=None) - analyze_parser.add_argument('-s', '--do-simple-parse', - help="""parse and export only summary statistics rather than full transfer/circuit/stream data""", - action="store_false", dest="do_complete", - default=True) - # visualize visualize_parser = sub_parser.add_parser('visualize', description=DESC_VISUALIZE, help=HELP_VISUALIZE, formatter_class=my_formatter_class) @@ -387,7 +382,7 @@ def analyze(args): analysis.add_tgen_file(args.tgen_logpath) if args.torctl_logpath is not None: analysis.add_torctl_file(args.torctl_logpath) - analysis.analyze(args.do_complete, date_filter=args.date_filter) + analysis.analyze(date_filter=args.date_filter) analysis.save(output_prefix=args.prefix, date_prefix=args.date_prefix) elif args.tgen_logpath is not None and os.path.isdir(args.tgen_logpath) and args.torctl_logpath is not None and os.path.isdir(args.torctl_logpath): @@ -396,7 +391,7 @@ def analyze(args): torctl_logs = reprocessing.collect_logs(args.torctl_logpath, '*torctl.log*') log_pairs = reprocessing.match(tgen_logs, torctl_logs, args.date_filter) logging.info("Found {0} matching log pairs to be reprocessed".format(len(log_pairs))) - reprocessing.multiprocess_logs(log_pairs, args.prefix, args.nickname, args.do_complete) + reprocessing.multiprocess_logs(log_pairs, args.prefix, args.nickname) else: logging.error("Given paths were an unrecognized mix of file and directory paths, nothing will be analyzed") diff --git a/onionperf/reprocessing.py b/onionperf/reprocessing.py index 8d8dcd4..d949dca 100644 --- a/onionperf/reprocessing.py +++ b/onionperf/reprocessing.py @@ -46,21 +46,21 @@ def match(tgen_logs, tor_logs, date_filter): return log_pairs -def analyze_func(prefix, nick, do_complete, pair): +def analyze_func(prefix, nick, pair): analysis = OPAnalysis(nickname=nick) logging.info('Analysing pair for date {0}'.format(pair[2])) analysis.add_tgen_file(pair[0]) analysis.add_torctl_file(pair[1]) - analysis.analyze(do_complete=do_complete, date_filter=pair[2]) + analysis.analyze(date_filter=pair[2]) analysis.save(output_prefix=prefix) return 1 -def multiprocess_logs(log_pairs, prefix, nick=None, do_complete=False): +def multiprocess_logs(log_pairs, prefix, nick=None): pool = Pool(cpu_count()) analyses = None try: - func = partial(analyze_func, prefix, nick, do_complete) + func = partial(analyze_func, prefix, nick) mr = pool.map_async(func, log_pairs) pool.close() while not mr.ready(): diff --git a/onionperf/tests/test_reprocessing.py b/onionperf/tests/test_reprocessing.py index 5e758d2..b51d154 100644 --- a/onionperf/tests/test_reprocessing.py +++ b/onionperf/tests/test_reprocessing.py @@ -61,7 +61,7 @@ def test_log_match_with_wrong_filter_date(): def test_analyze_func_json(): pair = (DATA_DIR + 'logs/onionperf_2019-01-10_23:59:59.tgen.log', DATA_DIR + 'logs/onionperf_2019-01-10_23:59:59.torctl.log', datetime.datetime(2019, 1, 10, 0, 0)) work_dir = tempfile.mkdtemp() - reprocessing.analyze_func(work_dir, None, False, pair) + reprocessing.analyze_func(work_dir, None, pair) json_file = os.path.join(work_dir, "2019-01-10.onionperf.analysis.json.xz") assert(os.path.exists(json_file)) shutil.rmtree(work_dir)

1 0

[onionperf/master] Merge branch 'task-40003' into develop
by karsten＠torproject.org 10 Aug '20

10 Aug '20

commit dff6ed7269e97e03e4d285eda2d8d329c779686a Merge: a2ef62c 7c1ae34 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Jul 16 22:10:42 2020 +0200 Merge branch 'task-40003' into develop schema/onionperf-3.0.json | 546 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 546 insertions(+)

1 0