July 2020 - tor-commits - lists.torproject.org

[snowflake/master] Refactor android script to be in android job
by cohosh＠torproject.org 14 Jul '20

14 Jul '20

commit c1fa4efe4b6e289758224a5a4c8bcaa3d7067449 Author: Cecylia Bocovich <cohosh(a)torproject.org> Date: Wed Jul 8 10:49:41 2020 -0400 Refactor android script to be in android job --- .gitlab-ci.yml | 79 +++++++++++++++++++++++++++++++--------------------------- 1 file changed, 42 insertions(+), 37 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 18902e6..04a58fc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,6 +41,15 @@ git lbzip2 +.go_test: &go-test + - test -z "$(go fmt ./...)" + - go vet ./... + - go test -v -race ./... + + - cd $CI_PROJECT_DIR/client/ + - go get + - go build + .test-template: &test-template artifacts: name: "${CI_PROJECT_PATH}_${CI_JOB_STAGE}_${CI_COMMIT_REF_NAME}_${CI_COMMIT_SHA}" @@ -52,17 +61,35 @@ when: on_success after_script: - echo "Download debug artifacts from https://gitlab.com/${CI_PROJECT_PATH}/-/jobs" - script: - - test -z "$(go fmt ./...)" - - go vet ./... - - go test -v -race ./... - - cd $CI_PROJECT_DIR/client/ - - go get - - go build +# -- jobs ------------------------------------------------------------ - # build for Android if this is the right job - - test "$CI_JOB_NAME" = "android" || exit 0 +android: + image: registry.gitlab.com/fdroid/ci-images-client + variables: + GOPATH: "/go" + cache: + paths: + - .gradle/wrapper + - .gradle/caches + before_script: + - apt-get -qy update + - apt-get -qy install --no-install-recommends + build-essential + gnupg + wget + - cd /usr/local + - export gotarball="go1.13.12.linux-amd64.tar.gz" + - wget -q https://dl.google.com/go/${gotarball} + - wget -q https://dl.google.com/go/${gotarball}.asc + - curl https://dl.google.com/linux/linux_signing_key.pub | gpg --import + - gpg --verify ${gotarball}.asc + - echo "9cacc6653563771b458c13056265aa0c21b8a23ca9408278484e4efde4160618 ${gotarball}" | sha256sum -c + - tar -xzf ${gotarball} + - export PATH="/usr/local/go/bin:$GOPATH/bin:$PATH" # putting this in 'variables:' cause weird runner errors + - cd $CI_PROJECT_DIR + script: + - *go-test - export GRADLE_USER_HOME=$PWD/.gradle # This build was setup before go.mod was a thing, go back to the old days! # 920f6791f3ec8e7467c43ee0cefffe63200bed2b broke the gomobile build. @@ -93,47 +120,25 @@ # gomobile builds a shared library not a CLI executable - sed -i 's,^package main$,package snowflakeclient,' snowflake.go client_test.go - gomobile bind -v -target=android git.torproject.org/pluggable-transports/snowflake/client - - -# -- jobs ------------------------------------------------------------ - -android: - image: registry.gitlab.com/fdroid/ci-images-client - variables: - GOPATH: "/go" - cache: - paths: - - .gradle/wrapper - - .gradle/caches - before_script: - - apt-get -qy update - - apt-get -qy install --no-install-recommends - build-essential - gnupg - wget - - cd /usr/local - - export gotarball="go1.13.12.linux-amd64.tar.gz" - - wget -q https://dl.google.com/go/${gotarball} - - wget -q https://dl.google.com/go/${gotarball}.asc - - curl https://dl.google.com/linux/linux_signing_key.pub | gpg --import - - gpg --verify ${gotarball}.asc - - echo "9cacc6653563771b458c13056265aa0c21b8a23ca9408278484e4efde4160618 ${gotarball}" | sha256sum -c - - tar -xzf ${gotarball} - - export PATH="/usr/local/go/bin:$GOPATH/bin:$PATH" # putting this in 'variables:' cause weird runner errors - - cd $CI_PROJECT_DIR <<: *test-template go-1.13: image: golang:1.13-stretch <<: *golang-docker-debian-template <<: *test-template + script: + - *go-test go-1.14: image: golang:1.14-stretch <<: *golang-docker-debian-template <<: *test-template + script: + - *go-test debian-testing: image: debian:testing <<: *debian-native-template <<: *test-template + script: + - *go-test

1 0

[snowflake/master] update .gitlab-ci.yml
by cohosh＠torproject.org 14 Jul '20

14 Jul '20

commit d44fc238150c64a7a5045e756d1e9e2dbe0a3e5a Author: Hans-Christoph Steiner <hans(a)eds.org> Date: Tue Jun 30 21:47:48 2020 +0200 update .gitlab-ci.yml --- .gitlab-ci.yml | 160 ++++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 135 insertions(+), 25 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b7fd956..18902e6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,29 +1,139 @@ -image: golang:1.10-stretch - -cache: - paths: - - .gradle/wrapper - - .gradle/caches - -before_script: - # Create symbolic links under $GOPATH, this is needed for local build - - export src=$GOPATH/src - - mkdir -p $src/git.torproject.org/pluggable-transports - - mkdir -p $src/gitlab.com/$CI_PROJECT_NAMESPACE - - ln -s $CI_PROJECT_DIR $src/git.torproject.org/pluggable-transports/snowflake.git - - ln -s $CI_PROJECT_DIR $src/gitlab.com/$CI_PROJECT_PATH - -build: - script: + +# Set things up to use the OS-native packages for Go. Anything that +# is downloaded by go during the `go fmt` stage is not coming from the +# Debian/Ubuntu repo. So those would need to be packaged for this to +# make it into Debian and/or Ubuntu. +.debian-native-template: &debian-native-template + variables: + DEBIAN_FRONTEND: noninteractive + GOPATH: /usr/share/gocode + before_script: - apt-get -qy update - - apt-get -qy install libx11-dev - - cd $src/gitlab.com/$CI_PROJECT_PATH/client - - go get ./... - - go build ./... + - apt-get -qy install --no-install-recommends + build-essential + ca-certificates + git + golang + golang-github-cheekybits-genny-dev + golang-github-jtolds-gls-dev + golang-github-klauspost-reedsolomon-dev + golang-github-lucas-clemente-quic-go-dev + golang-github-smartystreets-assertions-dev + golang-github-smartystreets-goconvey-dev + golang-github-tjfoc-gmsm-dev + golang-github-xtaci-kcp-dev + golang-github-xtaci-smux-dev + golang-golang-x-crypto-dev + golang-golang-x-net-dev + golang-golang-x-sys-dev + golang-golang-x-text-dev + golang-golang-x-xerrors-dev + lbzip2 + +# use Go installed as part of the official, Debian-based Docker images +.golang-docker-debian-template: &golang-docker-debian-template + variables: + DEBIAN_FRONTEND: noninteractive + before_script: + - apt-get -qy update + - apt-get -qy install --no-install-recommends + ca-certificates + git + lbzip2 + +.test-template: &test-template + artifacts: + name: "${CI_PROJECT_PATH}_${CI_JOB_STAGE}_${CI_COMMIT_REF_NAME}_${CI_COMMIT_SHA}" + paths: + - client/*.aar + - client/*.jar + - client/client + expire_in: 1 day + when: on_success + after_script: + - echo "Download debug artifacts from https://gitlab.com/${CI_PROJECT_PATH}/-/jobs" + script: + - test -z "$(go fmt ./...)" - go vet ./... - go test -v -race ./... -after_script: - # this file changes every time but should not be cached - - rm -f $GRADLE_USER_HOME/caches/modules-2/modules-2.lock - - rm -fr $GRADLE_USER_HOME/caches/*/plugin-resolution/ + - cd $CI_PROJECT_DIR/client/ + - go get + - go build + + # build for Android if this is the right job + - test "$CI_JOB_NAME" = "android" || exit 0 + - export GRADLE_USER_HOME=$PWD/.gradle + # This build was setup before go.mod was a thing, go back to the old days! + # 920f6791f3ec8e7467c43ee0cefffe63200bed2b broke the gomobile build. + # https://dev.to/maelvls/why-is-go111module-everywhere-and-everything-about-g… + - export GO111MODULE=off + - go version + - go env + + - go get golang.org/x/mobile/cmd/gomobile + - go get golang.org/x/mobile/cmd/gobind + - go install golang.org/x/mobile/cmd/gomobile + - go install golang.org/x/mobile/cmd/gobind + - echo y | $ANDROID_HOME/tools/bin/sdkmanager 'ndk-bundle' > /dev/null + - gomobile init + + # Create symbolic links under $GOPATH, this is needed for local build + - export src=$GOPATH/src + - mkdir -p $src/git.torproject.org/pluggable-transports + - mkdir -p $src/github.com/keroserene + - mkdir -p $src/gitlab.com/$CI_PROJECT_NAMESPACE + - ln -s $CI_PROJECT_DIR $src/git.torproject.org/pluggable-transports/snowflake + - ln -s $CI_PROJECT_DIR $src/github.com/keroserene/snowflake + - ln -s $CI_PROJECT_DIR $src/gitlab.com/$CI_PROJECT_PATH + + - git -C $CI_PROJECT_DIR reset --hard + - git -C $CI_PROJECT_DIR clean -fdx + - cd $CI_PROJECT_DIR/client + # gomobile builds a shared library not a CLI executable + - sed -i 's,^package main$,package snowflakeclient,' snowflake.go client_test.go + - gomobile bind -v -target=android git.torproject.org/pluggable-transports/snowflake/client + + +# -- jobs ------------------------------------------------------------ + +android: + image: registry.gitlab.com/fdroid/ci-images-client + variables: + GOPATH: "/go" + cache: + paths: + - .gradle/wrapper + - .gradle/caches + before_script: + - apt-get -qy update + - apt-get -qy install --no-install-recommends + build-essential + gnupg + wget + - cd /usr/local + - export gotarball="go1.13.12.linux-amd64.tar.gz" + - wget -q https://dl.google.com/go/${gotarball} + - wget -q https://dl.google.com/go/${gotarball}.asc + - curl https://dl.google.com/linux/linux_signing_key.pub | gpg --import + - gpg --verify ${gotarball}.asc + - echo "9cacc6653563771b458c13056265aa0c21b8a23ca9408278484e4efde4160618 ${gotarball}" | sha256sum -c + - tar -xzf ${gotarball} + - export PATH="/usr/local/go/bin:$GOPATH/bin:$PATH" # putting this in 'variables:' cause weird runner errors + - cd $CI_PROJECT_DIR + <<: *test-template + +go-1.13: + image: golang:1.13-stretch + <<: *golang-docker-debian-template + <<: *test-template + +go-1.14: + image: golang:1.14-stretch + <<: *golang-docker-debian-template + <<: *test-template + +debian-testing: + image: debian:testing + <<: *debian-native-template + <<: *test-template

1 0

[snowflake/master] Use go modules to build android library
by cohosh＠torproject.org 14 Jul '20

14 Jul '20

commit eaac9f5b6be90d2ba1d63f630c727090a4a4701e Author: Cecylia Bocovich <cohosh(a)torproject.org> Date: Wed Jul 8 16:13:05 2020 -0400 Use go modules to build android library This commit removes the symlinks and turns go modules back on to run gomobile bind locally on the project. --- .gitlab-ci.yml | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 04a58fc..2ac0aa0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -91,10 +91,6 @@ android: script: - *go-test - export GRADLE_USER_HOME=$PWD/.gradle - # This build was setup before go.mod was a thing, go back to the old days! - # 920f6791f3ec8e7467c43ee0cefffe63200bed2b broke the gomobile build. - # https://dev.to/maelvls/why-is-go111module-everywhere-and-everything-about-g… - - export GO111MODULE=off - go version - go env @@ -105,21 +101,12 @@ android: - echo y | $ANDROID_HOME/tools/bin/sdkmanager 'ndk-bundle' > /dev/null - gomobile init - # Create symbolic links under $GOPATH, this is needed for local build - - export src=$GOPATH/src - - mkdir -p $src/git.torproject.org/pluggable-transports - - mkdir -p $src/github.com/keroserene - - mkdir -p $src/gitlab.com/$CI_PROJECT_NAMESPACE - - ln -s $CI_PROJECT_DIR $src/git.torproject.org/pluggable-transports/snowflake - - ln -s $CI_PROJECT_DIR $src/github.com/keroserene/snowflake - - ln -s $CI_PROJECT_DIR $src/gitlab.com/$CI_PROJECT_PATH - - git -C $CI_PROJECT_DIR reset --hard - git -C $CI_PROJECT_DIR clean -fdx - cd $CI_PROJECT_DIR/client # gomobile builds a shared library not a CLI executable - sed -i 's,^package main$,package snowflakeclient,' snowflake.go client_test.go - - gomobile bind -v -target=android git.torproject.org/pluggable-transports/snowflake/client + - gomobile bind -v -target=android . <<: *test-template go-1.13:

1 0

[tor/master] conn: Fix doxygen comment in connstats.c
by dgoulet＠torproject.org 14 Jul '20

14 Jul '20

commit 852cf9b2e272a7d338510f1edf17f7cd5b0d6938 Author: David Goulet <dgoulet(a)torproject.org> Date: Tue Jul 14 09:09:14 2020 -0400 conn: Fix doxygen comment in connstats.c Issue found by our CI: https://travis-ci.org/github/torproject/tor/jobs/707731598#L6059 Signed-off-by: David Goulet <dgoulet(a)torproject.org> --- src/feature/stats/connstats.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/feature/stats/connstats.c b/src/feature/stats/connstats.c index f0c786efb3..827a332be1 100644 --- a/src/feature/stats/connstats.c +++ b/src/feature/stats/connstats.c @@ -139,7 +139,7 @@ conn_stats_terminate(void) } /** - * Record a single entry @a ent in the counts structure @cnt. + * Record a single entry @a ent in the counts structure @a cnt. */ static void add_entry_to_count(conn_counts_t *cnt, const bidi_map_entry_t *ent)

1 0

[translation/tpo-web] https://gitweb.torproject.org/translation.git/commit/?h=tpo-web
by translation＠torproject.org 14 Jul '20

14 Jul '20

commit 6ac85c5142f424db996bec24a2fc4263ffb1648f Author: Translation commit bot <translation(a)torproject.org> Date: Tue Jul 14 09:48:00 2020 +0000 https://gitweb.torproject.org/translation.git/commit/?h=tpo-web --- contents+sw.po | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contents+sw.po b/contents+sw.po index 82b08ab89f..42892057d7 100644 --- a/contents+sw.po +++ b/contents+sw.po @@ -917,7 +917,7 @@ msgstr "Lugha" #: templates/download-languages.html:12 templates/download-options.html:12 msgid "Windows" -msgstr "" +msgstr "Madirisha" #: templates/download-languages.html:13 msgid "macOS"

1 0

[onionperf/develop] Fix a bug introduced by the TGen 1.0.0 update.
by karsten＠torproject.org 14 Jul '20

14 Jul '20

commit c761ac65aa71452a4ddd85137e4313c639e5fc63 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Jul 14 10:31:14 2020 +0200 Fix a bug introduced by the TGen 1.0.0 update. The bug was that we were using the wrong Analysis class in measure mode: AttributeError: 'Analysis' object has no attribute 'add_torctl_file' --- onionperf/measurement.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onionperf/measurement.py b/onionperf/measurement.py index 1540bac..9d84d4e 100644 --- a/onionperf/measurement.py +++ b/onionperf/measurement.py @@ -150,7 +150,7 @@ def logrotate_thread_task(writables, tgen_writable, torctl_writable, docroot, ni public_measurement_ip_guess = util.get_ip_address() # set up the analysis object with our log files - anal = analysis.Analysis(nickname=nickname, ip_address=public_measurement_ip_guess) + anal = analysis.OPAnalysis(nickname=nickname, ip_address=public_measurement_ip_guess) if tgen_writable is not None: anal.add_tgen_file(tgen_writable.rotate_file(filename_datetime=next_midnight)) if torctl_writable is not None:

1 0

[onionperf/develop] Add TGenTools requirement, update code to use new OPAnalysis class
by karsten＠torproject.org 14 Jul '20

14 Jul '20

commit ae18e7a7cbf1330f096ecd569eb45e9cfc039dda Author: Ana Custura <ana(a)netstat.org.uk> Date: Fri Jun 26 11:08:28 2020 +0100 Add TGenTools requirement, update code to use new OPAnalysis class --- onionperf/onionperf | 8 ++++---- requirements.txt | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/onionperf/onionperf b/onionperf/onionperf index ddbeaf1..5e2c2fb 100755 --- a/onionperf/onionperf +++ b/onionperf/onionperf @@ -381,8 +381,8 @@ def analyze(args): logging.warning("No logfile paths were given, nothing will be analyzed") elif (args.tgen_logpath is None or os.path.isfile(args.tgen_logpath)) and (args.torctl_logpath is None or os.path.isfile(args.torctl_logpath)): - from onionperf.analysis import Analysis - analysis = Analysis(nickname=args.nickname, ip_address=args.ip_address) + from onionperf.analysis import OPAnalysis + analysis = OPAnalysis(nickname=args.nickname, ip_address=args.ip_address) if args.tgen_logpath is not None: analysis.add_tgen_file(args.tgen_logpath) if args.torctl_logpath is not None: @@ -403,13 +403,13 @@ def analyze(args): def visualize(args): from onionperf.visualization import TGenVisualization - from onionperf.analysis import Analysis + from onionperf.analysis import OPAnalysis tgen_viz = TGenVisualization() for (paths, label) in args.datasets: analyses = [] for path in paths: - analysis = Analysis.load(filename=path) + analysis = OPAnalysis.load(filename=path) if analysis is not None: analyses.append(analysis) tgen_viz.add_dataset(analyses, label) diff --git a/requirements.txt b/requirements.txt index f70e46a..46853f8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ pandas scipy seaborn stem >= 1.7.0 +tgentools

1 0

[onionperf/develop] Update Analysis and TGenParser classes to use TGenTools
by karsten＠torproject.org 14 Jul '20

14 Jul '20

commit 0a64de95106fcc3fb389165a74d99200cf4e18ea Author: Ana Custura <ana(a)netstat.org.uk> Date: Fri Jun 26 11:01:16 2020 +0100 Update Analysis and TGenParser classes to use TGenTools --- onionperf/analysis.py | 283 ++------------------------------------------------ 1 file changed, 8 insertions(+), 275 deletions(-) diff --git a/onionperf/analysis.py b/onionperf/analysis.py index f845dd2..2466aad 100644 --- a/onionperf/analysis.py +++ b/onionperf/analysis.py @@ -16,48 +16,28 @@ from stem import CircEvent, CircStatus, CircPurpose, StreamStatus from stem.response.events import CircuitEvent, CircMinorEvent, StreamEvent, BandwidthEvent, BuildTimeoutSetEvent from stem.response import ControlMessage, convert +# tgentools imports +from tgentools.analysis import Analysis, TGenParser + # onionperf imports from . import util -class Analysis(object): +class OPAnalysis(Analysis): def __init__(self, nickname=None, ip_address=None): - self.nickname = nickname - self.measurement_ip = ip_address - self.hostname = gethostname().split('.')[0] + super().__init__(nickname, ip_address) self.json_db = {'type':'onionperf', 'version':'2.0', 'data':{}} - self.tgen_filepaths = [] self.torctl_filepaths = [] - self.date_filter = None - self.did_analysis = False - - def add_tgen_file(self, filepath): - self.tgen_filepaths.append(filepath) def add_torctl_file(self, filepath): self.torctl_filepaths.append(filepath) - def get_nodes(self): - return list(self.json_db['data'].keys()) - def get_tor_bandwidth_summary(self, node, direction): try: return self.json_db['data'][node]['tor']['bandwidth_summary'][direction] except: return None - def get_tgen_transfers(self, node): - try: - return self.json_db['data'][node]['tgen']['transfers'] - except: - return None - - def get_tgen_transfers_summary(self, node): - try: - return self.json_db['data'][node]['tgen']['transfers_summary'] - except: - return None - def analyze(self, do_complete=False, date_filter=None): if self.did_analysis: return @@ -84,17 +64,11 @@ class Analysis(object): if self.measurement_ip is None: self.measurement_ip = "unknown" - self.json_db['data'].setdefault(self.nickname, {'measurement_ip': self.measurement_ip}).setdefault(json_db_key, parser.get_data()) - + self.json_db['data'].setdefault(self.nickname, {'measurement_ip' : self.measurement_ip}).setdefault(json_db_key, parser.get_data()) + self.json_db['data'][self.nickname]["tgen"].pop("heartbeats") + self.json_db['data'][self.nickname]["tgen"].pop("init_ts") self.did_analysis = True - def merge(self, analysis): - for nickname in analysis.json_db['data']: - if nickname in self.json_db['data']: - raise Exception("Merge does not yet support multiple Analysis objects from the same node \ - (add multiple files from the same node to the same Analysis object before calling analyze instead)") - else: - self.json_db['data'][nickname] = analysis.json_db['data'][nickname] def save(self, filename=None, output_prefix=os.getcwd(), do_compress=True, date_prefix=None): if filename is None: @@ -147,150 +121,6 @@ class Analysis(object): analysis_instance.json_db = db return analysis_instance -def subproc_analyze_func(analysis_args): - signal(SIGINT, SIG_IGN) # ignore interrupts - a = analysis_args[0] - do_complete = analysis_args[1] - a.analyze(do_complete=do_complete) - return a - -class ParallelAnalysis(Analysis): - - def analyze(self, search_path, do_complete=False, nickname=None, tgen_search_expressions=["tgen.*\.log"], - torctl_search_expressions=["torctl.*\.log"], num_subprocs=cpu_count()): - - pathpairs = util.find_file_paths_pairs(search_path, tgen_search_expressions, torctl_search_expressions) - logging.info("processing input from {0} nodes...".format(len(pathpairs))) - - analysis_jobs = [] - for (tgen_filepaths, torctl_filepaths) in pathpairs: - a = Analysis() - for tgen_filepath in tgen_filepaths: - a.add_tgen_file(tgen_filepath) - for torctl_filepath in torctl_filepaths: - a.add_torctl_file(torctl_filepath) - analysis_args = [a, do_complete] - analysis_jobs.append(analysis_args) - - analyses = None - pool = Pool(num_subprocs if num_subprocs > 0 else cpu_count()) - try: - mr = pool.map_async(subproc_analyze_func, analysis_jobs) - pool.close() - while not mr.ready(): mr.wait(1) - analyses = mr.get() - except KeyboardInterrupt: - logging.info("interrupted, terminating process pool") - pool.terminate() - pool.join() - sys.exit() - - logging.info("merging {0} analysis results now...".format(len(analyses))) - while analyses is not None and len(analyses) > 0: - self.merge(analyses.pop()) - logging.info("done merging results: {0} total nicknames present in json db".format(len(self.json_db['data']))) - -class TransferStatusEvent(object): - - def __init__(self, line): - self.is_success = False - self.is_error = False - self.is_complete = False - - parts = line.strip().split() - self.unix_ts_end = util.timestamp_to_seconds(parts[2]) - - transport_parts = parts[8].split(',') - self.endpoint_local = transport_parts[2] - self.endpoint_proxy = transport_parts[3] - self.endpoint_remote = transport_parts[4] - - transfer_parts = parts[10].split(',') - - # for id, combine the time with the transfer num; this is unique for each node, - # as long as the node was running tgen without restarting for 100 seconds or longer - # #self.transfer_id = "{0}-{1}".format(round(self.unix_ts_end, -2), transfer_num) - self.transfer_id = "{0}:{1}".format(transfer_parts[0], transfer_parts[1]) # id:count - - self.hostname_local = transfer_parts[2] - self.method = transfer_parts[3] # 'GET' or 'PUT' - self.filesize_bytes = int(transfer_parts[4]) - self.hostname_remote = transfer_parts[5] - self.error_code = transfer_parts[8].split('=')[1] - - self.total_bytes_read = int(parts[11].split('=')[1]) - self.total_bytes_write = int(parts[12].split('=')[1]) - - # the commander is the side that sent the command, - # i.e., the side that is driving the download, i.e., the client side - progress_parts = parts[13].split('=') - self.is_commander = (self.method == 'GET' and 'read' in progress_parts[0]) or \ - (self.method == 'PUT' and 'write' in progress_parts[0]) - self.payload_bytes_status = int(progress_parts[1].split('/')[0]) - - self.unconsumed_parts = None if len(parts) < 16 else parts[15:] - self.elapsed_seconds = {} - -class TransferCompleteEvent(TransferStatusEvent): - def __init__(self, line): - super(TransferCompleteEvent, self).__init__(line) - self.is_complete = True - - i = 0 - elapsed_seconds = 0.0 - # match up self.unconsumed_parts[0:11] with the events in the transfer_steps enum - for k in ['socket_create', 'socket_connect', 'proxy_init', 'proxy_choice', 'proxy_request', - 'proxy_response', 'command', 'response', 'first_byte', 'last_byte', 'checksum']: - # parse out the elapsed time value - keyval = self.unconsumed_parts[i] - i += 1 - - val = float(int(keyval.split('=')[1])) - if val >= 0.0: - elapsed_seconds = val / 1000000.0 # usecs to secs - self.elapsed_seconds.setdefault(k, elapsed_seconds) - - self.unix_ts_start = self.unix_ts_end - elapsed_seconds - del(self.unconsumed_parts) - -class TransferSuccessEvent(TransferCompleteEvent): - def __init__(self, line): - super(TransferSuccessEvent, self).__init__(line) - self.is_success = True - -class TransferErrorEvent(TransferCompleteEvent): - def __init__(self, line): - super(TransferErrorEvent, self).__init__(line) - self.is_error = True - -class Transfer(object): - def __init__(self, tid): - self.id = tid - self.last_event = None - self.payload_progress = {decile:None for decile in [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]} - self.payload_bytes = {partial:None for partial in [10240, 20480, 51200, 102400, 204800, 512000, 1048576, 2097152, 5242880]} - - def add_event(self, status_event): - progress_frac = float(status_event.payload_bytes_status) / float(status_event.filesize_bytes) - progress = float(status_event.payload_bytes_status) - for partial in sorted(self.payload_bytes.keys()): - if progress >= partial and self.payload_bytes[partial] is None: - self.payload_bytes[partial] = status_event.unix_ts_end - for decile in sorted(self.payload_progress.keys()): - if progress_frac >= decile and self.payload_progress[decile] is None: - self.payload_progress[decile] = status_event.unix_ts_end - self.last_event = status_event - - def get_data(self): - e = self.last_event - if e is None or not e.is_complete: - return None - d = e.__dict__ - if not e.is_error: - d['elapsed_seconds']['payload_progress'] = {decile: round(self.payload_progress[decile] - e.unix_ts_start, 6) for decile in self.payload_progress if self.payload_progress[decile] is not None} - d['elapsed_seconds']['payload_bytes'] = {partial: round(self.payload_bytes[partial] - e.unix_ts_start, 6) for partial in self.payload_bytes if self.payload_bytes[partial] is not None} - return d - class Parser(object, metaclass=ABCMeta): @abstractmethod def parse(self, source, do_complete): @@ -302,103 +132,6 @@ class Parser(object, metaclass=ABCMeta): def get_name(self): pass -class TGenParser(Parser): - - def __init__(self, date_filter=None): - ''' date_filter should be given in UTC ''' - self.state = {} - self.transfers = {} - self.transfers_summary = {'time_to_first_byte':{}, 'time_to_last_byte':{}, 'errors':{}} - self.name = None - self.date_filter = date_filter - - def __is_date_valid(self, date_to_check): - if self.date_filter is None: - # we are not asked to filter, so every date is valid - return True - else: - # we are asked to filter, so the line is only valid if the date matches the filter - # both the filter and the unix timestamp should be in UTC at this point - return util.do_dates_match(self.date_filter, date_to_check) - - def __parse_line(self, line, do_complete): - if self.name is None and re.search("Initializing traffic generator on host", line) is not None: - self.name = line.strip().split()[11] - - if self.date_filter is not None: - parts = line.split(' ', 3) - if len(parts) < 4: # the 3rd is the timestamp, the 4th is the rest of the line - return True - unix_ts = float(parts[2]) - line_date = datetime.datetime.utcfromtimestamp(unix_ts).date() - if not self.__is_date_valid(line_date): - return True - - if do_complete and re.search("state\sRESPONSE\sto\sstate\sPAYLOAD", line) is not None: - # another run of tgen starts the id over counting up from 1 - # if a prev transfer with the same id did not complete, we can be sure it never will - parts = line.strip().split() - transfer_parts = parts[7].strip().split(',') - transfer_id = "{0}:{1}".format(transfer_parts[0], transfer_parts[1]) # id:count - if transfer_id in self.state: - self.state.pop(transfer_id) - - elif do_complete and re.search("transfer-status", line) is not None: - status = TransferStatusEvent(line) - xfer = self.state.setdefault(status.transfer_id, Transfer(status.transfer_id)) - xfer.add_event(status) - - elif re.search("transfer-complete", line) is not None: - complete = TransferSuccessEvent(line) - - if do_complete: - xfer = self.state.setdefault(complete.transfer_id, Transfer(complete.transfer_id)) - xfer.add_event(complete) - self.transfers[xfer.id] = xfer.get_data() - self.state.pop(complete.transfer_id) - - filesize, second = complete.filesize_bytes, int(complete.unix_ts_end) - fb_secs = complete.elapsed_seconds['first_byte'] - complete.elapsed_seconds['command'] - lb_secs = complete.elapsed_seconds['last_byte'] - complete.elapsed_seconds['command'] - - fb_list = self.transfers_summary['time_to_first_byte'].setdefault(filesize, {}).setdefault(second, []) - fb_list.append(fb_secs) - lb_list = self.transfers_summary['time_to_last_byte'].setdefault(filesize, {}).setdefault(second, []) - lb_list.append(lb_secs) - - elif re.search("transfer-error", line) is not None: - error = TransferErrorEvent(line) - - if do_complete: - xfer = self.state.setdefault(error.transfer_id, Transfer(error.transfer_id)) - xfer.add_event(error) - self.transfers[xfer.id] = xfer.get_data() - self.state.pop(error.transfer_id) - - err_code, filesize, second = error.error_code, error.filesize_bytes, int(error.unix_ts_end) - - err_list = self.transfers_summary['errors'].setdefault(err_code, {}).setdefault(second, []) - err_list.append(filesize) - - return True - - def parse(self, source, do_complete=False): - source.open() - for line in source: - # ignore line parsing errors - try: - if not self.__parse_line(line, do_complete): - break - except: - logging.warning("TGenParser: skipping line due to parsing error: {0}".format(line)) - continue - source.close() - - def get_data(self): - return {'transfers':self.transfers, 'transfers_summary': self.transfers_summary} - - def get_name(self): - return self.name class TorStream(object): def __init__(self, sid):

1 0

[onionperf/develop] Update tor analysis class and method names to be tor-specific
by karsten＠torproject.org 14 Jul '20

14 Jul '20

commit 54911cd9fc2659e0ea4b5cd5a249489e3fed18eb Author: Ana Custura <ana(a)netstat.org.uk> Date: Fri Jun 26 10:58:57 2020 +0100 Update tor analysis class and method names to be tor-specific --- onionperf/analysis.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/onionperf/analysis.py b/onionperf/analysis.py index eaacbb9..f845dd2 100644 --- a/onionperf/analysis.py +++ b/onionperf/analysis.py @@ -400,7 +400,7 @@ class TGenParser(Parser): def get_name(self): return self.name -class Stream(object): +class TorStream(object): def __init__(self, sid): self.stream_id = sid self.circuit_id = None @@ -460,7 +460,7 @@ class Stream(object): ' '.join(['%s=%s' % (event, arrived_at) for (event, arrived_at) in sorted(self.elapsed_seconds, key=lambda item: item[1])]))) -class Circuit(object): +class TorCircuit(object): def __init__(self, cid): self.circuit_id = cid self.unix_ts_start = None @@ -543,7 +543,7 @@ class TorCtlParser(Parser): def __handle_circuit(self, event, arrival_dt): # first make sure we have a circuit object cid = int(event.id) - circ = self.circuits_state.setdefault(cid, Circuit(cid)) + circ = self.circuits_state.setdefault(cid, TorCircuit(cid)) is_hs_circ = True if event.purpose in (CircPurpose.HS_CLIENT_INTRO, CircPurpose.HS_CLIENT_REND, \ CircPurpose.HS_SERVICE_INTRO, CircPurpose.HS_SERVICE_REND) else False @@ -597,7 +597,7 @@ class TorCtlParser(Parser): def __handle_stream(self, event, arrival_dt): sid = int(event.id) - strm = self.streams_state.setdefault(sid, Stream(sid)) + strm = self.streams_state.setdefault(sid, TorStream(sid)) if event.circ_id is not None: strm.set_circ_id(event.circ_id)

1 0

[onionperf/develop] Update do_simple analysis param to new do_complete tgen semantics
by karsten＠torproject.org 14 Jul '20

14 Jul '20

commit 05eb9cdf56f6ae275ace65a0bdbdcf2c3b5e1c40 Author: Ana Custura <ana(a)netstat.org.uk> Date: Fri Jun 26 10:52:46 2020 +0100 Update do_simple analysis param to new do_complete tgen semantics --- onionperf/analysis.py | 42 +++++++++++++++++++++--------------------- onionperf/onionperf | 8 ++++---- onionperf/reprocessing.py | 8 ++++---- 3 files changed, 29 insertions(+), 29 deletions(-) diff --git a/onionperf/analysis.py b/onionperf/analysis.py index 20ca354..eaacbb9 100644 --- a/onionperf/analysis.py +++ b/onionperf/analysis.py @@ -58,7 +58,7 @@ class Analysis(object): except: return None - def analyze(self, do_simple=True, date_filter=None): + def analyze(self, do_complete=False, date_filter=None): if self.did_analysis: return @@ -70,7 +70,7 @@ class Analysis(object): if len(filepaths) > 0: for filepath in filepaths: logging.info("parsing log file at {0}".format(filepath)) - parser.parse(util.DataSource(filepath), do_simple=do_simple) + parser.parse(util.DataSource(filepath), do_complete=do_complete) if self.nickname is None: parsed_name = parser.get_name() @@ -150,13 +150,13 @@ class Analysis(object): def subproc_analyze_func(analysis_args): signal(SIGINT, SIG_IGN) # ignore interrupts a = analysis_args[0] - do_simple = analysis_args[1] - a.analyze(do_simple=do_simple) + do_complete = analysis_args[1] + a.analyze(do_complete=do_complete) return a class ParallelAnalysis(Analysis): - def analyze(self, search_path, do_simple=True, nickname=None, tgen_search_expressions=["tgen.*\.log"], + def analyze(self, search_path, do_complete=False, nickname=None, tgen_search_expressions=["tgen.*\.log"], torctl_search_expressions=["torctl.*\.log"], num_subprocs=cpu_count()): pathpairs = util.find_file_paths_pairs(search_path, tgen_search_expressions, torctl_search_expressions) @@ -169,7 +169,7 @@ class ParallelAnalysis(Analysis): a.add_tgen_file(tgen_filepath) for torctl_filepath in torctl_filepaths: a.add_torctl_file(torctl_filepath) - analysis_args = [a, do_simple] + analysis_args = [a, do_complete] analysis_jobs.append(analysis_args) analyses = None @@ -293,7 +293,7 @@ class Transfer(object): class Parser(object, metaclass=ABCMeta): @abstractmethod - def parse(self, source, do_simple): + def parse(self, source, do_complete): pass @abstractmethod def get_data(self): @@ -321,7 +321,7 @@ class TGenParser(Parser): # both the filter and the unix timestamp should be in UTC at this point return util.do_dates_match(self.date_filter, date_to_check) - def __parse_line(self, line, do_simple): + def __parse_line(self, line, do_complete): if self.name is None and re.search("Initializing traffic generator on host", line) is not None: self.name = line.strip().split()[11] @@ -334,7 +334,7 @@ class TGenParser(Parser): if not self.__is_date_valid(line_date): return True - if not do_simple and re.search("state\sRESPONSE\sto\sstate\sPAYLOAD", line) is not None: + if do_complete and re.search("state\sRESPONSE\sto\sstate\sPAYLOAD", line) is not None: # another run of tgen starts the id over counting up from 1 # if a prev transfer with the same id did not complete, we can be sure it never will parts = line.strip().split() @@ -343,7 +343,7 @@ class TGenParser(Parser): if transfer_id in self.state: self.state.pop(transfer_id) - elif not do_simple and re.search("transfer-status", line) is not None: + elif do_complete and re.search("transfer-status", line) is not None: status = TransferStatusEvent(line) xfer = self.state.setdefault(status.transfer_id, Transfer(status.transfer_id)) xfer.add_event(status) @@ -351,7 +351,7 @@ class TGenParser(Parser): elif re.search("transfer-complete", line) is not None: complete = TransferSuccessEvent(line) - if not do_simple: + if do_complete: xfer = self.state.setdefault(complete.transfer_id, Transfer(complete.transfer_id)) xfer.add_event(complete) self.transfers[xfer.id] = xfer.get_data() @@ -369,7 +369,7 @@ class TGenParser(Parser): elif re.search("transfer-error", line) is not None: error = TransferErrorEvent(line) - if not do_simple: + if do_complete: xfer = self.state.setdefault(error.transfer_id, Transfer(error.transfer_id)) xfer.add_event(error) self.transfers[xfer.id] = xfer.get_data() @@ -382,12 +382,12 @@ class TGenParser(Parser): return True - def parse(self, source, do_simple=True): + def parse(self, source, do_complete=False): source.open() for line in source: # ignore line parsing errors try: - if not self.__parse_line(line, do_simple): + if not self.__parse_line(line, do_complete): break except: logging.warning("TGenParser: skipping line due to parsing error: {0}".format(line)) @@ -526,7 +526,7 @@ class TorCtlParser(Parser): def __init__(self, date_filter=None): ''' date_filter should be given in UTC ''' - self.do_simple = True + self.do_complete = False self.bandwidth_summary = {'bytes_read':{}, 'bytes_written':{}} self.circuits_state = {} self.circuits = {} @@ -580,11 +580,11 @@ class TorCtlParser(Parser): self.circuits_summary['buildtimes'].append(built - started) if ended is not None and started is not None: self.circuits_summary['lifetimes'].append(ended - started) - if not self.do_simple: + if self.do_complete: self.circuits[cid] = data self.circuits_state.pop(cid) - elif not self.do_simple and isinstance(event, CircMinorEvent): + elif self.do_complete and isinstance(event, CircMinorEvent): if event.purpose != event.old_purpose or event.event != CircEvent.PURPOSE_CHANGED: key = "{0}:{1}".format(event.event, event.purpose) circ.add_event(key, arrival_dt) @@ -620,7 +620,7 @@ class TorCtlParser(Parser): data = strm.get_data() if data is not None: - if not self.do_simple: + if self.do_complete: self.streams[sid] = data self.streams_summary['lifetimes'].setdefault(stream_type, []).append(ended - started) self.streams_state.pop(sid) @@ -664,7 +664,7 @@ class TorCtlParser(Parser): elif re.search("BOOTSTRAP", line) is not None and re.search("PROGRESS=100", line) is not None: self.boot_succeeded = True - if self.do_simple is False or (self.do_simple is True and re.search("650\sBW", line) is not None): + if self.do_complete or (self.do_complete is False and re.search("650\sBW", line) is not None): # parse with stem timestamps, sep, raw_event_str = line.partition(" 650 ") if sep == '': @@ -683,8 +683,8 @@ class TorCtlParser(Parser): self.__handle_event(event, unix_ts) return True - def parse(self, source, do_simple=True): - self.do_simple = do_simple + def parse(self, source, do_complete=False): + self.do_complete = do_complete source.open(newline='\r\n') for line in source: # ignore line parsing errors diff --git a/onionperf/onionperf b/onionperf/onionperf index a7d32f6..ddbeaf1 100755 --- a/onionperf/onionperf +++ b/onionperf/onionperf @@ -281,8 +281,8 @@ files generated by this script will be written""", analyze_parser.add_argument('-s', '--do-simple-parse', help="""parse and export only summary statistics rather than full transfer/circuit/stream data""", - action="store_true", dest="do_simple", - default=False) + action="store_false", dest="do_complete", + default=True) # visualize visualize_parser = sub_parser.add_parser('visualize', description=DESC_VISUALIZE, help=HELP_VISUALIZE, @@ -387,7 +387,7 @@ def analyze(args): analysis.add_tgen_file(args.tgen_logpath) if args.torctl_logpath is not None: analysis.add_torctl_file(args.torctl_logpath) - analysis.analyze(args.do_simple, date_filter=args.date_filter) + analysis.analyze(args.do_complete, date_filter=args.date_filter) analysis.save(output_prefix=args.prefix, date_prefix=args.date_prefix) elif args.tgen_logpath is not None and os.path.isdir(args.tgen_logpath) and args.torctl_logpath is not None and os.path.isdir(args.torctl_logpath): @@ -396,7 +396,7 @@ def analyze(args): torctl_logs = reprocessing.collect_logs(args.torctl_logpath, '*torctl.log*') log_pairs = reprocessing.match(tgen_logs, torctl_logs, args.date_filter) logging.info("Found {0} matching log pairs to be reprocessed".format(len(log_pairs))) - reprocessing.multiprocess_logs(log_pairs, args.prefix, args.nickname, args.do_simple) + reprocessing.multiprocess_logs(log_pairs, args.prefix, args.nickname, args.do_complete) else: logging.error("Given paths were an unrecognized mix of file and directory paths, nothing will be analyzed") diff --git a/onionperf/reprocessing.py b/onionperf/reprocessing.py index f88f311..ad0308f 100644 --- a/onionperf/reprocessing.py +++ b/onionperf/reprocessing.py @@ -46,21 +46,21 @@ def match(tgen_logs, tor_logs, date_filter): return log_pairs -def analyze_func(prefix, nick, do_simple, pair): +def analyze_func(prefix, nick, do_complete, pair): analysis = Analysis(nickname=nick) logging.info('Analysing pair for date {0}'.format(pair[2])) analysis.add_tgen_file(pair[0]) analysis.add_torctl_file(pair[1]) - analysis.analyze(do_simple=do_simple, date_filter=pair[2]) + analysis.analyze(do_complete=do_complete, date_filter=pair[2]) analysis.save(output_prefix=prefix) return 1 -def multiprocess_logs(log_pairs, prefix, nick=None, do_simple=False): +def multiprocess_logs(log_pairs, prefix, nick=None, do_complete=False): pool = Pool(cpu_count()) analyses = None try: - func = partial(analyze_func, prefix, nick, do_simple) + func = partial(analyze_func, prefix, nick, do_complete) mr = pool.map_async(func, log_pairs) pool.close() while not mr.ready():

1 0