July 2020 - tor-commits - lists.torproject.org

[onionperf/develop] Remove README_JSON.md and point to JSON schema.
by karsten＠torproject.org 16 Jul '20

16 Jul '20

commit a817e5ae5ece1d753f3688ee579e87e93951a779 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Jul 16 22:14:28 2020 +0200 Remove README_JSON.md and point to JSON schema. --- README.md | 2 +- README_JSON.md | 213 --------------------------------------------------------- 2 files changed, 1 insertion(+), 214 deletions(-) diff --git a/README.md b/README.md index ecd7c0c..b9bf7e6 100644 --- a/README.md +++ b/README.md @@ -236,7 +236,7 @@ For example, the following command analyzes current log files of a running (or s onionperf analyze --tgen ~/onionperf-data/tgen-client/onionperf.tgen.log --torctl ~/onionperf-data/tor-client/onionperf.torctl.log ``` -The output analysis file is written to `onionperf.analysis.json.xz` in the current working directory. The file format is described in more detail in `README_JSON.md`. +The output analysis file is written to `onionperf.analysis.json.xz` in the current working directory. The file format is described in more detail in `schema/onionperf-3.0.json`. The same analysis files are written automatically as part of ongoing measurements once per day at UTC midnight and can be found in `onionperf-data/htdocs/`. diff --git a/README_JSON.md b/README_JSON.md deleted file mode 100644 index f0ad281..0000000 --- a/README_JSON.md +++ /dev/null @@ -1,213 +0,0 @@ -# DB Structure - -This document describes the structure of the json database file that gets exported in `analysis` mode and gets placed in the www docroot when running in `measure` mode. - -The structure is given here with variable keys marked as such. - - { - "data": { # generic keyword - "phantomtrain": { # nickname of the OnionPerf client, hostname if not set - "measurement_ip" : "192.168.1.1", # public-facing IP address of the machine used for the measurements - "tgen": { # to indicate data from TGen - "transfers": { # the records for transfers TGen attempted - "transfer1m:1": { # the id of a single transfer - "elapsed_seconds": { # timing for various steps in transfer, in seconds - "checksum": 0.0, # step 12 if using a proxy, else step 8 (initial GET/PUT) - "command": 0.319006, # step 7 if using a proxy, else step 3 (initial GET/PUT) - "first_byte": 0.0, # step 9 if using a proxy, else step 5 (initial GET/PUT) - "last_byte": 0.0, # step 11 if using a proxy, else step 7 (initial GET/PUT) - "payload_bytes": { # similar to payload_progress below - "10240": 0.0, # number of payload bytes completed : seconds to complete it - "20480": 0.0, - "51200": 0.0, - "102400": 0.0, - "204800": 0.0, - "512000": 0.0, - "1048576": 0.0, - "2097152": 0.0, - "5242880": 0.0 - }, - "payload_progress": { # step 10 if using a proxy, else step 6 (initial GET/PUT) - "0.0": 0.0, # percent of payload completed : seconds to complete it - "0.1": 0.0, - "0.2": 0.0, - "0.3": 0.0, - "0.4": 0.0, - "0.5": 0.0, - "0.6": 0.0, - "0.7": 0.0, - "0.8": 0.0, - "0.9": 0.0, - "1.0": 0.0 - }, - "proxy_choice": 0.000233, # step 4 if using a proxy, else absent - "proxy_init": 0.000151, # step 3 if using a proxy, else absent - "proxy_request": 0.010959, # step 5 if using a proxy, else absent - "proxy_response": 0.318873, # step 6 if using a proxy, else absent - "response": 0.0, # step 8 if using a proxy, else step 4 (initial GET/PUT) - "socket_connect": 0.000115, # step 2 - "socket_create": 2e-06 # step 1 - }, - "endpoint_local": "localhost:127.0.0.1:45416", # tgen client socket name:ip:port - "endpoint_proxy": "localhost:127.0.0.1:27942", # proxy socket name:ip:port, if present - "endpoint_remote": "server1.peach-hosting.com:216.17.99.183:6666", # tgen server hostname:ip:port - "error_code": "READ", # 'NONE' or a code to indicate the type of error - "filesize_bytes": 1048576, # size of the transfer payload - "hostname_local": "puntaburros.amerinoc.com", # client machine hostname - "hostname_remote": "(null)", # server machine hostname - "is_commander": true, # true if client (initiated the transfer), else false - "is_complete": true, # if the transfer finished, no matter the error state - "is_error": true, # if there was an error in the transfer - "is_success": false, # if the transfer completed and checksum passed - "method": "GET", # transfer method (GET,PUT) - "payload_bytes_status": 0, # cumulative number of payload bytes received - "total_bytes_read": 0, # total bytes read from the socket - "total_bytes_write": 50, # total written to the socket - "transfer_id": "transfer1m:1", # the id of this transfer, unique to this run of OnionPerf - "unix_ts_end": 1456699868.006196, # initial start time of the transfer - "unix_ts_start": 1456699868.006196 # final end time of the transfer - }, - }, - "transfers_summary": { # summary stats of all transfers in the 'transfers' section - "errors": { - "PROXY": { # PROXY type errors - "1456654221": [ # the second at which the error occurred - 51200 # transfer filesizes that had errors, one entry for each error during this second - ], - }, - "READ": { # READ type errors - "1456618782": [ # second at which the error occurred - 51200 # transfer filesize, one for each error at this time - ], - }, - "time_to_first_byte": { # time to receive the first byte of the payload - "51200": { # file size - "1456707932": [ # the second at which the transfer completed - 0.36213199999999995 # time to first byte, in seconds - ], - }, - }, - "time_to_last_byte": { # time to receive the last byte of the payload - "51200": { # file size - "1456707932": [ # the second at which the transfer completed - 0.6602399999999999 # time to last byte, in seconds - ], - } - } - }, - }, - "tor": { # indicates data from Tor - "bandwidth_summary": { # from Tor's BW controller event - "bytes_read": { - "1456617599": 0, # unix time in seconds : number of bytes - }, - "bytes_written": { - "1456617599": 0, # unix time in seconds : number of bytes - } - }, - "streams": { # info about each stream - "23": { # stream ID - "circuit_id": "4", # circuit on which this stream was attached - "elapsed_seconds": [ # time in seconds to reach various points, from STREAM Tor events - [ - "USER:NEW", # stream purpose : stream status (from Tor) - 0.0 - ], - [ - "USER:SENTCONNECT", - 0.0 - ], - [ - "USER:REMAP", - 0.31999993324279785 - ], - [ - "USER:SUCCEEDED", - 0.31999993324279785 - ], - [ - "USER:CLOSED", - 0.9800000190734863 - ] - ], - "source": "127.0.0.1:48786", # ip:port of client that created stream - "stream_id": 23, # id of the stream used by Tor - "target": "216.17.99.183:6666", # ip:port of the remote end of the stream - "unix_ts_end": 1456707932.67, # unix time in seconds that the stream started - "unix_ts_start": 1456707931.69 # unix time in seconds that the stream ended - }, - }, - "streams_summary": { # summary stats about all streams - "lifetimes": { # time streams were alive, in seconds - "DIR_FETCH": [ # streams of type DIR_FETCH, one entry for each - 3.950000047683716, - ], - "USER": [ # streams of type USER, one entry for each - 114.72000002861023, - ] - } - }, - "circuits": { # info about each circuit - "10": { # circuit ID - "build_quantile": 0.8, - "build_timeout": 1500, - "buildtime_seconds": 1.0900001525878906, - "circuit_id": 10, - "elapsed_seconds": [ # time in seconds to reach various points, from CIRC Tor events - [ - "GENERAL:LAUNCHED", # circuit purpose : circuit status (from Tor) - 0.0 - ], - [ - "GENERAL:EXTENDED", - 0.75 - ], - [ - "GENERAL:EXTENDED", - 0.9100000858306885 - ], - [ - "GENERAL:EXTENDED", - 1.0900001525878906 - ], - [ - "GENERAL:BUILT", - 1.0900001525878906 - ], - [ - "GENERAL:CLOSED", - 3601.0 - ] - ], - "path": [ # fingerprint~nickname for each relay in the path, seconds to extend to that relay - [ - "$BB60F5BA113A0B8B44B7B37DE3567FE561E92F78~Casper04", - 0.75 - ], - [ - "$2FD0BA57A34DC2792AF470398F72F37F9E51DC2D~serotonin", - 0.9100000858306885 - ], - [ - "$DE7DE889E0D1A5F397AE35642060B84999581203~DigiGesTor2e3", - 1.0900001525878906 - ] - ], - "unix_ts_end": 1456711533.59, # unix time in seconds that the circuit started - "unix_ts_start": 1456707932.59 # unix time in seconds that the circuit ended - }, - }, - "circuits_summary": { # summary stats about all circuits - "buildtimes": [ # time to build circuits in seconds, one entry for each circuit - 1.2100000381469727, - ], - "lifetimes": [ # lifetime of circuits in seconds, one entry for each circuit - 60.99000000953674, - ], - } - } - } - }, - "type": "onionperf", # document type - "version": "2.0" # document version - }

1 0

[tor/master] Merge remote-tracking branch 'tor-gitlab/mr/57'
by nickm＠torproject.org 16 Jul '20

16 Jul '20

commit 85219d887dd0e54e9655eff15831839aad304bc0 Merge: b83c6ab737 ca1003b791 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Jul 16 14:53:03 2020 -0400 Merge remote-tracking branch 'tor-gitlab/mr/57' doc/HACKING/CodingStandards.md | 12 ++++++++++++ 1 file changed, 12 insertions(+)

1 0

[tor/master] doc: Update our requirements for changes file
by nickm＠torproject.org 16 Jul '20

16 Jul '20

commit ca1003b79178b8a01dc5fe3a14550e521a4035b0 Author: David Goulet <dgoulet(a)torproject.org> Date: Thu Jul 16 13:34:52 2020 -0400 doc: Update our requirements for changes file Closes tpo/core/team#3. Signed-off-by: David Goulet <dgoulet(a)torproject.org> --- doc/HACKING/CodingStandards.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/doc/HACKING/CodingStandards.md b/doc/HACKING/CodingStandards.md index e9bb75b986..45e33a6777 100644 --- a/doc/HACKING/CodingStandards.md +++ b/doc/HACKING/CodingStandards.md @@ -87,6 +87,18 @@ months and expect it to merge cleanly. Try to merge pieces early and often. ## How we log changes +Before diving in the details on how to write a changes file, note that we do +NOT require a changes file for: + + * Any change to a file that is not distributed in the tarball. This + includes: + - Any change to our CI configuration that does not affect the distributed + source. + - Any change to developer-only tools, unless those tools are distributed + in the tarball. + * Non-functional code movement. + * Identifier re-namings, comment edits, spelling fixes, and so on. + When you do a commit that needs a ChangeLog entry, add a new file to the `changes` toplevel subdirectory. It should have the format of a one-entry changelog section from the current ChangeLog file, as in

1 0

[tor/master] Combine text into existing "what does not need a changes file" section.
by nickm＠torproject.org 16 Jul '20

16 Jul '20

commit 5f969681a0f67b3124d25a702b10796ba6f3d297 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Jul 16 14:54:17 2020 -0400 Combine text into existing "what does not need a changes file" section. --- doc/HACKING/CodingStandards.md | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/doc/HACKING/CodingStandards.md b/doc/HACKING/CodingStandards.md index 45e33a6777..1411ff408a 100644 --- a/doc/HACKING/CodingStandards.md +++ b/doc/HACKING/CodingStandards.md @@ -87,18 +87,6 @@ months and expect it to merge cleanly. Try to merge pieces early and often. ## How we log changes -Before diving in the details on how to write a changes file, note that we do -NOT require a changes file for: - - * Any change to a file that is not distributed in the tarball. This - includes: - - Any change to our CI configuration that does not affect the distributed - source. - - Any change to developer-only tools, unless those tools are distributed - in the tarball. - * Non-functional code movement. - * Identifier re-namings, comment edits, spelling fixes, and so on. - When you do a commit that needs a ChangeLog entry, add a new file to the `changes` toplevel subdirectory. It should have the format of a one-entry changelog section from the current ChangeLog file, as in @@ -197,6 +185,14 @@ What needs a changes file? What does not need a changes file? * Bugfixes for code that hasn't shipped in any released version of Tor + * Any change to a file that is not distributed in the tarball. This + includes: + * Any change to our CI configuration that does not affect the distributed + source. + * Any change to developer-only tools, unless those tools are distributed + in the tarball. + * Non-functional code movement. + * Identifier re-namings, comment edits, spelling fixes, and so on. Why use changes files instead of Git commit messages?

1 0

[tor/master] nodelist: Fix possible NULL deref
by dgoulet＠torproject.org 16 Jul '20

16 Jul '20

commit aa9561164a94c7ad8d61d3fdc153fac15d98143a Author: David Goulet <dgoulet(a)torproject.org> Date: Thu Jul 16 13:18:53 2020 -0400 nodelist: Fix possible NULL deref Found by coverity with CID 1465290. Fix unreleased code. Fixes #40049 Signed-off-by: David Goulet <dgoulet(a)torproject.org> --- src/feature/nodelist/nodelist.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/feature/nodelist/nodelist.c b/src/feature/nodelist/nodelist.c index c2e2f53038..c30684d2d8 100644 --- a/src/feature/nodelist/nodelist.c +++ b/src/feature/nodelist/nodelist.c @@ -1988,6 +1988,12 @@ node_set_country(node_t *node) else if (node->ri) ipv4_addr = &node->ri->ipv4_addr; + /* IPv4 is mandatory for a relay so this should not happen unless we are + * attempting to set the country code on a node without a descriptor. */ + if (BUG(!ipv4_addr)) { + node->country = -1; + return; + } node->country = geoip_get_country_by_addr(ipv4_addr); }

1 0

[tor/master] Merge branch 'tor-gitlab/mr/56'
by dgoulet＠torproject.org 16 Jul '20

16 Jul '20

commit b83c6ab737e403fd555c6fe0e0af6abaa5a48044 Merge: e66eba664a aa9561164a Author: David Goulet <dgoulet(a)torproject.org> Date: Thu Jul 16 14:15:25 2020 -0400 Merge branch 'tor-gitlab/mr/56' src/feature/nodelist/nodelist.c | 6 ++++++ 1 file changed, 6 insertions(+)

1 0

[tor/master] doc: Fix wrong path in include.am
by dgoulet＠torproject.org 16 Jul '20

16 Jul '20

commit e66eba664ae85b786097cdd22c1a601ad45d95b5 Author: David Goulet <dgoulet(a)torproject.org> Date: Thu Jul 16 14:07:33 2020 -0400 doc: Fix wrong path in include.am Tracing.md moved to doc/tracing/ so fix the distcheck path in include.am. Also add the new EventsCircuit.md file. Signed-off-by: David Goulet <dgoulet(a)torproject.org> --- doc/include.am | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/include.am b/doc/include.am index d3e757f3bb..7a8a64ed16 100644 --- a/doc/include.am +++ b/doc/include.am @@ -59,8 +59,9 @@ EXTRA_DIST+= doc/asciidoc-helper.sh \ doc/HACKING/HowToReview.md \ doc/HACKING/Module.md \ doc/HACKING/ReleasingTor.md \ - doc/HACKING/Tracing.md \ doc/HACKING/WritingTests.md + doc/HACKING/tracing/Tracing.md \ + doc/HACKING/tracing/EventsCircuit.md docdir = @docdir@

1 0

[tor/master] Use Debian for Gitlab CI.
by ahf＠torproject.org 16 Jul '20

16 Jul '20

commit 77b7cbb66382ccf77057746a707435abd47d746d Author: Alexander Færøy <ahf(a)torproject.org> Date: Thu Jul 16 16:48:03 2020 +0000 Use Debian for Gitlab CI. See: tor#40048. --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 402c7646c0..e8d6c5f418 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,3 +1,5 @@ +image: debian:stable + before_script: - apt-get update -qq - apt-get upgrade -qy

1 0

[tor/master] Merge remote-tracking branch 'tor-gitlab/mr/55'
by ahf＠torproject.org 16 Jul '20

16 Jul '20

commit 62201b0f716d12ec1fa088fac4c256f54a21da4f Merge: 4b24b36289 77b7cbb663 Author: Alexander Færøy <ahf(a)torproject.org> Date: Thu Jul 16 17:20:25 2020 +0000 Merge remote-tracking branch 'tor-gitlab/mr/55' .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+)

1 0

[styleguide/master] Change repo to tpo gitlab instance
by gus＠torproject.org 16 Jul '20

16 Jul '20

commit c2b8e5fd789af022409c6cacb8785c0239e80af1 Author: gus <gus(a)torproject.org> Date: Thu Jul 16 13:01:06 2020 -0400 Change repo to tpo gitlab instance --- content/getting-started/contents.lr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/getting-started/contents.lr b/content/getting-started/contents.lr index e1bd312..650225e 100644 --- a/content/getting-started/contents.lr +++ b/content/getting-started/contents.lr @@ -20,7 +20,7 @@ We also provide a minimal Lektor template website that you can start modifying. <div class="border p-5 mb-3"> <h6>Use The Tor Project Styleguide</h6> <p class="m-0">This package includes .scss and .css files</p> -<a class="btn btn-lg btn-link p-0" title="Tor Media Repo" href="https://oniongit.eu/infra/styleguide" target="_blank">Visit Repo</a> +<a class="btn btn-lg btn-link p-0" title="Tor Media Repo" href="https://gitlab.torproject.org/tpo/web/styleguide" target="_blank">Visit Repo</a> </div> ---

1 0