March 2019 - tor-commits - lists.torproject.org

[tor/master] Improve pre-push.git-hook description
by dgoulet＠torproject.org 05 Mar '19

05 Mar '19

commit 0deea98d021d105eeb5dc5c43c144847ad188ea5 Author: rl1987 <rl1987(a)sdf.lonestar.org> Date: Wed Feb 20 19:56:37 2019 +0200 Improve pre-push.git-hook description --- scripts/maint/pre-push.git-hook | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/maint/pre-push.git-hook b/scripts/maint/pre-push.git-hook index 14bc0690c..24043978b 100755 --- a/scripts/maint/pre-push.git-hook +++ b/scripts/maint/pre-push.git-hook @@ -1,9 +1,10 @@ #!/bin/bash -# git pre-push hook script to prevent "fixup!" and "squash!" commit -# from ending up in master, or in any branch if CUR_BRANCH check is removed. -# Furthermore, it disallows pushing branches other than master, release-* -# and maint-* to origin (e.g. gitweb.torproject.org) +# git pre-push hook script to: +# 1) prevent "fixup!" and "squash!" commit from ending up in master, release-* +# or maint-* +# 2) Disallow pushing branches other than master, release-* +# and maint-* to origin (e.g. gitweb.torproject.org) # # To install this script, copy it into .git/hooks/pre-push path in your # local copy of git repository. Make sure it has permission to execute.

1 0

[meek/master] Look for EOF on stdin while grepping for the helper address.
by dcf＠torproject.org 05 Mar '19

05 Mar '19

commit 01ea27b4c59cd8ae17e608006cd7a9a16ba282c4 Author: David Fifield <david(a)bamsoftware.com> Date: Sat Feb 23 00:23:51 2019 -0700 Look for EOF on stdin while grepping for the helper address. This way, we still exit when the parent process dies, even if for whatever reason Firefox doesn't give us a helper address. https://bugs.torproject.org/29559 --- meek-client-torbrowser/meek-client-torbrowser.go | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/meek-client-torbrowser/meek-client-torbrowser.go b/meek-client-torbrowser/meek-client-torbrowser.go index f3ffc68..16f0ebc 100644 --- a/meek-client-torbrowser/meek-client-torbrowser.go +++ b/meek-client-torbrowser/meek-client-torbrowser.go @@ -370,10 +370,25 @@ func main() { defer logKill(firefoxCmd.Process) // Find out the helper's listening address. - helperAddr, err := grepHelperAddr(stdout) - if err != nil { + addrChan := make(chan string) + errChan := make(chan error) + go func() { + addr, err := grepHelperAddr(stdout) + if err == nil { + addrChan <- addr + } else { + errChan <- err + } + }() + var helperAddr string + select { + case sig := <-sigChan: + log.Printf("sig %s", sig) + return + case err = <-errChan: log.Print(err) return + case helperAddr = <-addrChan: } // Start meek-client with the helper address.

1 0

[sbws/master] Release v1.0.3
by juga＠torproject.org 05 Mar '19

05 Mar '19

commit 2d8502a86643220b5f2680f9623ebfa5dfda0a80 Author: juga0 <juga(a)riseup.net> Date: Tue Mar 5 18:49:42 2019 +0000 Release v1.0.3 --- sbws/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbws/__init__.py b/sbws/__init__.py index 84b2787..f8f55c8 100644 --- a/sbws/__init__.py +++ b/sbws/__init__.py @@ -1,4 +1,4 @@ -__version__ = '1.0.3-dev0' +__version__ = '1.0.4-dev0' import threading # noqa

1 0

[sbws/master] fix: !minor. Catch SocketClosed when stopping
by juga＠torproject.org 05 Mar '19

05 Mar '19

commit 315215dc66089302a92bd3a01c37d008f698811f Author: juga0 <juga(a)riseup.net> Date: Tue Mar 5 04:26:42 2019 +0000 fix: !minor. Catch SocketClosed when stopping When SocketClosed is raised and the scanner is stopping, catch the exception. In #28869 similar exceptions were catched, but this was forgotten. Bugfix v0.6.0. --- sbws/core/scanner.py | 13 ++++++++++--- sbws/lib/circuitbuilder.py | 2 +- sbws/util/requests.py | 3 +++ sbws/util/stem.py | 14 ++++++++++++-- 4 files changed, 26 insertions(+), 6 deletions(-) diff --git a/sbws/core/scanner.py b/sbws/core/scanner.py index 4ad2cb5..cdf0b9b 100644 --- a/sbws/core/scanner.py +++ b/sbws/core/scanner.py @@ -240,6 +240,9 @@ def measure_relay(args, conf, destinations, cb, rl, relay): log.debug('Measuring %s %s', relay.nickname, relay.fingerprint) s = requests_utils.make_session( cb.controller, conf.getfloat('general', 'http_timeout')) + # Probably because the scanner is stopping. + if s is None: + return None # Pick a destionation dest = destinations.next() # If there is no any destination at this point, it can not continue. @@ -401,13 +404,17 @@ def result_putter_error(target): measurement -- and return that function so it can be used by someone else ''' def closure(object): + if settings.end_event.is_set(): + return # The only object that can be here if there is not any uncatched # exception is stem.SocketClosed when stopping sbws # An exception here means that the worker thread finished. log.warning(FILLUP_TICKET_MSG) - # To print the traceback that happened in the thread, not here in the - # main process - traceback.print_exception(type(object), object, object.__traceback__) + # To print the traceback that happened in the thread, not here in + # the main process. + log.warning("".join(traceback.format_exception( + type(object), object, object.__traceback__)) + ) return closure diff --git a/sbws/lib/circuitbuilder.py b/sbws/lib/circuitbuilder.py index c5bae5c..45b807f 100644 --- a/sbws/lib/circuitbuilder.py +++ b/sbws/lib/circuitbuilder.py @@ -66,7 +66,7 @@ class CircuitBuilder: circ_id = c.new_circuit( path, await_build=True, timeout=timeout) except (InvalidRequest, CircuitExtensionFailed, - ProtocolError, Timeout) as e: + ProtocolError, Timeout, SocketClosed) as e: return None, str(e) return circ_id, None diff --git a/sbws/util/requests.py b/sbws/util/requests.py index 615916d..ede847d 100644 --- a/sbws/util/requests.py +++ b/sbws/util/requests.py @@ -24,6 +24,9 @@ def make_session(controller, timeout): """ s = TimedSession() socks_info = stem_utils.get_socks_info(controller) + # Probably because scanner is stopping. + if socks_info is None: + return None s.proxies = { 'http': 'socks5h://{}:{}'.format(*socks_info), 'https': 'socks5h://{}:{}'.format(*socks_info), diff --git a/sbws/util/stem.py b/sbws/util/stem.py index cd8ca6e..05f8e90 100644 --- a/sbws/util/stem.py +++ b/sbws/util/stem.py @@ -14,6 +14,7 @@ import os from sbws.globals import fail_hard from sbws.globals import (TORRC_STARTING_POINT, TORRC_RUNTIME_OPTIONS, TORRC_OPTIONS_CAN_FAIL) +from sbws import settings log = logging.getLogger(__name__) stream_building_lock = RLock() @@ -50,6 +51,11 @@ def add_event_listener(controller, func, event): def remove_event_listener(controller, func): try: controller.remove_event_listener(func) + except SocketClosed as e: + if not settings.end_event.is_set(): + log.debug(e) + else: + log.exception(e) except ProtocolError as e: log.exception("Exception trying to remove event %s", e) @@ -245,9 +251,13 @@ def get_socks_info(controller): try: socks_ports = controller.get_listeners(Listener.SOCKS) return socks_ports[0] + except SocketClosed as e: + if not settings.end_event.is_set(): + log.debug(e) + # This might need to return the eception if this happen in other cases + # than when stopping the scanner. except ControllerError as e: - log.exception("Exception trying to get socks info: %e.", e) - exit(1) + log.debug(e) def only_relays_with_bandwidth(controller, relays, min_bw=None, max_bw=None):

1 0

[sbws/master] chg: Changelog: Remove .md and add link to .rst
by juga＠torproject.org 05 Mar '19

05 Mar '19

commit fd1ab125e9ae673457e5b58498533e8931432e80 Author: juga0 <juga(a)riseup.net> Date: Tue Mar 5 18:42:50 2019 +0000 chg: Changelog: Remove .md and add link to .rst --- CHANGELOG.md | 300 ---------------------------------------------- docs/source/CHANGELOG.md | 1 - docs/source/CHANGELOG.rst | 1 + 3 files changed, 1 insertion(+), 301 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index d43f7a0..0000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,300 +0,0 @@ -# Changelog - -All notable changes to this project will be documented in this file. - -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) -and this project adheres to -[Semantic Versioning](https://semver.org/spec/v2.0.0.html). - -## [Unreleased] - -### Changed - -- Ensure de default scaling method is torflow, leave cli option but ignore it, - update documentation reflecting this. (#28446). - -### Fixed - -- Make sbws round to 3 significant figures in torflow rounding mode, rather - than rounding to the nearest 1000 kilobytes (#28442). -- Continue running with defaults when the config argument is provided - but not found (#28500). -- Make sbws round to 2 significant figures by default. This implements part - of proposal 276 (#28451). -- Stop trying to resolve destination domain locally and unify criteria to - choose relays as exits (#28458, #28471). Bugfix 1.0.4 - -## [1.0.2] - 2018-11-10 - -### Fixed - -- Update bandwidth file specification version in the `generator` (#28366). -- Use 5 "=" characters as terminator in the bandwidth files (#28379) - -### Changed - -- Include the headers about eligible relays in all the bandwidth files, - not only in the ones that does not have enough eligible relays (#28365). - -## [1.0.1] - 2018-11-01 - -### Changed - -- Change default directories when sbws is run from a system service (#28268). - -## [1.0.0] - 2018-10-29 - -**Important changes**: - -- `generate` includes extra statistics header lines when the number of - eligible relays to include is less than the 60% of the network. - It does not include the relays' lines. -- Speed up `scanner` by disabling RTT measurements and waiting for - measurement threads before prioritizing again the list of relays to measure. - -### Fixed - -- Update python minimal version in setup (#28043) -- Catch unhandled exception when we fail to resolve a domain name (#28141) -- Bandwidth filtered is the maximum between the bandwidth measurements and - their mean, not the minimum (#28215) -- Stop measuring the same relay by two threads(#28061) - -### Changed - -- Move ``examples/`` to ``docs/`` (#28040) -- Number of results comparison and number of results away from each other are - incorrect (#28041) -- Stop removing results that are not away from some other X secs (#28103) -- Use secs-away when provided instead of data_period (#28105) -- Disable measuring RTTs (#28159) -- Rename bandwidth file keyvalues (#28197) - -## Added - -- Write bw file only when the percentage of measured relays is bigger than 60% - (#28062) -- When the percentage of measured relays is less than the 60%, do not include - the relays in the bandwidth file and instead include some statistics in the - header (#28076) -- When the percentage of measured relays is less than the 60% and it was more - before, warn about it (#28155) -- When the difference between the total consensus bandwidth and the total - in the bandwidth lines is larger than 50%, warn (#28216) -- Add documentation about how the bandwidth measurements are selected and - scaled before writing them to the Bandwidth File (#27692) - -## [0.8.0] - 2018-10-08 - -**Important changes**: - -- Implement Torflow scaling/aggregation to be able to substitute Torflow with - sbws without affecting the bandwidth files results. -- Change stem dependency to 1.7.0, which removes the need for `dependency_links`` -- Update and cleanup documentation - -### Added - -- Add system physical requirements section to INSTALL (#26937) -- Warn when there is not enough disk space (#26937) -- Implement Torflow scaling (#27108) -- Create methods to easy graph generation and obtain statistics - to compare with current torflow results.(#27688) -- Implement rounding bw in bandwidth files to 2 insignificant digits(#27337) -- Filter results in order to include relays in the bandwidth file that:(#27338) - - have at least two measured bandwidths - - the measured bandwidths are within 24 hours of each other - - have at least two descriptor observed bandwidths - - the descriptor observed bandwidths are within 24 hours of each other - -### Fixed - -- Broken environment variable in default sbws config. To use envvar $FOO, write - $$FOO in the config. -- Stop using directory as argument in integration tests (#27342) -- Fix typo getting configuration option to allow logging to file (#27960) -- Set int type to new arguments that otherwise would be string (#27918) -- Stop printing arguments default values, since they are printed by default - (#27916) -- Use dash instead of underscore in new cli argument names (#27917) - -### Changed - -- sbws install doc is confusing (#27341) - - Include system and Python dependencies in `INSTALL`. - - Include dependencies for docs and tests in `INSTALL`. - - Point to `DEPLOY` to run sbws. - - Remove obsolete sections in `INSTALL` - - Simplify `DEPLOY`, reuse terms in the `glossary`. - - Remove obsolete ``sbws init`` from `DEPLOY`. - - Point to config documentation. - - Add, unify and reuse terms in `glossary`. -- refactor v3bwfile (#27386): move scaling method inside class -- use custom ``install_command`` to test installation commands while - ``dependency_links`` is needed until #26914 is fixed. (#27704) -- documentation cleanup (#27773) - - split, merge, simplify, extend, reorganize sections and files -- generate scales as Torflow by default (#27976) -- Replace stem ``dependency_links`` by stem 1.7.0 (#27705). This also eliminates - the need for custom ``install_command`` in tox. - -## [0.7.0] - 2018-08-09 - -**Important changes**: - -- `cleanup/stale_days` is renamed to `cleanup/data_files_compress_after_days` -- `cleanup/rotten_days` is renamed to `cleanup/data_files_delete_after_days` -- sbws now takes as an argument the path to a config file (which contains - `sbws_home`) instead of `sbws_home` (which contains the path to a config -file) - -### Added - -- Log line on start up with sbws version, platform info, and library versions -(trac#26751) -- Manual pages (#26926) - -### Fixed - -- Stop deleting the latest.v3bw symlink. Instead, do an atomic rename. - (#26740) -- State file for storing the last time `sbws scanner` was started, and able to - be used for storing many other types of state in the future. (GH#166) -- Log files weren't rotating. Now they are. (#26881) - -### Changed - -- Remove test data v3bw file and generate it from the same test. (#26736) -- Stop using food terms for cleanup-related config options -- Cleanup command now cleans up old v3bw files too (#26701) -- Make sbws more compatible with system packages: (#26862) - - Allow a configuration file argument - - Remove directory argument - - Create minimal user configuration when running - - Do not require to run a command to initialize - - Initialize directories when running - - Do not require configuration file inside directories specified by the - configuration - -## [0.6.0] - 2018-07-11 - -**Important changes**: - -- The way users configure logging has changed. No longer are most users - expected to be familiar with how to configure python's standard logging -library with a config file. Instead we've abstracted out the setting of log -level, format, and destinations to make these settings more accessible to -users. Expert users familiar with [the logging config file format][logconffmt] -can still make tweaks. - -Summary of changes: - -- Make logging configuration easier for the user. -- Add UML diagrams to documentation. They can be found in docs/source/images/ - and regenerated with `make umlsvg` in docs/. - -[logconffmt]: https://docs.python.org/3/library/logging.config.html#logging-config-filefo… - -### Added - -- UML diagrams to documentation. In docs/ run `make umlsvg` to rebuild them. - Requires graphviz to be installed.(GHPR#226) -- Add metadata to setup.py, useful for source/binary distributions. -- Add possibility to log to system log. (#26683) -- Add option to cleanup v3bw files. (#26701) - -### Fixed - -- Measure relays that have both Exit and BadExit as non-exits, which is how - clients would use them. (GH#217) -- Could not init sbws because of a catch-22 related to logging configuration. - Overhaul how logging is configured. (GH#186 GHPR#224) -- Call write method of V3BWFile class from the object instance. (#26671) -- Stop calculating median on empty list .(#26666) - -### Changed - -- Remove is_controller_ok. Instead catch possible controller exceptions and -log them - -### Removed - -- Two parsing/plotting scripts in scripts/tools/ that can now be found at -<https://github.com/pastly/v3bw-tools> - -## [0.5.0] - 2018-06-26 - -**Important changes**: - -- Result format changed, causing a version bump to 4. Updating sbws to 0.5.0 - will cause it to ignore results with version less than 4. - -Summary of changes: - -- Keep previously-generated v3bw files -- Allow a relay to limit its weight based on - RelayBandwidthRate/MaxAdvertisedBandwidth -- 1 CPU usage optimization -- 1 memory usage optimization - -### Added - -- Use a relay's {,Relay}BandwidthRate/MaxAdvertisedBandwidth as an upper bound - on the measurements we make for it. (GH#155) -- Ability to only consider results for a given relay valid if they came from - when that relay is using its most recent known IP address. Thanks Juga. -(GH#154 GHPR#199) -- Maintenance script to help us find functions that are (probably) no longer - being called. -- Integration test(s) for RelayPrioritizer (GHPR#206) -- Git/GitHub usage guidelines to CONTRIBUTING document (GH#208 GHPR#215) - -### Fixed - -- Make relay priority calculations take only ~5% of the time they used to (3s - vs 60s) by using sets instead of lists when selecting non-Authority relays. -(GH#204) -- Make relay list refreshing take much less time by not allowing worker threads - to dogpile on the CPU. Before they would all start requesting descriptors -from Tor at roughly the same time, causing us to overload our CPU core and make -the process take unnecessarily long. Now we let one thread do the work so it -can peg the CPU on its own and get the refresh done ASAP. -(GH#205) -- Catch a JSON decode exception on malformed results so sbws can continue - gracefully (GH#210 GHPR#212) - -### Changed - -- Change the path where the Bandwidth List files are generated: now they are - stored in `v3bw` directory, named `YYmmdd_HHMMSS.v3bw`, and previously -generated ones are kept. A `latest.v3bw` symlink is updated. (GH#179 GHPR#190) -- Code refactoring in the v3bw classes and generation area -- Replace v3bw-into-xy bash script with python script to handle a more complex - v3bw file format (GH#182) - -## [0.4.1] - 2018-06-14 - -### Changed - -- If the relay to measure is an exit, put it in the exit position and choose a - non-exit to help. Previously the relay to measure would always be the first -hop. (GH#181) -- Try harder to find a relay to help measure the target relay with two changes. - Essentially: (1) Instead of only picking from relays that are 1.25 - 2.00 -times faster than it by consensus weight, try (in order) to find a relay that -is at least 2.00, 1.75, 1.50, 1.25, or 1.00 times as fast. If that fails, -instead of giving up, (2) pick the fastest relay in the network instead of -giving up. This compliments the previous change about measuring target exits in -the exit position. - -### Fixed - -- Exception that causes sbws to fall back to one measurement thread. We first - tried fixing something in this area with `88fae60bc` but neglected to -remember that `.join()` wants only string arguments and can't handle a `None`. -So fix that. -- Exception when failing to get a relay's `ed25519_master_key` from Tor and - trying to do `.rstrip()` on a None. -- `earliest_bandwidth` being the newest bw not the oldest (thanks juga0) -- `node_id` was missing the character "$" at the beginning - diff --git a/docs/source/CHANGELOG.md b/docs/source/CHANGELOG.md deleted file mode 120000 index 699cc9e..0000000 --- a/docs/source/CHANGELOG.md +++ /dev/null @@ -1 +0,0 @@ -../../CHANGELOG.md \ No newline at end of file diff --git a/docs/source/CHANGELOG.rst b/docs/source/CHANGELOG.rst new file mode 120000 index 0000000..bfa394d --- /dev/null +++ b/docs/source/CHANGELOG.rst @@ -0,0 +1 @@ +../../CHANGELOG.rst \ No newline at end of file

1 0

[sbws/master] chg: Convert changelog to rst
by juga＠torproject.org 05 Mar '19

05 Mar '19

commit 55851157108d5bb6853c21256a4b4a8e3eae4b15 Author: juga0 <juga(a)riseup.net> Date: Sat Nov 24 14:21:30 2018 +0000 chg: Convert changelog to rst Leave CHANGELOG.md until there's an actual new release, in case the unreleased changes are lost. Once CHANGELOG.md is removed, update the symlinks. --- CHANGELOG.rst | 331 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 331 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst new file mode 100644 index 0000000..a5b8514 --- /dev/null +++ b/CHANGELOG.rst @@ -0,0 +1,331 @@ +Changelog +========= + +All notable changes to this project will be documented in this file. + +The format is based on `Keep a +Changelog <http://keepachangelog.com/en/1.0.0/>`__ and this project +adheres to `Semantic Versioning <http://semver.org/spec/v2.0.0.html>`__. + + +v1.0.2 (2018-11-10) +------------------- + +Fixed +~~~~~ + +- Update bandwidth file specification version in the ``generator`` + (#28366). +- Use 5 "=" characters as terminator in the bandwidth files (#28379) + +Changed +~~~~~~~ + +- Include the headers about eligible relays in all the bandwidth files, + not only in the ones that does not have enough eligible relays + (#28365). + +v1.0.1 (2018-11-01) +------------------- + +Changed +~~~~~~~ + +- Change default directories when sbws is run from a system service + (#28268). + +v1.0.0 (2018-10-29) +------------------- + +**Important changes**: + +- ``generate`` includes extra statistics header lines when the number + of eligible relays to include is less than the 60% of the network. It + does not include the relays' lines. +- Speed up ``scanner`` by disabling RTT measurements and waiting for + measurement threads before prioritizing again the list of relays to + measure. + +Fixed +~~~~~ + +- Update python minimal version in setup (#28043) +- Catch unhandled exception when we fail to resolve a domain name + (#28141) +- Bandwidth filtered is the maximum between the bandwidth measurements + and their mean, not the minimum (#28215) +- Stop measuring the same relay by two threads(#28061) + +Changed +~~~~~~~ + +- Move ``examples/`` to ``docs/`` (#28040) +- Number of results comparison and number of results away from each + other are incorrect (#28041) +- Stop removing results that are not away from some other X secs + (#28103) +- Use secs-away when provided instead of data\_period (#28105) +- Disable measuring RTTs (#28159) +- Rename bandwidth file keyvalues (#28197) + +Added +----- + +- Write bw file only when the percentage of measured relays is bigger + than 60% (#28062) +- When the percentage of measured relays is less than the 60%, do not + include the relays in the bandwidth file and instead include some + statistics in the header (#28076) +- When the percentage of measured relays is less than the 60% and it + was more before, warn about it (#28155) +- When the difference between the total consensus bandwidth and the + total in the bandwidth lines is larger than 50%, warn (#28216) +- Add documentation about how the bandwidth measurements are selected + and scaled before writing them to the Bandwidth File (#27692) + +v0.8.0 (2018-10-08) +------------------- + +**Important changes**: + +- Implement Torflow scaling/aggregation to be able to substitute + Torflow with sbws without affecting the bandwidth files results. +- Change stem dependency to 1.7.0, which removes the need for + ``dependency_links`` +- Update and cleanup documentation + +Added +~~~~~ + +- Add system physical requirements section to INSTALL (#26937) +- Warn when there is not enough disk space (#26937) +- Implement Torflow scaling (#27108) +- Create methods to easy graph generation and obtain statistics to + compare with current torflow results.(#27688) +- Implement rounding bw in bandwidth files to 2 insignificant + digits(#27337) +- Filter results in order to include relays in the bandwidth file + that:(#27338) +- have at least two measured bandwidths +- the measured bandwidths are within 24 hours of each other +- have at least two descriptor observed bandwidths +- the descriptor observed bandwidths are within 24 hours of each other + +Fixed +~~~~~ + +- Broken environment variable in default sbws config. To use envvar + $FOO, write $$FOO in the config. +- Stop using directory as argument in integration tests (#27342) +- Fix typo getting configuration option to allow logging to file + (#27960) +- Set int type to new arguments that otherwise would be string (#27918) +- Stop printing arguments default values, since they are printed by + default (#27916) +- Use dash instead of underscore in new cli argument names (#27917) + +Changed +~~~~~~~ + +- sbws install doc is confusing (#27341) +- Include system and Python dependencies in ``INSTALL``. +- Include dependencies for docs and tests in ``INSTALL``. +- Point to ``DEPLOY`` to run sbws. +- Remove obsolete sections in ``INSTALL`` +- Simplify ``DEPLOY``, reuse terms in the ``glossary``. +- Remove obsolete ``sbws init`` from ``DEPLOY``. +- Point to config documentation. +- Add, unify and reuse terms in ``glossary``. +- refactor v3bwfile (#27386): move scaling method inside class +- use custom ``install_command`` to test installation commands while + ``dependency_links`` is needed until #26914 is fixed. (#27704) +- documentation cleanup (#27773) +- split, merge, simplify, extend, reorganize sections and files +- generate scales as Torflow by default (#27976) +- Replace stem ``dependency_links`` by stem 1.7.0 (#27705). This also + eliminates the need for custom ``install_command`` in tox. + +v0.7.0 (2018-08-09) +------------------- + +**Important changes**: + +- ``cleanup/stale_days`` is renamed to + ``cleanup/data_files_compress_after_days`` +- ``cleanup/rotten_days`` is renamed to + ``cleanup/data_files_delete_after_days`` +- sbws now takes as an argument the path to a config file (which + contains ``sbws_home``) instead of ``sbws_home`` (which contains the + path to a config file) + +Added +~~~~~ + +- Log line on start up with sbws version, platform info, and library + versions (trac#26751) +- Manual pages (#26926) + +Fixed +~~~~~ + +- Stop deleting the latest.v3bw symlink. Instead, do an atomic rename. + (#26740) +- State file for storing the last time ``sbws scanner`` was started, + and able to be used for storing many other types of state in the + future. (GH#166) +- Log files weren't rotating. Now they are. (#26881) + +Changed +~~~~~~~ + +- Remove test data v3bw file and generate it from the same test. + (#26736) +- Stop using food terms for cleanup-related config options +- Cleanup command now cleans up old v3bw files too (#26701) +- Make sbws more compatible with system packages: (#26862) +- Allow a configuration file argument +- Remove directory argument +- Create minimal user configuration when running +- Do not require to run a command to initialize +- Initialize directories when running +- Do not require configuration file inside directories specified by the + configuration + +v0.6.0 (2018-07-11) +------------------- + +**Important changes**: + +- The way users configure logging has changed. No longer are most users + expected to be familiar with how to configure python's standard + logging library with a config file. Instead we've abstracted out the + setting of log level, format, and destinations to make these settings + more accessible to users. Expert users familiar with `the logging + config file + format <https://docs.python.org/3/library/logging.config.html#logging-config-filefo…>`__ + can still make tweaks. + +Summary of changes: + +- Make logging configuration easier for the user. +- Add UML diagrams to documentation. They can be found in + docs/source/images/ and regenerated with ``make umlsvg`` in docs/. + +Added +~~~~~ + +- UML diagrams to documentation. In docs/ run ``make umlsvg`` to + rebuild them. Requires graphviz to be installed.(GHPR#226) +- Add metadata to setup.py, useful for source/binary distributions. +- Add possibility to log to system log. (#26683) +- Add option to cleanup v3bw files. (#26701) + +Fixed +~~~~~ + +- Measure relays that have both Exit and BadExit as non-exits, which is + how clients would use them. (GH#217) +- Could not init sbws because of a catch-22 related to logging + configuration. Overhaul how logging is configured. (GH#186 GHPR#224) +- Call write method of V3BWFile class from the object instance. + (#26671) +- Stop calculating median on empty list .(#26666) + +Changed +~~~~~~~ + +- Remove is\_controller\_ok. Instead catch possible controller + exceptions and log them + +Removed +~~~~~~~ + +- Two parsing/plotting scripts in scripts/tools/ that can now be found + at https://github.com/pastly/v3bw-tools + +v0.5.0 (2018-06-26) +------------------- + +**Important changes**: + +- Result format changed, causing a version bump to 4. Updating sbws to + 0.5.0 will cause it to ignore results with version less than 4. + +Summary of changes: + +- Keep previously-generated v3bw files +- Allow a relay to limit its weight based on + RelayBandwidthRate/MaxAdvertisedBandwidth +- 1 CPU usage optimization +- 1 memory usage optimization + +Added +~~~~~ + +- Use a relay's {,Relay}BandwidthRate/MaxAdvertisedBandwidth as an + upper bound on the measurements we make for it. (GH#155) +- Ability to only consider results for a given relay valid if they came + from when that relay is using its most recent known IP address. + Thanks Juga. (GH#154 GHPR#199) +- Maintenance script to help us find functions that are (probably) no + longer being called. +- Integration test(s) for RelayPrioritizer (GHPR#206) +- Git/GitHub usage guidelines to CONTRIBUTING document (GH#208 + GHPR#215) + +Fixed +~~~~~ + +- Make relay priority calculations take only ~5% of the time they used + to (3s vs 60s) by using sets instead of lists when selecting + non-Authority relays. (GH#204) +- Make relay list refreshing take much less time by not allowing worker + threads to dogpile on the CPU. Before they would all start requesting + descriptors from Tor at roughly the same time, causing us to overload + our CPU core and make the process take unnecessarily long. Now we let + one thread do the work so it can peg the CPU on its own and get the + refresh done ASAP. (GH#205) +- Catch a JSON decode exception on malformed results so sbws can + continue gracefully (GH#210 GHPR#212) + +Changed +~~~~~~~ + +- Change the path where the Bandwidth List files are generated: now + they are stored in ``v3bw`` directory, named ``YYmmdd_HHMMSS.v3bw``, + and previously generated ones are kept. A ``latest.v3bw`` symlink is + updated. (GH#179 GHPR#190) +- Code refactoring in the v3bw classes and generation area +- Replace v3bw-into-xy bash script with python script to handle a more + complex v3bw file format (GH#182) + +v0.4.1 (2018-06-14) +------------------- + +Changed +~~~~~~~ + +- If the relay to measure is an exit, put it in the exit position and + choose a non-exit to help. Previously the relay to measure would + always be the first hop. (GH#181) +- Try harder to find a relay to help measure the target relay with two + changes. Essentially: (1) Instead of only picking from relays that + are 1.25 - 2.00 times faster than it by consensus weight, try (in + order) to find a relay that is at least 2.00, 1.75, 1.50, 1.25, or + v1.00 times as fast. If that fails, instead of giving up, (2) pick the + fastest relay in the network instead of giving up. This compliments + the previous change about measuring target exits in the exit + position. + +Fixed +~~~~~ + +- Exception that causes sbws to fall back to one measurement thread. We + first tried fixing something in this area with ``88fae60bc`` but + neglected to remember that ``.join()`` wants only string arguments + and can't handle a ``None``. So fix that. +- Exception when failing to get a relay's ``ed25519_master_key`` from + Tor and trying to do ``.rstrip()`` on a None. +- ``earliest_bandwidth`` being the newest bw not the oldest (thanks + juga0) +- ``node_id`` was missing the character "$" at the beginning

1 0

[sbws/master] chg: Changelog: Add v1.0.3 section
by juga＠torproject.org 05 Mar '19

05 Mar '19

commit f90f3dd291fe6874b15b6485bebc682efad8755b Author: juga0 <juga(a)riseup.net> Date: Tue Mar 5 08:38:10 2019 +0000 chg: Changelog: Add v1.0.3 section --- CHANGELOG.rst | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 85 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a5b8514..b9cda32 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,6 +8,90 @@ Changelog <http://keepachangelog.com/en/1.0.0/>`__ and this project adheres to `Semantic Versioning <http://semver.org/spec/v2.0.0.html>`__. +v1.0.3 (2019-02-28) +------------------- + +Fixed +~~~~~~ + +- scanner: check that ResultDump queue is not full + Fixes bug #28866. Bugfix v0.1.0. +- config: set stdout log level to cli argument. Closes: #29199 +- cleanup: Use getpath to get configuration paths. Bugfix v0.7.0. +- destination: stop running twice usability tests. + Fixes bug #28897. Bugfix v0.3.0 +- globals, stem: explain where torrc options are. + Fixes bug #28646. Bugfix v0.4.0 +- stem: disable pad connections. Fixes bug 28692. Bugfix v0.4.0 +- generate: Load all results, including error ones. + Closes #29568. Bugfix v0.4.0 (line introduced in v0.1.0). +- relayprioritizer: Stop prioritizing relays that tend to fail. + Fixes bug #28868. Bugfix v0.1.0 +- circuitbuilder: Stop building the circuit 3 times. + Fixes bug #29295. Bugfix v0.1.0. +- docs: add verify option to man and example. + Closes bug #28788. Bugfix v0.4.0. +- CI: run scanner using the test network. Fixes bug #28933. Bugfix v0.1.0. +- scanner: catch SIGINT in the main loop. Fixes bug #28869. Bugfix v0.1.0. +- Stop including tests network as binary blob. Fixes bug #28590. Bugfix v0.4.0. +- relaylist: remove assertions that fail measurement. + Closes #28870. Bugfix v0.4.0 +- config: Use configuration provided as argument. + Fixes bug #28724. Bugfix v0.7.0. +- stem: parse torrc options that are only a key. + Fixes bug #28715. Bugfix v0.1.1 +- stem: Stop merging multiple torrc options with the same name. + Fixes bug #28738. Bugfix v0.1.1 +- docs: add note about syslog when running systemd. + Closes bug #28761. Bugfix v0.6.0 +- CI: include deb.torproject.org key. + Closes #28922. Bugfix v1.0.3-dev0 +- config: stop allowing http servers without tls. + Fixes bug #28789. Bugfix v0.2.0. +- Make info level logs more clear and consistent. + Closes bug #28736. Bugfix v0.3.0. +- CI: check broken links in the docs. Closes #28670. +- docs: add scanner and destination requirements. + Closes bug #28647. Bugfix v0.4.0 +- generate: use round_digs variable name in methods. + Closes bug #28602. Bugfix 1.0.3-dev0 +- docs: Change old broken links in the documentation. Closes #28662. +- docs: replace http by https in links. Closes #28661. +- Fix git repository link. Fixes bug #28762. Bugfix v1.0.0. +- docs: add example destination in DEPLOY. Closes #28649. +- docs: Change links to be interpreted by ReST. Closes #28648. +- Force rtfd.io to install the package. Closes bug #28601. +- config: continue when the file is not found. Closes: #28550. +- Stop resolving domains locally and check same flags for the 2nd hop. + Closes bug #28458, #28471. Bugfix 1.0.4. +- Limit the relays' bandwidth to their consensus bandwidth. Closes #28598. +- globals: add torrc logging options. Closes #28645. Bugfix v0.2.0. +- Limit bandwidth to the relay MaxAdvertisedBandwidth + Fixes bug #28588. Bugfix 0.8.0. +- Exclude results, then check for the minimum number. Closes bug 28572. +- Make sbws round to 3 significant figures in torflow rounding mode. + Bugfix on 27337 in sbws 1.0. Part of 28442. + +Changed +~~~~~~~~ + +- tests: remove unused testnets. Fixes bug #29046. Bugfix v0.4.0. +- scanner, destination: Log all possible exceptions. +- docs: Update/improve documentation on how the scanner/generator work. + Closes: #29149 +- Requests: Change make_session to use the TimedSession. +- CI: change to Ubuntu Xenial. +- docs: stop editing changelog on every bug/ticket. Closes ticket #28572. +- Change sbws scaling method to torflow. Closes: #28446. +- Round bandwidths to 2 significant digits by default. + Implements part of proposal 276. Implements 28451. + +Added +~~~~~~ + +- Send scanner metadata as part of every HTTP request. Closes: #28741 +- scanner: log backtrace when not progressing. Closes: 28932 + v1.0.2 (2018-11-10) ------------------- @@ -181,7 +265,7 @@ Changed - Remove test data v3bw file and generate it from the same test. (#26736) - Stop using food terms for cleanup-related config options -- Cleanup command now cleans up old v3bw files too (#26701) +- cleanup command now cleans up old v3bw files too (#26701) - Make sbws more compatible with system packages: (#26862) - Allow a configuration file argument - Remove directory argument

1 0

[sbws/master] Merge branch 'ticket28585'
by juga＠torproject.org 05 Mar '19

05 Mar '19

commit 295e82acbbaea9e7a41176bc63d7d4d7b53abb50 Merge: 315215d fd1ab12 Author: juga0 <juga(a)riseup.net> Date: Tue Mar 5 18:46:01 2019 +0000 Merge branch 'ticket28585' CHANGELOG.md | 300 --------------------------------- CHANGELOG.rst | 415 ++++++++++++++++++++++++++++++++++++++++++++++ docs/source/CHANGELOG.md | 1 - docs/source/CHANGELOG.rst | 1 + 4 files changed, 416 insertions(+), 301 deletions(-)

1 0

[tor/release-0.4.0] Merge branch 'maint-0.4.0' into release-0.4.0
by nickm＠torproject.org 05 Mar '19

05 Mar '19

commit ea1de844323e9eecdce833e24bcf7fda130f3b3e Merge: 5529449bb a999cb43d Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Mar 5 13:00:05 2019 -0500 Merge branch 'maint-0.4.0' into release-0.4.0 changes/ticket29631 | 4 ++++ src/core/or/protover.h | 22 +++++++++++----------- src/rust/protover/ffi.rs | 1 + 3 files changed, 16 insertions(+), 11 deletions(-)

1 0

[tor/master] protover: Add missing Padding to translate_to_rust
by nickm＠torproject.org 05 Mar '19

05 Mar '19

commit a999cb43df3dc4e64820347486a2d861636bfc2e Author: David Goulet <dgoulet(a)torproject.org> Date: Fri Mar 1 13:50:00 2019 -0500 protover: Add missing Padding to translate_to_rust This commit also explicitly set the value of the PRT enum so we can match/pin the C enum values to the Rust one in protover/ffi.rs. Fixes #29631 Signed-off-by: David Goulet <dgoulet(a)torproject.org> --- changes/ticket29631 | 4 ++++ src/core/or/protover.h | 22 +++++++++++----------- src/rust/protover/ffi.rs | 1 + 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/changes/ticket29631 b/changes/ticket29631 new file mode 100644 index 000000000..9fc194ba9 --- /dev/null +++ b/changes/ticket29631 @@ -0,0 +1,4 @@ + o Minor bugfixes (Rust, protover): + - The Rust implementation of protover was missing the "Padding" value in + the translate function from C to Rust. Fixes bug 29631; bugfix on + 0.4.0.1-alpha. diff --git a/src/core/or/protover.h b/src/core/or/protover.h index 27106d4be..567b94a16 100644 --- a/src/core/or/protover.h +++ b/src/core/or/protover.h @@ -33,17 +33,17 @@ struct smartlist_t; /// C_RUST_COUPLED: src/rust/protover/ffi.rs `translate_to_rust` /// C_RUST_COUPLED: src/rust/protover/protover.rs `Proto` typedef enum protocol_type_t { - PRT_LINK, - PRT_LINKAUTH, - PRT_RELAY, - PRT_DIRCACHE, - PRT_HSDIR, - PRT_HSINTRO, - PRT_HSREND, - PRT_DESC, - PRT_MICRODESC, - PRT_CONS, - PRT_PADDING, + PRT_LINK = 0, + PRT_LINKAUTH = 1, + PRT_RELAY = 2, + PRT_DIRCACHE = 3, + PRT_HSDIR = 4, + PRT_HSINTRO = 5, + PRT_HSREND = 6, + PRT_DESC = 7, + PRT_MICRODESC = 8, + PRT_CONS = 9, + PRT_PADDING = 10, } protocol_type_t; bool protover_contains_long_protocol_names(const char *s); diff --git a/src/rust/protover/ffi.rs b/src/rust/protover/ffi.rs index 6ee63adb1..066b08edd 100644 --- a/src/rust/protover/ffi.rs +++ b/src/rust/protover/ffi.rs @@ -30,6 +30,7 @@ fn translate_to_rust(c_proto: uint32_t) -> Result<Protocol, ProtoverError> { 7 => Ok(Protocol::Desc), 8 => Ok(Protocol::Microdesc), 9 => Ok(Protocol::Cons), + 10 => Ok(Protocol::Padding), _ => Err(ProtoverError::UnknownProtocol), } }

1 0