June 2018 - tor-commits - lists.torproject.org

[tor-browser-build/master] Bug 25837: Integrate fxc2 for Windows Tor Browser builds
by gk＠torproject.org 14 Jun '18

14 Jun '18

commit 57693e01837f0fc7c60bb056e5b752f5f67a987b Author: Sukhbir Singh <sukhbir(a)torproject.org> Date: Thu Jun 14 13:21:18 2018 +0000 Bug 25837: Integrate fxc2 for Windows Tor Browser builds This commit adds support for building and bundling fxc2 which is required for Windows cross-compilation on Linux. --- projects/firefox/build | 18 ++++++++++++++++++ projects/firefox/config | 8 ++++++++ projects/fxc2/build | 30 ++++++++++++++++++++++++++++++ projects/fxc2/config | 14 ++++++++++++++ 4 files changed, 70 insertions(+) diff --git a/projects/firefox/build b/projects/firefox/build index 0fd3da2..81032c6 100644 --- a/projects/firefox/build +++ b/projects/firefox/build @@ -34,6 +34,23 @@ EOF export PATH="$mingwdir/helpers:$PATH" [% END -%] +[% IF c("var/windows") %] + # Unpack fxc2. + mkdir -p /var/tmp/dist + tar -C /var/tmp/dist -xf [% c('input_files_by_name/fxc2') %] + fxcdir=/var/tmp/dist/fxc2/bin + cp $mingwdir/[% c("arch") %]-w64-mingw32/bin/libwinpthread-1.dll $fxcdir + export PATH="$fxcdir:$PATH" + # fxc2 requires Wine. + [% IF c("var/windows-x86_64") %] + export WINEARCH=win64 + [% END %] + export HOME=/var/tmp/home + mkdir -p $HOME + WINEROOT=$HOME/.wine/drive_c + wine wineboot -i +[% END -%] + [% IF c("var/linux") %] mkdir -p /var/tmp/dist tar -C /var/tmp/dist -xf $rootdir/[% c('input_files_by_name/binutils') %] @@ -161,6 +178,7 @@ rm -f js/src/configure cp -a obj-*/dist/firefox/* $distdir/Browser/ cp -a $rootdir/msvcr100.dll $distdir/Browser cp -a $gcclibs/libssp-0.dll $distdir/Browser + cp -a $fxcdir/d3dcompiler_47.dll $distdir/Browser [% END %] # Make MAR-based update tools available for use during the bundle phase. diff --git a/projects/firefox/config b/projects/firefox/config index 6cb17c7..5bde93d 100644 --- a/projects/firefox/config +++ b/projects/firefox/config @@ -65,6 +65,11 @@ targets: arch_deps: - rsync + windows: + var: + arch_deps: + - wine + windows-i686: var: martools_filename: mar-tools-win32.zip @@ -107,3 +112,6 @@ input_files: - project: llvm name: llvm enable: '[% c("var/linux") %]' + - project: fxc2 + name: fxc2 + enable: '[% c("var/windows") %]' diff --git a/projects/fxc2/build b/projects/fxc2/build new file mode 100644 index 0000000..46a7db8 --- /dev/null +++ b/projects/fxc2/build @@ -0,0 +1,30 @@ +#!/bin/bash +[% c("var/set_default_env") -%] +distdir="/var/tmp/dist/[% project %]" + +[% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %] + +mkdir -p /var/tmp/build +tar -C /var/tmp/build -xf [% project %]-[% c("version") %].tar.gz +cd /var/tmp/build/[% project %]-[% c("version") %] + +[% IF c("var/windows-i686") -%] + make x86 +[% ELSE -%] + make x64 +[% END -%] + +mkdir -p $distdir/bin/ +cp fxc2.exe $distdir/bin/ +[% IF c("var/windows-i686") -%] + cp dll/d3dcompiler_47_32.dll $distdir/bin/d3dcompiler_47.dll +[% ELSE -%] + cp dll/d3dcompiler_47.dll $distdir/bin/ +[% END -%] + +cd /var/tmp/dist + +[% c('tar', { + tar_src => [ project ], + tar_args => '-czf ' _ dest_dir _ '/' _ c('filename'), + }) %] diff --git a/projects/fxc2/config b/projects/fxc2/config new file mode 100644 index 0000000..aac9158 --- /dev/null +++ b/projects/fxc2/config @@ -0,0 +1,14 @@ +# vim: filetype=yaml sw=2 +version: '[% c("abbrev") %]' +git_url: https://github.com/azadi/fxc2.git +git_hash: 079f5c7b9f6ea0b7e5504bb53abf8709cc92835d +filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz' + +var: + container: + use_container: 1 + +input_files: + - project: container-image + - name: '[% c("var/compiler") %]' + project: '[% c("var/compiler") %]'

1 0

[tor-browser-build/master] Bug 26329: Fix Rust cross-compilation for 32bit Windows
by boklm＠torproject.org 14 Jun '18

14 Jun '18

commit 3751474686ba0d2cc7b8a4086114332cbc36e29c Author: Georg Koppen <gk(a)torproject.org> Date: Thu Jun 14 11:26:47 2018 +0000 Bug 26329: Fix Rust cross-compilation for 32bit Windows --- projects/rust/build | 4 +- projects/rust/config | 2 +- projects/rust/panic-abort.patch | 36 ---------- projects/rust/unwind.patch | 154 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 157 insertions(+), 39 deletions(-) diff --git a/projects/rust/build b/projects/rust/build index 74ce74a..af0f62b 100644 --- a/projects/rust/build +++ b/projects/rust/build @@ -65,8 +65,8 @@ cd /var/tmp/build/rustc-[% c('version') %]-src # toolchains on Linux, targeting Windows 32bit, use SjLj unwinding. # See: https://github.com/rust-lang/rust/issues/12859 for discussion about # that and https://github.com/rust-lang/rust/pull/49633 for a newer attempt to - # fix this problem. - patch -p1 < $rootdir/panic-abort.patch + # fix this problem. We backport and apply the patch from neersighted. + patch -p1 < $rootdir/unwind.patch [% END %] mkdir build diff --git a/projects/rust/config b/projects/rust/config index 9d3030d..771b50b 100644 --- a/projects/rust/config +++ b/projects/rust/config @@ -73,5 +73,5 @@ input_files: gpg_keyring: rust.gpg - filename: binaryen.patch enable: '[% c("var/linux") %]' - - filename: panic-abort.patch + - filename: unwind.patch enable: '[% c("var/windows-i686") %]' diff --git a/projects/rust/panic-abort.patch b/projects/rust/panic-abort.patch deleted file mode 100644 index 1711ff7..0000000 --- a/projects/rust/panic-abort.patch +++ /dev/null @@ -1,36 +0,0 @@ -From 2fe471643721f3967f1bdf28907b0a7247fdb705 Mon Sep 17 00:00:00 2001 -From: Georg Koppen <gk(a)torproject.org> -Date: Thu, 26 Apr 2018 13:18:27 +0000 -Subject: [PATCH] Avoid cross-compilation breakage due to unwinding - incompatibilities - - -diff --git a/src/bootstrap/bin/rustc.rs b/src/bootstrap/bin/rustc.rs -index 55d104b182..85330f973b 100644 ---- a/src/bootstrap/bin/rustc.rs -+++ b/src/bootstrap/bin/rustc.rs -@@ -143,7 +143,8 @@ fn main() { - // workaround undefined references to `rust_eh_unwind_resume` generated - // otherwise, see issue https://github.com/rust-lang/rust/issues/43095. - if crate_name == "panic_abort" || -- crate_name == "compiler_builtins" && stage != "0" { -+ crate_name == "compiler_builtins" && stage != "0" || -+ target == "i686-pc-windows-gnu" { - cmd.arg("-C").arg("panic=abort"); - } - -diff --git a/src/libtest/lib.rs b/src/libtest/lib.rs -index 9ea5f39b71..bb689120af 100644 ---- a/src/libtest/lib.rs -+++ b/src/libtest/lib.rs -@@ -45,7 +45,6 @@ extern crate getopts; - extern crate term; - #[cfg(any(unix, target_os = "cloudabi"))] - extern crate libc; --extern crate panic_unwind; - - pub use self::TestFn::*; - pub use self::ColorConfig::*; --- -2.17.0 - diff --git a/projects/rust/unwind.patch b/projects/rust/unwind.patch new file mode 100644 index 0000000..8e2e0fc --- /dev/null +++ b/projects/rust/unwind.patch @@ -0,0 +1,154 @@ +From f967c575e502e507ed51b7ed8d1e2a08525e6ae6 Mon Sep 17 00:00:00 2001 +From: Bjorn Neergaard <bjorn(a)neersighted.com> +Date: Tue, 3 Apr 2018 19:01:07 -0600 +Subject: [PATCH] Fix cross-compiling i686-pc-windows-gnu from Linux + +This is still very rough and serves as a proof-of-concept for fixing +Linux -> 32-bit MinGW cross compilation workflow. Currently, clang and +GCC's MinGW targets both only support DW2 (DWARF) or SJLJ (Set Jump Long +Jump) unwinding on 32-bit Windows. + +The default for GCC (and the way it is shipped on every major distro) is +to use SJLJ on Windows, as DWARF cannot traverse non-DWARF frames. This +would work fine, except for the fact that libgcc (our C runtime on the +MinGW platform) exports symbols under a different name when configured +to use SJLJ-style unwinding, and uses a preprocessor macro internally to +alias them. + +Because of this, we have to detect this scenario and link to the correct +symbols ourselves. Linking has been tested with a full bootstrap on both +x86_64-unknown-linux-gnu and i686-pc-windows-gnu, as well as +cross-compilation of some of my own projects. + +Obviously, the detection is a bit unrefined. Right now we +unconditionally use SJLJ when compiling Linux -> MinGW. I'd like to add +feature detection using compiler build flags or autotools-style +compilation and object analysis. Input on the best way to proceed here +is welcome. + +Also, currently there is copy-pasted/duplicated code in libunwind. +Ideally, this could be reduced, but this would likely require a +rethinking of how iOS is special-cased above, to avoid further +duplication. Input on how to best structure this file is requested. + +diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs +index c85b04ddc0..2f58f0d9fa 100644 +--- a/src/bootstrap/compile.rs ++++ b/src/bootstrap/compile.rs +@@ -145,6 +145,11 @@ pub fn std_cargo(build: &Build, + cargo.env("MACOSX_DEPLOYMENT_TARGET", target); + } + ++ // FIXME: Temporary detection of SJLJ MinGW compilers. ++ if build.build.contains("linux") && target == "i686-pc-windows-gnu" { ++ features.push_str(" sjlj_eh"); ++ } ++ + // When doing a local rebuild we tell cargo that we're stage1 rather than + // stage0. This works fine if the local rust and being-built rust have the + // same view of what the default allocator is, but fails otherwise. Since +diff --git a/src/libstd/Cargo.toml b/src/libstd/Cargo.toml +index c1fe4a89d6..6c8cb330d8 100644 +--- a/src/libstd/Cargo.toml ++++ b/src/libstd/Cargo.toml +@@ -49,3 +49,4 @@ force_alloc_system = [] + panic-unwind = ["panic_unwind"] + profiler = ["profiler_builtins"] + wasm_syscall = [] ++sjlj_eh = ["unwind/sjlj_eh"] +diff --git a/src/libunwind/Cargo.toml b/src/libunwind/Cargo.toml +index fbd9789d2f..5e319c67fa 100644 +--- a/src/libunwind/Cargo.toml ++++ b/src/libunwind/Cargo.toml +@@ -14,3 +14,6 @@ doc = false + [dependencies] + core = { path = "../libcore" } + libc = { path = "../rustc/libc_shim" } ++ ++[features] ++sjlj_eh = [] +diff --git a/src/libunwind/libunwind.rs b/src/libunwind/libunwind.rs +index e6fff7963f..6a57cf7c93 100644 +--- a/src/libunwind/libunwind.rs ++++ b/src/libunwind/libunwind.rs +@@ -10,11 +10,6 @@ + + #![allow(bad_style)] + +-macro_rules! cfg_if { +- ( $( if #[cfg( $meta:meta )] { $($it1:item)* } else { $($it2:item)* } )* ) => +- ( $( $( #[cfg($meta)] $it1)* $( #[cfg(not($meta))] $it2)* )* ) +-} +- + use libc::{c_int, c_void, uintptr_t}; + + #[repr(C)] +@@ -84,7 +79,6 @@ pub type _Unwind_Exception_Cleanup_Fn = extern "C" fn(unwind_code: _Unwind_Reaso + exception: *mut _Unwind_Exception); + extern "C" { + #[unwind] +- pub fn _Unwind_Resume(exception: *mut _Unwind_Exception) -> !; + pub fn _Unwind_DeleteException(exception: *mut _Unwind_Exception); + pub fn _Unwind_GetLanguageSpecificData(ctx: *mut _Unwind_Context) -> *mut c_void; + pub fn _Unwind_GetRegionStart(ctx: *mut _Unwind_Context) -> _Unwind_Ptr; +@@ -216,26 +210,49 @@ if #[cfg(all(any(target_os = "ios", not(target_arch = "arm"))))] { + pc + } + } ++} // cfg_if! + +-if #[cfg(not(all(target_os = "ios", target_arch = "arm")))] { +- // Not 32-bit iOS ++cfg_if! { ++if #[cfg(all(target_os = "ios", target_arch = "arm"))] { ++ // 32-bit iOS uses SjLj and does not provide _Unwind_Backtrace() + extern "C" { + #[unwind] +- pub fn _Unwind_RaiseException(exception: *mut _Unwind_Exception) -> _Unwind_Reason_Code; +- pub fn _Unwind_Backtrace(trace: _Unwind_Trace_Fn, +- trace_argument: *mut c_void) +- -> _Unwind_Reason_Code; ++ pub fn _Unwind_Resume(exception: *mut _Unwind_Exception) -> !; ++ pub fn _Unwind_SjLj_RaiseException(e: *mut _Unwind_Exception) -> _Unwind_Reason_Code; + } +-} else { +- // 32-bit iOS uses SjLj and does not provide _Unwind_Backtrace() ++ ++ #[inline] ++ pub unsafe fn _Unwind_RaiseException(exc: *mut _Unwind_Exception) -> _Unwind_Reason_Code { ++ _Unwind_SjLj_RaiseException(exc) ++ } ++ ++} else if #[cfg(feature = "sjlj_eh")] { + extern "C" { + #[unwind] ++ pub fn _Unwind_SjLj_Resume(e: *mut _Unwind_Exception) -> !; + pub fn _Unwind_SjLj_RaiseException(e: *mut _Unwind_Exception) -> _Unwind_Reason_Code; ++ pub fn _Unwind_Backtrace(trace: _Unwind_Trace_Fn, ++ trace_argument: *mut c_void) ++ -> _Unwind_Reason_Code; ++ } ++ ++ #[inline] ++ pub unsafe fn _Unwind_Resume(exc: *mut _Unwind_Exception) -> ! { ++ _Unwind_SjLj_Resume(exc) + } + + #[inline] + pub unsafe fn _Unwind_RaiseException(exc: *mut _Unwind_Exception) -> _Unwind_Reason_Code { + _Unwind_SjLj_RaiseException(exc) + } ++} else { ++ extern "C" { ++ #[unwind] ++ pub fn _Unwind_Resume(exception: *mut _Unwind_Exception) -> !; ++ pub fn _Unwind_RaiseException(exception: *mut _Unwind_Exception) -> _Unwind_Reason_Code; ++ pub fn _Unwind_Backtrace(trace: _Unwind_Trace_Fn, ++ trace_argument: *mut c_void) ++ -> _Unwind_Reason_Code; ++ } + } + } // cfg_if! +-- +2.17.1 +

1 0

[sbws/master] Unify ed25519 master key terms
by pastly＠torproject.org 14 Jun '18

14 Jun '18

commit f33706ce183f6d5370c7755beca8831f1cd512c5 Author: juga0 <juga(a)riseup.net> Date: Mon Jun 11 10:43:17 2018 +0000 Unify ed25519 master key terms to make it less confussing. Note that stem call it ed25519_master_key instead of master_key_ed25519, but dir-spec.txt uses master-key-ed25519, which is the term chosen here. --- sbws/lib/relaylist.py | 4 +++- sbws/lib/resultdump.py | 22 +++++++++++----------- sbws/lib/v3bwfile.py | 4 ++-- tests/integration/lib/test_relaylist.py | 3 ++- tests/unit/core/test_generate.py | 12 ++++++------ tests/unit/lib/data/results.txt | 4 ++-- tests/unit/lib/test_results.py | 10 +++++----- tests/unit/lib/test_v3bwfile.py | 4 ++-- 8 files changed, 33 insertions(+), 30 deletions(-) diff --git a/sbws/lib/relaylist.py b/sbws/lib/relaylist.py index 5d584ea..4f7055c 100644 --- a/sbws/lib/relaylist.py +++ b/sbws/lib/relaylist.py @@ -78,12 +78,14 @@ class Relay: return self._from_ns('address') @property - def ed25519_master_key(self): + def master_key_ed25519(self): """Obtain ed25519 master key of the relay in server descriptors. :returns: str, the ed25519 master key base 64 encoded without trailing '='s. """ + # Even if this key is called master-key-ed25519 in dir-spec.txt, + # it seems that stem parses it as ed25519_master_key return self._from_desc('ed25519_master_key').rstrip('=') diff --git a/sbws/lib/resultdump.py b/sbws/lib/resultdump.py index 4401412..32fd1cd 100644 --- a/sbws/lib/resultdump.py +++ b/sbws/lib/resultdump.py @@ -150,15 +150,15 @@ class Result: class Relay: ''' Implements just enough of a stem RouterStatusEntryV3 for this Result class to be happy ''' - def __init__(self, fingerprint, nickname, address, ed25519_master_key): + def __init__(self, fingerprint, nickname, address, master_key_ed25519): self.fingerprint = fingerprint self.nickname = nickname self.address = address - self.ed25519_master_key = ed25519_master_key + self.master_key_ed25519 = master_key_ed25519 def __init__(self, relay, circ, dest_url, scanner_nick, t=None): self._relay = Result.Relay(relay.fingerprint, relay.nickname, - relay.address, relay.ed25519_master_key) + relay.address, relay.master_key_ed25519) self._circ = circ self._dest_url = dest_url self._scanner = scanner_nick @@ -181,8 +181,8 @@ class Result: return self._relay.address @property - def ed25519_master_key(self): - return self._relay.ed25519_master_key + def master_key_ed25519(self): + return self._relay.master_key_ed25519 @property def circ(self): @@ -209,7 +209,7 @@ class Result: 'fingerprint': self.fingerprint, 'nickname': self.nickname, 'address': self.address, - 'ed25519_master_key': self.ed25519_master_key, + 'master_key_ed25519': self.master_key_ed25519, 'circ': self.circ, 'dest_url': self.dest_url, 'time': self.time, @@ -283,7 +283,7 @@ class ResultError(Result): return ResultError( Result.Relay( d['fingerprint'], d['nickname'], d['address'], - d['ed25519_master_key']), + d['master_key_ed25519']), d['circ'], d['dest_url'], d['scanner'], msg=d['msg'], t=d['time']) @@ -324,7 +324,7 @@ class ResultErrorCircuit(ResultError): return ResultErrorCircuit( Result.Relay( d['fingerprint'], d['nickname'], d['address'], - d['ed25519_master_key']), + d['master_key_ed25519']), d['circ'], d['dest_url'], d['scanner'], msg=d['msg'], t=d['time']) @@ -347,7 +347,7 @@ class ResultErrorStream(ResultError): return ResultErrorStream( Result.Relay( d['fingerprint'], d['nickname'], d['address'], - d['ed25519_master_key']), + d['master_key_ed25519']), d['circ'], d['dest_url'], d['scanner'], msg=d['msg'], t=d['time']) @@ -383,7 +383,7 @@ class ResultErrorAuth(ResultError): return ResultErrorAuth( Result.Relay( d['fingerprint'], d['nickname'], d['address'], - d['ed25519_master_key']), + d['master_key_ed25519']), d['circ'], d['dest_url'], d['scanner'], msg=d['msg'], t=d['time']) @@ -417,7 +417,7 @@ class ResultSuccess(Result): d['rtts'], d['downloads'], Result.Relay( d['fingerprint'], d['nickname'], d['address'], - d['ed25519_master_key']), + d['master_key_ed25519']), d['circ'], d['dest_url'], d['scanner'], t=d['time']) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 9a2dfdd..4091912 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -336,8 +336,8 @@ class V3BWLine(object): bw = cls.bw_from_results(success_results) kwargs = dict() kwargs['nick'] = results[0].nickname - if getattr(results[0], 'ed25519_master_key'): - kwargs['master_key_ed25519'] = results[0].ed25519_master_key + if getattr(results[0], 'master_key_ed25519'): + kwargs['master_key_ed25519'] = results[0].master_key_ed25519 kwargs['rtt'] = cls.rtt_from_results(success_results) kwargs['time'] = cls.last_time_from_results(results) kwargs.update(cls.result_types_from_results(results)) diff --git a/tests/integration/lib/test_relaylist.py b/tests/integration/lib/test_relaylist.py index 8f16ef8..a3fbbf0 100644 --- a/tests/integration/lib/test_relaylist.py +++ b/tests/integration/lib/test_relaylist.py @@ -10,8 +10,9 @@ def test_relay_properties(persistent_launch_tor): assert relay.fingerprint == 'AA45C13025C037F056E734169891878ED0880231' assert 'Authority' in relay.flags assert not relay.exit_policy or not relay.exit_policy.is_exiting_allowed() + print(relay) assert relay.average_bandwidth == 1073741824 assert relay.bandwidth == 0 assert relay.address == '127.10.0.1' - assert relay.ed25519_master_key == \ + assert relay.master_key_ed25519 == \ 'wLglSEw9/DHfpNrlrqjVRSnGLVWfnm0vYxkryH4aT6Q' diff --git a/tests/unit/core/test_generate.py b/tests/unit/core/test_generate.py index fae4fe0..ad1fff2 100644 --- a/tests/unit/core/test_generate.py +++ b/tests/unit/core/test_generate.py @@ -118,7 +118,7 @@ def test_generate_single_success_noscale(dotsbws_success_result, caplog, rtt = median([round(r * 1000) for r in result.rtts]) bw_line = V3BWLine(result.fingerprint, bw, nick=result.nickname, rtt=rtt, time=unixts_to_isodt_str(round(result.time)), - master_key_ed25519=result.ed25519_master_key, + master_key_ed25519=result.master_key_ed25519, success=1, error_circ=0, error_misc=0, error_stream=0) assert stdout_lines[NUM_LINES_HEADER_V110] + '\n' == str(bw_line) @@ -151,7 +151,7 @@ def test_generate_single_success_scale(dotsbws_success_result, parser, rtt = median([round(r * 1000) for r in result.rtts]) bw_line = V3BWLine(result.fingerprint, bw, nick=result.nickname, rtt=rtt, time=unixts_to_isodt_str(round(result.time)), - master_key_ed25519=result.ed25519_master_key, + master_key_ed25519=result.master_key_ed25519, success=1, error_circ=0, error_misc=0, error_stream=0) assert stdout_lines[NUM_LINES_HEADER_V110] + '\n' == str(bw_line) @@ -185,7 +185,7 @@ def test_generate_single_relay_success_noscale( speed = round(median(speeds)) rtt = round(median([round(r * 1000) for r in result.rtts])) bw_line = V3BWLine(result.fingerprint, speed, nick=result.nickname, - rtt=rtt, master_key_ed25519=result.ed25519_master_key, + rtt=rtt, master_key_ed25519=result.master_key_ed25519, time=unixts_to_isodt_str(round(result.time)), success=2, error_circ=0, error_misc=0, error_stream=0) @@ -219,7 +219,7 @@ def test_generate_single_relay_success_scale( rtt = round(median([round(r * 1000) for r in result.rtts])) bw_line = V3BWLine(result.fingerprint, speed, nick=result.nickname, rtt=rtt, - master_key_ed25519=result.ed25519_master_key, + master_key_ed25519=result.master_key_ed25519, time=unixts_to_isodt_str(round(result.time)), success=2, error_circ=0, error_misc=0, error_stream=0) @@ -253,7 +253,7 @@ def test_generate_two_relays_success_noscale( r1_time = unixts_to_isodt_str(round(max([r.time for r in r1_results]))) r1_name = r1_results[0].nickname r1_fingerprint = r1_results[0].fingerprint - r1_ed25519 = r1_results[0].ed25519_master_key + r1_ed25519 = r1_results[0].master_key_ed25519 r1_speeds = [dl['amount'] / dl['duration'] / 1024 for r in r1_results for dl in r.downloads] r1_speed = round(median(r1_speeds)) @@ -269,7 +269,7 @@ def test_generate_two_relays_success_noscale( r2_time = unixts_to_isodt_str(round(max([r.time for r in r2_results]))) r2_name = r2_results[0].nickname r2_fingerprint = r2_results[0].fingerprint - r2_ed25519 = r2_results[0].ed25519_master_key + r2_ed25519 = r2_results[0].master_key_ed25519 r2_speeds = [dl['amount'] / dl['duration'] / 1024 for r in r2_results for dl in r.downloads] r2_speed = round(median(r2_speeds)) diff --git a/tests/unit/lib/data/results.txt b/tests/unit/lib/data/results.txt index ecf0b68..d2005f6 100644 --- a/tests/unit/lib/data/results.txt +++ b/tests/unit/lib/data/results.txt @@ -1,2 +1,2 @@ -{"version": 3, "time": 1523887747, "circ": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"], "type": "success", "rtts": [0.4596822261810303, 0.44872617721557617, 0.4563450813293457, 0.44872212409973145, 0.4561030864715576, 0.4765200614929199, 0.4495084285736084, 0.45711588859558105, 0.45520496368408203, 0.4635589122772217], "fingerprint": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "scanner": "IDidntEditTheSBWSConfig", "downloads": [{"amount": 590009, "duration": 6.1014368534088135}, {"amount": 590009, "duration": 8.391342878341675}, {"amount": 321663, "duration": 7.064587831497192}, {"amount": 321663, "duration": 8.266003131866455}, {"amount": 321663, "duration": 5.779450178146362}], "dest_url": "http://y.z", "nickname": "A", "address": "111.111.111.111", "ed25519_master_key": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s"} -{"version": 3, "time": 1523974147, "circ": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"], "type": "error-stream", "msg": "Something bad happened while measuring bandwidth", "fingerprint": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "scanner": "IDidntEditTheSBWSConfig", "dest_url": "http://y.z", "nickname": "A", "address": "111.111.111.111", "ed25519_master_key": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s"} +{"version": 3, "time": 1523887747, "circ": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"], "type": "success", "rtts": [0.4596822261810303, 0.44872617721557617, 0.4563450813293457, 0.44872212409973145, 0.4561030864715576, 0.4765200614929199, 0.4495084285736084, 0.45711588859558105, 0.45520496368408203, 0.4635589122772217], "fingerprint": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "scanner": "IDidntEditTheSBWSConfig", "downloads": [{"amount": 590009, "duration": 6.1014368534088135}, {"amount": 590009, "duration": 8.391342878341675}, {"amount": 321663, "duration": 7.064587831497192}, {"amount": 321663, "duration": 8.266003131866455}, {"amount": 321663, "duration": 5.779450178146362}], "dest_url": "http://y.z", "nickname": "A", "address": "111.111.111.111", "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s"} +{"version": 3, "time": 1523974147, "circ": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"], "type": "error-stream", "msg": "Something bad happened while measuring bandwidth", "fingerprint": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "scanner": "IDidntEditTheSBWSConfig", "dest_url": "http://y.z", "nickname": "A", "address": "111.111.111.111", "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s"} diff --git a/tests/unit/lib/test_results.py b/tests/unit/lib/test_results.py index 010d505..3d4770e 100644 --- a/tests/unit/lib/test_results.py +++ b/tests/unit/lib/test_results.py @@ -112,7 +112,7 @@ def test_ResultSuccess_from_dict(time_mock): 'nickname': nick, 'address': relay_ip, 'circ': circ, 'dest_url': dest_url, 'scanner': scanner_nick, 'version': RESULT_VERSION, 'type': _ResultType.Success, 'time': t, - 'ed25519_master_key': ed25519, + 'master_key_ed25519': ed25519, } r2 = Result.from_dict(d) assert isinstance(r1, ResultSuccess) @@ -169,7 +169,7 @@ def test_ResultError_from_dict(time_mock): 'nickname': nick, 'address': relay_ip, 'circ': circ, 'dest_url': dest_url, 'scanner': scanner_nick, 'version': RESULT_VERSION, 'type': _ResultType.Error, 'time': t, - 'ed25519_master_key': ed25519, + 'master_key_ed25519': ed25519, } r2 = Result.from_dict(d) assert isinstance(r1, ResultError) @@ -227,7 +227,7 @@ def test_ResultErrorCircuit_from_dict(time_mock): 'nickname': nick, 'address': relay_ip, 'circ': circ, 'dest_url': dest_url, 'scanner': scanner_nick, 'version': RESULT_VERSION, 'type': _ResultType.ErrorCircuit, 'time': t, - 'ed25519_master_key': ed25519, + 'master_key_ed25519': ed25519, } r2 = Result.from_dict(d) assert isinstance(r1, ResultErrorCircuit) @@ -285,7 +285,7 @@ def test_ResultErrorStream_from_dict(time_mock): 'nickname': nick, 'address': relay_ip, 'circ': circ, 'dest_url': dest_url, 'scanner': scanner_nick, 'version': RESULT_VERSION, 'type': _ResultType.ErrorStream, 'time': t, - 'ed25519_master_key': ed25519, + 'master_key_ed25519': ed25519, } r2 = Result.from_dict(d) assert isinstance(r1, ResultErrorStream) @@ -343,7 +343,7 @@ def test_ResultErrorAuth_from_dict(time_mock): 'nickname': nick, 'address': relay_ip, 'circ': circ, 'dest_url': dest_url, 'scanner': scanner_nick, 'version': RESULT_VERSION, 'type': _ResultType.ErrorAuth, 'time': t, - 'ed25519_master_key': ed25519, + 'master_key_ed25519': ed25519, } r2 = Result.from_dict(d) assert isinstance(r1, ResultErrorAuth) diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index b61c303..750ab63 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -54,7 +54,7 @@ RESULT_ERROR_STREAM_DICT = { "type": "error-stream", "msg": "Something bad happened while measuring bandwidth", "nickname": "A", - "ed25519_master_key": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s" + "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s" } RESULT_SUCCESS_DICT = { @@ -78,7 +78,7 @@ RESULT_SUCCESS_DICT = { {"amount": 321663, "duration": 8.266003131866455}, {"amount": 321663, "duration": 5.779450178146362}], "nickname": "A", - "ed25519_master_key": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s" + "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s" } RESULT_SUCCESS_STR = str(RESULT_SUCCESS_DICT) RESULT_ERROR_STREAM_STR = str(RESULT_ERROR_STREAM_DICT)

1 0

[sbws/master] Add docstring for ed25519, from where the key comes
by pastly＠torproject.org 14 Jun '18

14 Jun '18

commit d63b5ef4fbca4191fe79ba77cb6ffb311e7755ed Author: juga0 <juga(a)riseup.net> Date: Mon Jun 11 10:35:19 2018 +0000 Add docstring for ed25519, from where the key comes --- sbws/lib/relaylist.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sbws/lib/relaylist.py b/sbws/lib/relaylist.py index c0ca9ad..5d584ea 100644 --- a/sbws/lib/relaylist.py +++ b/sbws/lib/relaylist.py @@ -79,6 +79,11 @@ class Relay: @property def ed25519_master_key(self): + """Obtain ed25519 master key of the relay in server descriptors. + + :returns: str, the ed25519 master key base 64 encoded without + trailing '='s. + """ return self._from_desc('ed25519_master_key').rstrip('=')

1 0

[sbws/master] Merge pull request #174 from juga0/obtain_ed25519
by pastly＠torproject.org 14 Jun '18

14 Jun '18

commit c4ed68372ba5d221a687d331e773460b37a24a8f Merge: a9b3967 d3f2184 Author: Matt Traudt <pastly(a)users.noreply.github.com> Date: Mon Jun 11 06:13:20 2018 -0400 Merge pull request #174 from juga0/obtain_ed25519 Add Relay abstraction layer and function to obtain relay ed25519 sbws/globals.py | 5 ++- sbws/lib/circuitbuilder.py | 23 +++------- sbws/lib/relaylist.py | 75 ++++++++++++++++++++++++++++++++- sbws/lib/resultdump.py | 34 +++++++++++---- sbws/lib/v3bwfile.py | 2 + sbws/util/stem.py | 14 +----- tests/integration/lib/test_relaylist.py | 17 ++++++++ tests/integration/util/test_stem.py | 32 -------------- tests/unit/conftest.py | 25 ++++------- tests/unit/core/test_generate.py | 20 ++++----- tests/unit/core/test_stats.py | 8 ++-- tests/unit/lib/data/results.txt | 4 +- tests/unit/lib/data/v3bw.txt | 2 +- tests/unit/lib/test_results.py | 38 ++++++++++++----- tests/unit/lib/test_v3bwfile.py | 14 +++--- 15 files changed, 189 insertions(+), 124 deletions(-)

1 0

[sbws/master] Remove print that doesn't need to exist
by pastly＠torproject.org 14 Jun '18

14 Jun '18

commit f80b285f66b68e0c686f88cd3c4e0d2e061e01cf Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Tue Jun 12 09:06:38 2018 -0400 Remove print that doesn't need to exist --- tests/integration/lib/test_relaylist.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/lib/test_relaylist.py b/tests/integration/lib/test_relaylist.py index a3fbbf0..8488c8b 100644 --- a/tests/integration/lib/test_relaylist.py +++ b/tests/integration/lib/test_relaylist.py @@ -10,7 +10,6 @@ def test_relay_properties(persistent_launch_tor): assert relay.fingerprint == 'AA45C13025C037F056E734169891878ED0880231' assert 'Authority' in relay.flags assert not relay.exit_policy or not relay.exit_policy.is_exiting_allowed() - print(relay) assert relay.average_bandwidth == 1073741824 assert relay.bandwidth == 0 assert relay.address == '127.10.0.1'

1 0

[sbws/master] Fix changelog link
by pastly＠torproject.org 14 Jun '18

14 Jun '18

commit f6f3eba439317e450ef701f4324fe6764100f09d Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Tue Jun 12 20:34:50 2018 -0400 Fix changelog link --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 648d149..3097ce2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,4 +15,4 @@ remember that `.join()` wants only string arguments and can't handle a `None`. So fix that. -[Unreleased]: https://github.com/pastly/simple-bw-scanner/v0.4.0...master +[Unreleased]: https://github.com/pastly/simple-bw-scanner/compare/v0.4.0...master

1 0

[sbws/master] Bump to 0.4.1
by pastly＠torproject.org 14 Jun '18

14 Jun '18

commit 26069fe22c72c85f5021ddeb4f736fbe2b36e4e2 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Thu Jun 14 09:19:08 2018 -0400 Bump to 0.4.1 --- CHANGELOG.md | 5 ++++- sbws/__init__.py | 2 +- tests/unit/lib/data/v3bw.txt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd5ed8f..79d781f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +## [0.4.1] + ### Changed - If the relay to measure is an exit, put it in the exit position and choose a @@ -31,4 +33,5 @@ So fix that. - `earliest_bandwidth` being the newest bw not the oldest (thanks juga0) - `node_id` was missing the character "$" at the beginning -[Unreleased]: https://github.com/pastly/simple-bw-scanner/compare/v0.4.0...master +[Unreleased]: https://github.com/pastly/simple-bw-scanner/compare/v0.4.1...master +[0.4.1]: https://github.com/pastly/simple-bw-scanner/compare/v0.4.0...v0.4.1 diff --git a/sbws/__init__.py b/sbws/__init__.py index cb2db29..f0ede3d 100644 --- a/sbws/__init__.py +++ b/sbws/__init__.py @@ -1 +1 @@ -__version__ = '0.4.1-dev' +__version__ = '0.4.1' diff --git a/tests/unit/lib/data/v3bw.txt b/tests/unit/lib/data/v3bw.txt index debbe5e..eb37634 100644 --- a/tests/unit/lib/data/v3bw.txt +++ b/tests/unit/lib/data/v3bw.txt @@ -5,6 +5,6 @@ file_created=2018-04-25T13:10:57 generator_started=2018-04-16T14:09:05 latest_bandwidth=2018-04-17T14:09:07 software=sbws -software_version=0.4.1-dev +software_version=0.4.1 ==== bw=54 error_circ=0 error_misc=0 error_stream=1 master_key_ed25519=g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s nick=A node_id=$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA rtt=456 success=1 time=2018-04-17T14:09:07

1 0

[sbws/master] Measure exits in the exit position; try harder to find a helper
by pastly＠torproject.org 14 Jun '18

14 Jun '18

commit 2fc8f53329caf2406870007e53110060ad3a73f1 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Tue Jun 12 21:18:13 2018 -0400 Measure exits in the exit position; try harder to find a helper --- CHANGELOG.md | 13 ++++++++++ sbws/core/scanner.py | 67 ++++++++++++++++++++++++++++++++++++++++----------- sbws/lib/relaylist.py | 20 +++++++++++++++ 3 files changed, 86 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3097ce2..918421a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +### Changed + +- If the relay to measure is an exit, put it in the exit position and choose a + non-exit to help. Previously the relay to measure would always be the first +hop. (GH#181) +- Try harder to find a relay to help measure the target relay with two changes. + Essentially: (1) Instead of only picking from relays that are 1.25 - 2.00 +times faster than it by consensus weight, try (in order) to find a relay that +is at least 2.00, 1.75, 1.50, 1.25, or 1.00 times as fast. If that fails, +instead of giving up, (2) pick the fastest relay in the network instead of +giving up. This compliments the previous change about measuring target exits in +the exit position. + ### Fixed - Exception that causes sbws to fall back to one measurement thread. We first diff --git a/sbws/core/scanner.py b/sbws/core/scanner.py index 8e9f9c4..6af1b0c 100644 --- a/sbws/core/scanner.py +++ b/sbws/core/scanner.py @@ -131,6 +131,40 @@ def measure_bandwidth_to_server(session, conf, dest, content_length): return results +def _pick_ideal_second_hop(relay, dest, rl, cont, is_exit): + ''' + Sbws builds two hop circuits. Given the **relay** to measure with + destination **dest**, pick a second relay that is or is not an exit + according to **is_exit**. + ''' + candidates = [] + candidates.extend(rl.exits if is_exit else rl.non_exits) + if not len(candidates): + return None + log.debug('Picking a 2nd hop to measure %s from %d choices. is_exit=%s', + relay.nickname, len(candidates), is_exit) + for min_bw_factor in [2, 1.75, 1.5, 1.25, 1]: + min_bw = relay.bandwidth * min_bw_factor + new_candidates = stem_utils.only_relays_with_bandwidth( + cont, candidates, min_bw=min_bw) + if len(new_candidates) > 0: + chosen = rng.choice(new_candidates) + log.debug( + 'Found %d candidate 2nd hops with at least %sx the bandwidth ' + 'of %s. Returning %s (bw=%s).', + len(new_candidates), min_bw_factor, relay.nickname, + chosen.nickname, chosen.bandwidth) + return chosen + candidates = sorted(candidates, key=lambda r: r.bandwidth, reverse=True) + chosen = candidates[0] + log.debug( + 'Didn\'t find any 2nd hops at least as fast as %s (bw=%s). It\'s ' + 'probably really fast. Returning %s (bw=%s), the fastest ' + 'candidate we have.', relay.nickname, relay.bandwidth, + chosen.nickname, chosen.bandwidth) + return chosen + + def measure_relay(args, conf, destinations, cb, rl, relay): s = requests_utils.make_session( cb.controller, conf.getfloat('general', 'http_timeout')) @@ -140,24 +174,29 @@ def measure_relay(args, conf, destinations, cb, rl, relay): log.warning('Unable to get destination to measure %s %s', relay.nickname, relay.fingerprint[0:8]) return None - # Pick an exit - exits = rl.exits_can_exit_to(dest.hostname, dest.port) - exits = [e for e in exits if e.fingerprint != relay.fingerprint] - exits = stem_utils.only_relays_with_bandwidth( - cb.controller, exits, min_bw=round(relay.bandwidth*1.25), - max_bw=max(round(relay.bandwidth*2.00), 100)) - if len(exits) < 1: - log.warning('No available exits to help measure %s %s', relay.nickname, - relay.fingerprint[0:8]) + # Pick a relay to help us measure the given relay. If the given relay is an + # exit, then pick a non-exit. Otherwise pick an exit. + helper = None + circ_fps = None + if relay.can_exit_to(dest.hostname, dest.port): + helper = _pick_ideal_second_hop( + relay, dest, rl, cb.controller, is_exit=False) + if helper: + circ_fps = [helper.fingerprint, relay.fingerprint] + else: + helper = _pick_ideal_second_hop( + relay, dest, rl, cb.controller, is_exit=True) + if helper: + circ_fps = [relay.fingerprint, helper.fingerprint] + if not helper: # TODO: Return ResultError of some sort + log.warning('Unable to pick a 2nd hop to help measure %s %s', + relay.nickname, relay.fingerprint[0:8]) return None - exit = rng.choice(exits) + assert helper + assert circ_fps is not None and len(circ_fps) == 2 # Build the circuit - log.debug('We selected exit %s %s (cw=%d) to help measure %s %s (cw=%d)', - exit.nickname, exit.fingerprint[0:8], exit.bandwidth, - relay.nickname, relay.fingerprint[0:8], relay.bandwidth) our_nick = conf['scanner']['nickname'] - circ_fps = [relay.fingerprint, exit.fingerprint] circ_id = cb.build_circuit(circ_fps) if not circ_id: log.warning('Could not build circuit involving %s', relay.nickname) diff --git a/sbws/lib/relaylist.py b/sbws/lib/relaylist.py index 7b3d58a..2cc5fef 100644 --- a/sbws/lib/relaylist.py +++ b/sbws/lib/relaylist.py @@ -88,6 +88,22 @@ class Relay: # it seems that stem parses it as ed25519_master_key return self._from_desc('ed25519_master_key').rstrip('=') + def can_exit_to(self, host, port): + if not self.exit_policy: + return False + assert isinstance(host, str) + assert isinstance(port, int) + if not is_valid_ipv4_address(host) and not is_valid_ipv6_address(host): + # It certainly isn't perfect trying to guess if an exit can connect + # to an ipv4/6 address based on the DNS result we got locally. But + # it's the best we can do. + # + # Also, only use the first ipv4/6 we get even if there is more than + # one. + host = resolve(host)[0] + assert is_valid_ipv4_address(host) or is_valid_ipv6_address(host) + return self.exit_policy.can_exit_to(host, port) + class RelayList: ''' Keeps a list of all relays in the current Tor network and updates it @@ -116,6 +132,10 @@ class RelayList: return self._relays_with_flag(Flag.EXIT) @property + def non_exits(self): + return self._relays_without_flag(Flag.EXIT) + + @property def guards(self): return self._relays_with_flag(Flag.GUARD)

1 0

[sbws/master] Add changelog entry for earliest_bandwidth fix
by pastly＠torproject.org 14 Jun '18

14 Jun '18

commit 4086f4d13378046f46cb4b4b074e5384f6ebdf3d Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Wed Jun 13 06:09:41 2018 -0400 Add changelog entry for earliest_bandwidth fix --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8f2bd9..35bdfd8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ remember that `.join()` wants only string arguments and can't handle a `None`. So fix that. - Exception when failing to get a relay's `ed25519_master_key` from Tor and trying to do `.rstrip()` on a None. +- `earliest_bandwidth` being the newest bw not the oldest (thanks juga0) [Unreleased]: https://github.com/pastly/simple-bw-scanner/compare/v0.4.0...master

1 0