commit bd9ebb3763b90a8ef429376ee578e0571371145f Author: Nick Mathewson nickm@torproject.org Date: Mon Jun 18 17:08:23 2018 -0400

Use a rust build script to set linker options correctly for tests.

We need this trick because some of our Rust tests depend on our C code, which in turn depend on other native libraries, which thereby pulls a whole mess of our build system into "cargo test".

To solve this, we add a build script (build.rs) to set most of the options that we want based on the contents of config.rust. Some options can't be set, and need to go to the linker directly: we use a linker replacement (link_rust.sh) for these. Both config.rust and link_rust.sh are generated by autoconf for us.

This patch on its own should enough to make the crypto test build, but not necessarily enough to make it pass. --- .gitignore | 2 + config.rust.in | 22 ++++++ configure.ac | 20 +++++ link_rust.sh.in | 10 +++ src/rust/build.rs | 179 +++++++++++++++++++++++++++++++++++++++++++++ src/rust/crypto/Cargo.toml | 2 +- src/test/include.am | 7 +- 7 files changed, 239 insertions(+), 3 deletions(-)

diff --git a/.gitignore b/.gitignore index c00fbe97e..b3b13694a 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,7 @@ uptime-*.json /autom4te.cache /build-stamp /compile +/config.rust /configure /Doxyfile /orconfig.h @@ -55,6 +56,7 @@ uptime-*.json /config.guess /config.sub /conftest* +/link_rust.sh /micro-revision.* /patch-stamp /stamp-h diff --git a/config.rust.in b/config.rust.in new file mode 100644 index 000000000..4ca5351ae --- /dev/null +++ b/config.rust.in @@ -0,0 +1,22 @@ +# Used by our cargo build.rs script to get variables from autoconf. +# +# The "configure" script will generate "config.rust" from "config.rust.in", +# and then build.rs will read "config.rust". + +BUILDDIR=@BUILDDIR@ +TOR_LDFLAGS_zlib=@TOR_LDFLAGS_zlib@ +TOR_LDFLAGS_openssl=@TOR_LDFLAGS_openssl@ +TOR_LDFLAGS_libevent=@TOR_LDFLAGS_libevent@ +TOR_ZLIB_LIBS=@TOR_ZLIB_LIBS@ +TOR_LIB_MATH=@TOR_LIB_MATH@ +TOR_LIBEVENT_LIBS=@TOR_LIBEVENT_LIBS@ +TOR_OPENSSL_LIBS=@TOR_OPENSSL_LIBS@ +TOR_LIB_WS32=@TOR_LIB_WS32@ +TOR_LIB_GDI=@TOR_LIB_GDI@ +TOR_LIB_USERENV=@TOR_LIB_USERENV@ +CURVE25519_LIBS=@CURVE25519_LIBS@ +TOR_SYSTEMD_LIBS=@TOR_SYSTEMD_LIBS@ +TOR_LZMA_LIBS=@TOR_LZMA_LIBS@ +TOR_ZSTD_LIBS=@TOR_ZSTD_LIBS@ +LIBS=@LIBS@ +LDFLAGS=@LDFLAGS@ diff --git a/configure.ac b/configure.ac index 48ca5bae1..9ea1bc321 100644 --- a/configure.ac +++ b/configure.ac @@ -1117,6 +1117,24 @@ if test "$fragile_hardening" = "yes"; then TOR_CHECK_CFLAGS([-fno-omit-frame-pointer]) fi

+dnl Find the correct libraries to add in order to use the sanitizers. +dnl +dnl When building Rust, Cargo will run the linker with the -nodefaultlibs +dnl option, which will prevent the compiler from linking the sanitizer +dnl libraries it needs. We need to specify them manually. +dnl +dnl What's more, we need to specify them in a linker script rather than +dnl from build.rs: these options aren't allowed in the cargo:rustc-flags +dnl variable. +RUST_LINKER_OPTIONS="" +if test "x$CFLAGS_ASAN" != "x"; then + RUST_LINKER_OPTIONS="$RUST_LINKER_OPTIONS -static-libasan" +fi +if test "x$CFLAGS_UBSAN" != "x"; then + RUST_LINKER_OPTIONS="$RUST_LINKER_OPTIONS -static-libubsan" +fi +AC_SUBST(RUST_LINKER_OPTIONS) + CFLAGS_BUGTRAP="$CFLAGS_FTRAPV $CFLAGS_ASAN $CFLAGS_UBSAN" CFLAGS_CONSTTIME="$CFLAGS_FWRAPV"

@@ -2266,6 +2284,8 @@ CPPFLAGS="$CPPFLAGS $TOR_CPPFLAGS_libevent $TOR_CPPFLAGS_openssl $TOR_CPPFLAGS_z AC_CONFIG_FILES([ Doxyfile Makefile + config.rust + link_rust.sh contrib/dist/suse/tor.sh contrib/operator-tools/tor.logrotate contrib/dist/tor.sh diff --git a/link_rust.sh.in b/link_rust.sh.in new file mode 100644 index 000000000..2b972fdf2 --- /dev/null +++ b/link_rust.sh.in @@ -0,0 +1,10 @@ +#!/bin/sh +# +# A linker script used when building Rust tests. Autoconf makes link_rust.sh +# from link_rust_sh.in, and uses it to pass extra options to the linker +# when linking Rust stuff. +# +# We'd like to remove the need for this, but build.rs doesn't let us pass +# -static-libasan and -static-libubsan to the linker. + +"$CCLD" @RUST_LINKER_OPTIONS@ "$@" diff --git a/src/rust/build.rs b/src/rust/build.rs new file mode 100644 index 000000000..b943aa553 --- /dev/null +++ b/src/rust/build.rs @@ -0,0 +1,179 @@ +//! Build script for Rust modules in Tor. +//! +//! We need to use this because some of our Rust tests need to use some +//! of our C modules, which need to link some external libraries. +//! +//! This script works by looking at a "config.rust" file generated by our +//! configure script, and then building a set of options for cargo to pass to +//! the compiler. + +use std::collections::HashMap; +use std::env; +use std::fs::File; +use std::io::prelude::*; +use std::io; +use std::path::PathBuf; + +/// Wrapper around a key-value map. +struct Config( + HashMap<String,String> +); + +/// Locate a config.rust file generated by autoconf, starting in the OUT_DIR +/// location provided by cargo and recursing up the directory tree. Note that +/// we need to look in the OUT_DIR, since autoconf will place generated files +/// in the build directory. +fn find_cfg() -> io::Result<String> { + let mut path = PathBuf::from(env::var("OUT_DIR").unwrap()); + loop { + path.push("config.rust"); + if path.exists() { + return Ok(path.to_str().unwrap().to_owned()); + } + path.pop(); // remove config.rust + if ! path.pop() { // can't remove last part of directory + return Err(io::Error::new(io::ErrorKind::NotFound, + "No config.rust")); + } + } +} + +impl Config { + /// Find the config.rust file and try to parse it. + /// + /// The file format is a series of lines of the form KEY=VAL, with + /// any blank lines and lines starting with # ignored. + fn load() -> io::Result<Config> { + let path = find_cfg()?; + let f = File::open(&path)?; + let reader = io::BufReader::new(f); + let mut map = HashMap::new(); + for line in reader.lines() { + let s = line?; + if s.trim().starts_with("#") || s.trim() == "" { + continue; + } + let idx = match s.find("=") { + None => { + return Err(io::Error::new(io::ErrorKind::InvalidData, + "missing =")); + }, + Some(x) => x + }; + let (var,eq_val) = s.split_at(idx); + let val = &eq_val[1..]; + map.insert(var.to_owned(), val.to_owned()); + } + Ok(Config(map)) + } + + /// Return a reference to the value whose key is 'key'. + /// + /// Panics if 'key' is not found in the configuration. + fn get(&self, key : &str) -> &str { + self.0.get(key).unwrap() + } + + /// Add a dependency on a static C library that is part of Tor, by name. + fn component(&self, s : &str) { + println!("cargo:rustc-link-lib=static={}", s); + } + + /// Add a dependency on a native library that is not part of Tor, by name. + fn dependency(&self, s : &str) { + println!("cargo:rustc-link-lib={}", s); + } + + /// Add a link path, relative to Tor's build directory. + fn link_relpath(&self, s : &str) { + let builddir = self.get("BUILDDIR"); + println!("cargo:rustc-link-search=native={}/{}", builddir, s); + } + + /// Add an absolute link path. + fn link_path(&self, s : &str) { + println!("cargo:rustc-link-search=native={}", s); + } + + /// Parse the CFLAGS in s, looking for -l and -L items, and adding + /// rust configuration as appropriate. + fn from_cflags(&self, s : &str) { + let mut next_is_lib = false; + let mut next_is_path = false; + for ent in self.get(s).split_whitespace() { + if next_is_lib { + self.dependency(ent); + next_is_lib = false; + } else if next_is_path { + self.link_path(ent); + next_is_path = false; + } else if ent == "-l" { + next_is_lib = true; + } else if ent == "-L" { + next_is_path = true; + } else if ent.starts_with("-L") { + self.link_path(&ent[2..]); + } else if ent.starts_with("-l") { + self.dependency(&ent[2..]); + } + } + } +} + +pub fn main() { + let cfg = Config::load().unwrap(); + let package = env::var("CARGO_PKG_NAME").unwrap(); + + match package.as_ref() { + "crypto" => { + // Right now, I'm having a separate configuration for each Rust + // package, since I'm hoping we can trim them down. Once we have a + // second Rust package that needs to use this build script, let's + // extract some of this stuff into a module. + // + // This is a ridiculous amount of code to be pulling in just + // to test our crypto library: modularity would be our + // friend here. + cfg.from_cflags("TOR_LDFLAGS_zlib"); + cfg.from_cflags("TOR_LDFLAGS_openssl"); + cfg.from_cflags("TOR_LDFLAGS_libevent"); + + cfg.link_relpath("src/common"); + cfg.link_relpath("src/ext/keccak-tiny"); + cfg.link_relpath("src/ext/keccak-tiny"); + cfg.link_relpath("src/ext/ed25519/ref10"); + cfg.link_relpath("src/ext/ed25519/donna"); + cfg.link_relpath("src/trunnel"); + + // Note that we can't pull in "libtor-testing", or else we + // will have dependencies on all the other rust packages that + // tor uses. We must be careful with factoring and dependencies + // moving forward! + cfg.component("or-crypto-testing"); + cfg.component("or-ctime-testing"); + cfg.component("or-testing"); + cfg.component("or-event-testing"); + cfg.component("or-ctime-testing"); + cfg.component("curve25519_donna"); + cfg.component("keccak-tiny"); + cfg.component("ed25519_ref10"); + cfg.component("ed25519_donna"); + cfg.component("or-trunnel-testing"); + + cfg.from_cflags("TOR_ZLIB_LIBS"); + cfg.from_cflags("TOR_LIB_MATH"); + cfg.from_cflags("TOR_OPENSSL_LIBS"); + cfg.from_cflags("TOR_LIBEVENT_LIBS"); + cfg.from_cflags("TOR_LIB_WS32"); + cfg.from_cflags("TOR_LIB_GDI"); + cfg.from_cflags("TOR_LIB_USERENV"); + cfg.from_cflags("CURVE25519_LIBS"); + cfg.from_cflags("TOR_LZMA_LIBS"); + cfg.from_cflags("TOR_ZSTD_LIBS"); + cfg.from_cflags("LIBS"); + }, + _ => { + panic!("No configuration in build.rs for package {}", package); + } + } +} diff --git a/src/rust/crypto/Cargo.toml b/src/rust/crypto/Cargo.toml index c0c5e7bf9..9d302ee3b 100644 --- a/src/rust/crypto/Cargo.toml +++ b/src/rust/crypto/Cargo.toml @@ -4,6 +4,7 @@ authors = ["The Tor Project", name = "crypto" version = "0.0.1" publish = false +build = "../build.rs"

[lib] name = "crypto" @@ -26,4 +27,3 @@ rand_core = { version = "=0.2.0-pre.0", default-features = false }

[features] testing = ["tor_log/testing"] - diff --git a/src/test/include.am b/src/test/include.am index 2ae598b22..4146de73b 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -10,7 +10,10 @@ TESTS_ENVIRONMENT = \ export TESTING_TOR_BINARY="$(TESTING_TOR_BINARY)"; \ export CARGO="$(CARGO)"; \ export EXTRA_CARGO_OPTIONS="$(EXTRA_CARGO_OPTIONS)"; \ - export CARGO_ONLINE="$(CARGO_ONLINE)"; + export CARGO_ONLINE="$(CARGO_ONLINE)"; \ + export CCLD="$(CCLD)"; \ + chmod +x "$(abs_top_builddir)/link_rust.sh"; \ + export RUSTFLAGS="-C linker=$(abs_top_builddir)/link_rust.sh";

TESTSCRIPTS = \ src/test/fuzz_static_testcases.sh \ @@ -347,7 +350,7 @@ src_test_test_bt_cl_LDADD = src/common/libor-testing.a \ src/trace/libor-trace.a \ $(rust_ldadd) \ @TOR_LIB_MATH@ \ - @TOR_LIB_WS32@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ + @TOR_LIB_WS32@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ src_test_test_bt_cl_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) src_test_test_bt_cl_CPPFLAGS= $(src_test_AM_CPPFLAGS) $(TEST_CPPFLAGS)