[tor-commits] [tor-browser-build/master] Add snowflake

commit 832c2ce17672d5cd1f554354902df670e6ee928a Author: Nicolas Vigier boklm@torproject.org Date: Mon Mar 13 19:39:02 2017 +0100

Add snowflake

Only for linux-x86_64 for now, as it fails to build on linux-i686. --- .gitignore | 1 + projects/depot_tools/build | 8 +++ projects/depot_tools/config | 5 ++ projects/gcc/config | 2 +- projects/go-webrtc/config | 53 ++++++++++++++++++++ projects/go/config | 1 + projects/snowflake/build | 38 +++++++++++++++ projects/snowflake/config | 35 ++++++++++++++ projects/tor-browser/build | 3 ++ projects/tor-browser/config | 3 ++ projects/webrtc/build | 80 ++++++++++++++++++++++++++++++ projects/webrtc/config | 115 ++++++++++++++++++++++++++++++++++++++++++++ rbm.conf | 4 ++ 13 files changed, 347 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore index 9b6754f..0fe0aec 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ git_clones hg_clones +gclient out release alpha diff --git a/projects/depot_tools/build b/projects/depot_tools/build new file mode 100644 index 0000000..ac0f21b --- /dev/null +++ b/projects/depot_tools/build @@ -0,0 +1,8 @@ +#!/bin/bash +[% c("var/set_default_env") -%] +tar xf [% project %]-[% c("version") %].tar.gz +mv [% project %]-[% c("version") %] [% project %] +[% c('tar', { + tar_src => [ project ], + tar_args => '-czf ' _ dest_dir _ '/' _ c('filename'), + }) %] diff --git a/projects/depot_tools/config b/projects/depot_tools/config new file mode 100644 index 0000000..40a6d85 --- /dev/null +++ b/projects/depot_tools/config @@ -0,0 +1,5 @@ +# vim: filetype=yaml sw=2 +version: '[% c("abbrev") %]' +git_hash: 28216cd14b44716db5c83634afbdc6e90492652b +git_url: https://chromium.googlesource.com/chromium/tools/depot_tools.git +filename: '[% project %]-[% c("version") %].tar.gz' diff --git a/projects/gcc/config b/projects/gcc/config index bf73b7d..391e453 100644 --- a/projects/gcc/config +++ b/projects/gcc/config @@ -10,7 +10,7 @@ var: - libc6-dev-i386 setup: | mkdir -p /var/tmp/dist - tar -C /var/tmp/dist -xf [% c("compiler_tarfile") %] + tar -C /var/tmp/dist -xf $rootdir/[% c("compiler_tarfile") %] export PATH="/var/tmp/dist/gcc/bin:$PATH" [% IF c("var/osname") == "linux-i686" -%] export LD_LIBRARY_PATH=/var/tmp/dist/gcc/lib32 diff --git a/projects/go-webrtc/config b/projects/go-webrtc/config new file mode 100644 index 0000000..778d3db --- /dev/null +++ b/projects/go-webrtc/config @@ -0,0 +1,53 @@ +# vim: filetype=yaml sw=2 +version: '[% c("abbrev") %]' +git_url: https://github.com/keroserene/go-webrtc.git +git_hash: 69c5ca70533142494d145b43ef2f46ca94715557 +filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz' +remote_docker: 1 + +build: '[% c("projects/go/var/build_go_lib") %]' + +var: + go_lib: github.com/keroserene/go-webrtc + build_go_lib_pre: | + [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %] + [% IF c("var/linux") %] + tar -C /var/tmp/dist -xf $rootdir/[% c('input_files_by_name/binutils') %] + export PATH="/var/tmp/dist/binutils/bin:$PATH" + [% END -%] + tar -C /var/tmp/dist -xf $rootdir/[% c('input_files_by_name/webrtc') %] + # Replace the prebuilt webrtc library with our own one. + rm -rf include/ lib/ + cp -a /var/tmp/dist/webrtc/{include,lib} . + # The go-webrtc package sets _GLIBCXX_USE_CXX11_ABI=0 for compatibility with + # the prebuilt libwebrtc-magic.a libraries. Since we build our own + # libwebrtc-magic.a with the C++11 ABI, we have to undo that setting here, + # using CGO_CXXFLAGS. + export CGO_CXXFLAGS=-D_GLIBCXX_USE_CXX11_ABI=1 + +targets: + master: + git_hash: master + linux-x86_64: + var: + arch_deps: + - pkg-config + - libx11-dev + linux-i686: + var: + arch_deps: + - pkg-config:i386 + - libx11-dev:i386 + - lib32stdc++6 + +input_files: + - project: docker-image + - name: go + project: go + - name: webrtc + project: webrtc + - name: '[% c("var/compiler") %]' + project: '[% c("var/compiler") %]' + - name: binutils + project: binutils + enable: '[% c("var/linux") %]' diff --git a/projects/go/config b/projects/go/config index 5e55a09..870dc50 100644 --- a/projects/go/config +++ b/projects/go/config @@ -35,6 +35,7 @@ var: do patch -p1 < $p done + [% IF c("var/build_go_lib_pre"); GET c("var/build_go_lib_pre"); END; -%] [% IF c("var/go_lib_install") -%] [% FOREACH inst IN c("var/go_lib_install") %] go install [% inst %] diff --git a/projects/snowflake/build b/projects/snowflake/build new file mode 100644 index 0000000..6e08b4e --- /dev/null +++ b/projects/snowflake/build @@ -0,0 +1,38 @@ +#!/bin/bash +[% c("var/set_default_env") -%] +[% pc('go', 'var/setup', { go_tarfile => c('input_files_by_name/go') }) %] +[% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %] +[% IF c("var/linux") %] + tar -C /var/tmp/dist -xf $rootdir/[% c('input_files_by_name/binutils') %] + export PATH="/var/tmp/dist/binutils/bin:$PATH" +[% END -%] +distdir=/var/tmp/dist/[% project %] +[% IF c("var/osx") %] + PTDIR="$distdir/Contents/MacOS/Tor/PluggableTransports" + DOCSDIR="$distdir/Contents/Resources/TorBrowser/Docs/snowflake" +[% ELSE %] + PTDIR="$distdir/TorBrowser/Tor/PluggableTransports" + DOCSDIR="$distdir/TorBrowser/Docs/snowflake" +[% END %] +mkdir -p $PTDIR $DOCSDIR + +tar -C /var/tmp/dist -xf [% c('input_files_by_name/go-webrtc') %] +tar -C /var/tmp/dist -xf [% c('input_files_by_name/uniuri') %] +tar -C /var/tmp/dist -xf [% c('input_files_by_name/goptlib') %] + +mkdir -p /var/tmp/build +tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.gz +cd /var/tmp/build/[% project %]-[% c('version') %] + +cd client +go build -ldflags '-s' +cp -a client[% IF c("var/windows") %].exe[% END %] $PTDIR/snowflake-client[% IF c("var/windows") %].exe[% END %] + +cd .. +cp -a README.md LICENSE $DOCSDIR + +cd $distdir +[% c('tar', { + tar_src => [ '.' ], + tar_args => '-czf ' _ dest_dir _ '/' _ c('filename'), + }) %] diff --git a/projects/snowflake/config b/projects/snowflake/config new file mode 100644 index 0000000..5ba006b --- /dev/null +++ b/projects/snowflake/config @@ -0,0 +1,35 @@ +# vim: filetype=yaml sw=2 +version: '[% c("abbrev") %]' +git_url: https://git.torproject.org/pluggable-transports/snowflake.git +git_hash: 6cecd31fd896eb26e64ad8bab8a9ea510ec3b21d +filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz' +remote_docker: 1 + +targets: + linux-i686: + var: + arch_deps: + - pkg-config + - libx11-dev:i386 + - lib32stdc++6 + linux-x86_64: + var: + arch_deps: + - pkg-config + - libx11-dev + +input_files: + - project: docker-image + - name: go + project: go + - name: '[% c("var/compiler") %]' + project: '[% c("var/compiler") %]' + - name: binutils + project: binutils + enable: '[% c("var/linux") %]' + - name: go-webrtc + project: go-webrtc + - name: uniuri + project: uniuri + - name: goptlib + project: goptlib diff --git a/projects/tor-browser/build b/projects/tor-browser/build index 904f1b6..814f7af 100644 --- a/projects/tor-browser/build +++ b/projects/tor-browser/build @@ -52,6 +52,9 @@ mv [% c('input_files_by_name/noscript') %] $TBDIR/$EXTSPATH/{73a6fe31-595d-460b- tar -C $TBDIR -xf [% c('input_files_by_name/obfs4') %] tar -C $TBDIR -xf [% c('input_files_by_name/meek') %] mv $TBDIR/meek-http-helper@bamsoftware.com.xpi $TBDIR/$MEEKPROFILEPATH/extensions/ +[% IF c("var/snowflake") %] + tar -C $TBDIR -xf [% c('input_files_by_name/snowflake') -%] +[% END -%]

tar -C $TBDIR[% IF c("var/osx") %]/Contents/Resources[% END %] -xf [% c('input_files_by_name/fonts') %]

diff --git a/projects/tor-browser/config b/projects/tor-browser/config index 95f444d..f2c1d72 100644 --- a/projects/tor-browser/config +++ b/projects/tor-browser/config @@ -48,6 +48,9 @@ input_files: name: meek - project: obfs4 name: obfs4 + - project: snowflake + name: snowflake + enable: '[% c("var/snowflake") %]' - filename: Bundle-Data - URL: https://addons.cdn.mozilla.net/user-media/addons/722/noscript_security_suite... name: noscript diff --git a/projects/webrtc/build b/projects/webrtc/build new file mode 100644 index 0000000..5a70fbe --- /dev/null +++ b/projects/webrtc/build @@ -0,0 +1,80 @@ +#!/bin/bash +[% c("var/set_default_env") -%] +distdir=/var/tmp/dist/[% project %] +[% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %] + +mkdir -p /var/tmp/dist + +[% IF c("var/linux") %] + tar -C /var/tmp/dist -xf $rootdir/[% c('input_files_by_name/binutils') %] + export PATH="/var/tmp/dist/binutils/bin:$PATH" +[% END -%] + +# Setting up depot_tools +# https://dev.chromium.org/developers/how-tos/install-depot-tools +tar -C /var/tmp/dist -xf $rootdir/[% c('input_files_by_name/depot_tools') %] +export PATH="$PATH:/var/tmp/dist/depot_tools" +# Disable automatic updating. +export DEPOT_TOOLS_UPDATE=0 + +mkdir -p /var/tmp/build +tar -C /var/tmp/build -xf webrtc-sources-[% c('version') %].tar.gz + +builddir=/var/tmp/build/[% project %]/src +cd $builddir +export GYP_CROSSCOMPILE=1 +export GYP_DEFINES="OS=[% c("var/webrtc/os") %] target_arch=[% c("var/webrtc/arch") %]" +# Do not use bundled utilities. +GYP_DEFINES+=" clang=0 host_clang=0 use_sysroot=0" +GYP_DEFINES+=" linux_use_bundled_gold=0 linux_use_gold_flags=0 linux_use_bundled_binutils=0" +# Avoid some dependencies. Some of these don't seem to work fully; for +# example even with use_alsa=0 we need libasound2-dev. +GYP_DEFINES+=" use_x11=0 use_gtk=0 use_gconf=0 use_alsa=0 use_pulseaudio=0 use_libpci=0 include_internal_audio_device=0 enabled_libjingle_device_manager=0" +# examples and tests also bring in dependencies. +GYP_DEFINES+=" include_examples=0 include_tests=0" +# embedded=1 is supposed to turn of various features; see +# https://bugs.chromium.org/p/chromium/issues/detail?id=318413. +GYP_DEFINES+=" embedded=1" +GYP_DEFINES+=" werror=" +[% IF c("var/linux") -%] + export CC=gcc + export CXX=g++ +[% END -%] +[% IF c("var/linux-i686") -%] + GYP_DEFINES+=" target_arch=ia32" + export CC='gcc -m32' + export CXX='g++ -m32' + export CFLAGS=-m32 + export CXXFLAGS=-m32 + export LDFLAGS=-m32 +[% END -%] +JAVA_HOME=/usr/lib/jvm/default-java webrtc/build/gyp_webrtc.py +ninja -C out/Release +# Run ninja once more, without include_tests=0, in order to build just the +# FakeAudioCaptureModule that go-webrtc uses. +GYP_DEFINES="$(echo "$GYP_DEFINES" | sed -e 's/include_tests=0//g')" +JAVA_HOME=/usr/lib/jvm/default-java webrtc/build/gyp_webrtc.py +ninja -C out/Release obj/webrtc/api/test/peerconnection_unittests.fakeaudiocapturemodule.o +# https://github.com/keroserene/go-webrtc/issues/23#issuecomment-175312648 +# dump_syms_regtest.o is actually an executable, not an object file. If not +# excluded, it results in the error: +# libwebrtc-linux-386-magic.a(dump_syms_regtest.o): unsupported ELF file type 2 +ar crs libwebrtc-magic.a $(find . -name '*.o' -not -name '*.main.o' -not -name 'dump_syms_regtest.o' | sort) + +mkdir -p $distdir +cd $distdir +mkdir -p include lib +cp -f $builddir/libwebrtc-magic.a [% c("var/webrtc/lib_path") %] +INCLUDE_DIR="$PWD/include" +cd $builddir +for h in $(find talk/ webrtc/ -type f -name '*.h'); +do + mkdir -p "$INCLUDE_DIR/$(dirname $h)" + cp -f "$h" "$INCLUDE_DIR/$h" +done + +cd /var/tmp/dist +[% c('tar', { + tar_src => [ project ], + tar_args => '-czf ' _ dest_dir _ '/' _ c('filename'), + }) %] diff --git a/projects/webrtc/config b/projects/webrtc/config new file mode 100644 index 0000000..52618c2 --- /dev/null +++ b/projects/webrtc/config @@ -0,0 +1,115 @@ +# vim: filetype=yaml sw=2 +version: '[% c("var/webrtc_tag") %]' +remote_docker: 1 +filename: 'webrtc-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz' + +var: + webrtc_tag: f33698296719f956497d2dbff81b5080864a8804 + +input_files: + - project: docker-image + - project: webrtc + pkg_type: fetch_sources + - project: depot_tools + name: depot_tools + - name: '[% c("var/compiler") %]' + project: '[% c("var/compiler") %]' + - project: binutils + name: binutils + enable: '[% c("var/linux") %]' + +targets: + linux: + var: + webrtc: + os: linux + linux-i686: + var: + dockerbuild: "[% pc('docker-image', 'pre') %]" + sort_deps: 0 + arch_deps: + - default-jre-headless + - default-jdk + - lib32asound2-dev + - lib64expat1 + - libexpat1-dev:i386 + - libudev-dev:i386 + - libx11-dev:i386 + - libxext-dev:i386 + - libxrandr-dev:i386 + - lib32stdc++6 + - hardening-wrapper + webrtc: + arch: ia32 + lib_arch: 386 + lib_path: lib/libwebrtc-linux-386-magic.a + linux-x86_64: + var: + arch_deps: + - default-jre-headless + - default-jdk + - libasound2-dev + - libexpat1-dev + - libudev-dev + - libx11-dev + - libxext-dev + - libxrandr-dev + webrtc: + arch: x64 + lib_arch: amd64 + lib_path: lib/libwebrtc-linux-amd64-magic.a + +steps: + fetch_sources: + remote_docker: 0 + filename: 'webrtc-sources-[% c("var/webrtc_tag") %].tar.gz' + fetch_sources: | + #!/bin/bash + [% c("var/set_default_env") -%] + # WebRTC is special, having its own build system that brings in lots of Chromium dependencies. + # https://webrtc.org/native-code/development/ + tar xf [% c('input_files_by_name/depot_tools') %] + export PATH="$rootdir/depot_tools:$PATH" + # GYP_CROSSCOMPILE=1 and GYP_DEFINES="use_x11=0" prevent probing for certain dependencies. + # Use --no-history because the whole checkout with history is about 12 GB. + export GYP_CROSSCOMPILE=1 + export GYP_DEFINES="use_x11=0" + # JAVA_HOME is needed in a hook for libjingle. The readlink line tries to find the current JRE. + # default-java comes from the package default-jdk-headless. + export JAVA_HOME=/usr/lib/jvm/default-java + clone_dir='[% c("basedir") %]/gclient/webrtc' + mkdir -p "$clone_dir" + cd "$clone_dir" + if [ ! -d "src" ]; + then + # "fetch" is part of depot_tools. + #fetch --nohooks --no-history webrtc + # FIXME: To avoid the unconditional `gclient sync` in the call to fetch, + # we inline the result of a `fetch --dry-run` + gclient root + gclient config --spec 'solutions = [ + { + "managed": False, + "name": "src", + "url": "https://chromium.googlesource.com/external/webrtc.git", + "custom_deps": {}, + "deps_file": "DEPS", + "safesync_url": "", + }, + ] + ' + gclient sync --nohooks --no-history --with_branch_heads -r [% c("var/webrtc_tag") %] + cd src + git submodule foreach 'git config -f $toplevel/.git/config submodule.$name.ignore all' + git config --add remote.origin.fetch '+refs/tags/*:refs/tags/*' + git config diff.ignoreSubmodules all + cd .. + fi + # "gclient" is part of depot_tools. This download takes a long time the first time. + gclient sync --no-history --with_branch_heads -r [% c("var/webrtc_tag") %] + cd .. + tar --exclude .git -czf [% dest_dir _ '/' _ c('filename') %] webrtc + input_files: + - project: depot_tools + name: depot_tools + pkg_type: build diff --git a/rbm.conf b/rbm.conf index e5c69b0..e206193 100644 --- a/rbm.conf +++ b/rbm.conf @@ -105,6 +105,7 @@ targets: linux-x86_64: arch: x86_64 var: + linux-x86_64: 1 osname: linux-x86_64 deps: - build-essential @@ -115,9 +116,12 @@ targets: - libtool - zip - unzip + # We only build snowflake on linux-x86_64 for now + snowflake: 1 linux-i686: arch: i686 var: + linux-i686: 1 osname: linux-i686 configure_opt: '--host=i686-linux-gnu CFLAGS=-m32 CXXFLAGS=-m32 LDFLAGS=-m32' pre_pkginst: dpkg --add-architecture i386