[tor-commits] [tor-browser-build/master] Add snowflake
boklm at torproject.org
boklm at torproject.org
Mon Mar 13 18:50:00 UTC 2017
commit 832c2ce17672d5cd1f554354902df670e6ee928a
Author: Nicolas Vigier <boklm at torproject.org>
Date: Mon Mar 13 19:39:02 2017 +0100
Add snowflake
Only for linux-x86_64 for now, as it fails to build on linux-i686.
---
.gitignore | 1 +
projects/depot_tools/build | 8 +++
projects/depot_tools/config | 5 ++
projects/gcc/config | 2 +-
projects/go-webrtc/config | 53 ++++++++++++++++++++
projects/go/config | 1 +
projects/snowflake/build | 38 +++++++++++++++
projects/snowflake/config | 35 ++++++++++++++
projects/tor-browser/build | 3 ++
projects/tor-browser/config | 3 ++
projects/webrtc/build | 80 ++++++++++++++++++++++++++++++
projects/webrtc/config | 115 ++++++++++++++++++++++++++++++++++++++++++++
rbm.conf | 4 ++
13 files changed, 347 insertions(+), 1 deletion(-)
diff --git a/.gitignore b/.gitignore
index 9b6754f..0fe0aec 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
git_clones
hg_clones
+gclient
out
release
alpha
diff --git a/projects/depot_tools/build b/projects/depot_tools/build
new file mode 100644
index 0000000..ac0f21b
--- /dev/null
+++ b/projects/depot_tools/build
@@ -0,0 +1,8 @@
+#!/bin/bash
+[% c("var/set_default_env") -%]
+tar xf [% project %]-[% c("version") %].tar.gz
+mv [% project %]-[% c("version") %] [% project %]
+[% c('tar', {
+ tar_src => [ project ],
+ tar_args => '-czf ' _ dest_dir _ '/' _ c('filename'),
+ }) %]
diff --git a/projects/depot_tools/config b/projects/depot_tools/config
new file mode 100644
index 0000000..40a6d85
--- /dev/null
+++ b/projects/depot_tools/config
@@ -0,0 +1,5 @@
+# vim: filetype=yaml sw=2
+version: '[% c("abbrev") %]'
+git_hash: 28216cd14b44716db5c83634afbdc6e90492652b
+git_url: https://chromium.googlesource.com/chromium/tools/depot_tools.git
+filename: '[% project %]-[% c("version") %].tar.gz'
diff --git a/projects/gcc/config b/projects/gcc/config
index bf73b7d..391e453 100644
--- a/projects/gcc/config
+++ b/projects/gcc/config
@@ -10,7 +10,7 @@ var:
- libc6-dev-i386
setup: |
mkdir -p /var/tmp/dist
- tar -C /var/tmp/dist -xf [% c("compiler_tarfile") %]
+ tar -C /var/tmp/dist -xf $rootdir/[% c("compiler_tarfile") %]
export PATH="/var/tmp/dist/gcc/bin:$PATH"
[% IF c("var/osname") == "linux-i686" -%]
export LD_LIBRARY_PATH=/var/tmp/dist/gcc/lib32
diff --git a/projects/go-webrtc/config b/projects/go-webrtc/config
new file mode 100644
index 0000000..778d3db
--- /dev/null
+++ b/projects/go-webrtc/config
@@ -0,0 +1,53 @@
+# vim: filetype=yaml sw=2
+version: '[% c("abbrev") %]'
+git_url: https://github.com/keroserene/go-webrtc.git
+git_hash: 69c5ca70533142494d145b43ef2f46ca94715557
+filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz'
+remote_docker: 1
+
+build: '[% c("projects/go/var/build_go_lib") %]'
+
+var:
+ go_lib: github.com/keroserene/go-webrtc
+ build_go_lib_pre: |
+ [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %]
+ [% IF c("var/linux") %]
+ tar -C /var/tmp/dist -xf $rootdir/[% c('input_files_by_name/binutils') %]
+ export PATH="/var/tmp/dist/binutils/bin:$PATH"
+ [% END -%]
+ tar -C /var/tmp/dist -xf $rootdir/[% c('input_files_by_name/webrtc') %]
+ # Replace the prebuilt webrtc library with our own one.
+ rm -rf include/ lib/
+ cp -a /var/tmp/dist/webrtc/{include,lib} .
+ # The go-webrtc package sets _GLIBCXX_USE_CXX11_ABI=0 for compatibility with
+ # the prebuilt libwebrtc-magic.a libraries. Since we build our own
+ # libwebrtc-magic.a with the C++11 ABI, we have to undo that setting here,
+ # using CGO_CXXFLAGS.
+ export CGO_CXXFLAGS=-D_GLIBCXX_USE_CXX11_ABI=1
+
+targets:
+ master:
+ git_hash: master
+ linux-x86_64:
+ var:
+ arch_deps:
+ - pkg-config
+ - libx11-dev
+ linux-i686:
+ var:
+ arch_deps:
+ - pkg-config:i386
+ - libx11-dev:i386
+ - lib32stdc++6
+
+input_files:
+ - project: docker-image
+ - name: go
+ project: go
+ - name: webrtc
+ project: webrtc
+ - name: '[% c("var/compiler") %]'
+ project: '[% c("var/compiler") %]'
+ - name: binutils
+ project: binutils
+ enable: '[% c("var/linux") %]'
diff --git a/projects/go/config b/projects/go/config
index 5e55a09..870dc50 100644
--- a/projects/go/config
+++ b/projects/go/config
@@ -35,6 +35,7 @@ var:
do
patch -p1 < $p
done
+ [% IF c("var/build_go_lib_pre"); GET c("var/build_go_lib_pre"); END; -%]
[% IF c("var/go_lib_install") -%]
[% FOREACH inst IN c("var/go_lib_install") %]
go install [% inst %]
diff --git a/projects/snowflake/build b/projects/snowflake/build
new file mode 100644
index 0000000..6e08b4e
--- /dev/null
+++ b/projects/snowflake/build
@@ -0,0 +1,38 @@
+#!/bin/bash
+[% c("var/set_default_env") -%]
+[% pc('go', 'var/setup', { go_tarfile => c('input_files_by_name/go') }) %]
+[% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %]
+[% IF c("var/linux") %]
+ tar -C /var/tmp/dist -xf $rootdir/[% c('input_files_by_name/binutils') %]
+ export PATH="/var/tmp/dist/binutils/bin:$PATH"
+[% END -%]
+distdir=/var/tmp/dist/[% project %]
+[% IF c("var/osx") %]
+ PTDIR="$distdir/Contents/MacOS/Tor/PluggableTransports"
+ DOCSDIR="$distdir/Contents/Resources/TorBrowser/Docs/snowflake"
+[% ELSE %]
+ PTDIR="$distdir/TorBrowser/Tor/PluggableTransports"
+ DOCSDIR="$distdir/TorBrowser/Docs/snowflake"
+[% END %]
+mkdir -p $PTDIR $DOCSDIR
+
+tar -C /var/tmp/dist -xf [% c('input_files_by_name/go-webrtc') %]
+tar -C /var/tmp/dist -xf [% c('input_files_by_name/uniuri') %]
+tar -C /var/tmp/dist -xf [% c('input_files_by_name/goptlib') %]
+
+mkdir -p /var/tmp/build
+tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.gz
+cd /var/tmp/build/[% project %]-[% c('version') %]
+
+cd client
+go build -ldflags '-s'
+cp -a client[% IF c("var/windows") %].exe[% END %] $PTDIR/snowflake-client[% IF c("var/windows") %].exe[% END %]
+
+cd ..
+cp -a README.md LICENSE $DOCSDIR
+
+cd $distdir
+[% c('tar', {
+ tar_src => [ '.' ],
+ tar_args => '-czf ' _ dest_dir _ '/' _ c('filename'),
+ }) %]
diff --git a/projects/snowflake/config b/projects/snowflake/config
new file mode 100644
index 0000000..5ba006b
--- /dev/null
+++ b/projects/snowflake/config
@@ -0,0 +1,35 @@
+# vim: filetype=yaml sw=2
+version: '[% c("abbrev") %]'
+git_url: https://git.torproject.org/pluggable-transports/snowflake.git
+git_hash: 6cecd31fd896eb26e64ad8bab8a9ea510ec3b21d
+filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz'
+remote_docker: 1
+
+targets:
+ linux-i686:
+ var:
+ arch_deps:
+ - pkg-config
+ - libx11-dev:i386
+ - lib32stdc++6
+ linux-x86_64:
+ var:
+ arch_deps:
+ - pkg-config
+ - libx11-dev
+
+input_files:
+ - project: docker-image
+ - name: go
+ project: go
+ - name: '[% c("var/compiler") %]'
+ project: '[% c("var/compiler") %]'
+ - name: binutils
+ project: binutils
+ enable: '[% c("var/linux") %]'
+ - name: go-webrtc
+ project: go-webrtc
+ - name: uniuri
+ project: uniuri
+ - name: goptlib
+ project: goptlib
diff --git a/projects/tor-browser/build b/projects/tor-browser/build
index 904f1b6..814f7af 100644
--- a/projects/tor-browser/build
+++ b/projects/tor-browser/build
@@ -52,6 +52,9 @@ mv [% c('input_files_by_name/noscript') %] $TBDIR/$EXTSPATH/{73a6fe31-595d-460b-
tar -C $TBDIR -xf [% c('input_files_by_name/obfs4') %]
tar -C $TBDIR -xf [% c('input_files_by_name/meek') %]
mv $TBDIR/meek-http-helper at bamsoftware.com.xpi $TBDIR/$MEEKPROFILEPATH/extensions/
+[% IF c("var/snowflake") %]
+ tar -C $TBDIR -xf [% c('input_files_by_name/snowflake') -%]
+[% END -%]
tar -C $TBDIR[% IF c("var/osx") %]/Contents/Resources[% END %] -xf [% c('input_files_by_name/fonts') %]
diff --git a/projects/tor-browser/config b/projects/tor-browser/config
index 95f444d..f2c1d72 100644
--- a/projects/tor-browser/config
+++ b/projects/tor-browser/config
@@ -48,6 +48,9 @@ input_files:
name: meek
- project: obfs4
name: obfs4
+ - project: snowflake
+ name: snowflake
+ enable: '[% c("var/snowflake") %]'
- filename: Bundle-Data
- URL: https://addons.cdn.mozilla.net/user-media/addons/722/noscript_security_suite-2.9.5.3-fx+sm.xpi
name: noscript
diff --git a/projects/webrtc/build b/projects/webrtc/build
new file mode 100644
index 0000000..5a70fbe
--- /dev/null
+++ b/projects/webrtc/build
@@ -0,0 +1,80 @@
+#!/bin/bash
+[% c("var/set_default_env") -%]
+distdir=/var/tmp/dist/[% project %]
+[% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %]
+
+mkdir -p /var/tmp/dist
+
+[% IF c("var/linux") %]
+ tar -C /var/tmp/dist -xf $rootdir/[% c('input_files_by_name/binutils') %]
+ export PATH="/var/tmp/dist/binutils/bin:$PATH"
+[% END -%]
+
+# Setting up depot_tools
+# https://dev.chromium.org/developers/how-tos/install-depot-tools
+tar -C /var/tmp/dist -xf $rootdir/[% c('input_files_by_name/depot_tools') %]
+export PATH="$PATH:/var/tmp/dist/depot_tools"
+# Disable automatic updating.
+export DEPOT_TOOLS_UPDATE=0
+
+mkdir -p /var/tmp/build
+tar -C /var/tmp/build -xf webrtc-sources-[% c('version') %].tar.gz
+
+builddir=/var/tmp/build/[% project %]/src
+cd $builddir
+export GYP_CROSSCOMPILE=1
+export GYP_DEFINES="OS=[% c("var/webrtc/os") %] target_arch=[% c("var/webrtc/arch") %]"
+# Do not use bundled utilities.
+GYP_DEFINES+=" clang=0 host_clang=0 use_sysroot=0"
+GYP_DEFINES+=" linux_use_bundled_gold=0 linux_use_gold_flags=0 linux_use_bundled_binutils=0"
+# Avoid some dependencies. Some of these don't seem to work fully; for
+# example even with use_alsa=0 we need libasound2-dev.
+GYP_DEFINES+=" use_x11=0 use_gtk=0 use_gconf=0 use_alsa=0 use_pulseaudio=0 use_libpci=0 include_internal_audio_device=0 enabled_libjingle_device_manager=0"
+# examples and tests also bring in dependencies.
+GYP_DEFINES+=" include_examples=0 include_tests=0"
+# embedded=1 is supposed to turn of various features; see
+# https://bugs.chromium.org/p/chromium/issues/detail?id=318413.
+GYP_DEFINES+=" embedded=1"
+GYP_DEFINES+=" werror="
+[% IF c("var/linux") -%]
+ export CC=gcc
+ export CXX=g++
+[% END -%]
+[% IF c("var/linux-i686") -%]
+ GYP_DEFINES+=" target_arch=ia32"
+ export CC='gcc -m32'
+ export CXX='g++ -m32'
+ export CFLAGS=-m32
+ export CXXFLAGS=-m32
+ export LDFLAGS=-m32
+[% END -%]
+JAVA_HOME=/usr/lib/jvm/default-java webrtc/build/gyp_webrtc.py
+ninja -C out/Release
+# Run ninja once more, without include_tests=0, in order to build just the
+# FakeAudioCaptureModule that go-webrtc uses.
+GYP_DEFINES="$(echo "$GYP_DEFINES" | sed -e 's/include_tests=0//g')"
+JAVA_HOME=/usr/lib/jvm/default-java webrtc/build/gyp_webrtc.py
+ninja -C out/Release obj/webrtc/api/test/peerconnection_unittests.fakeaudiocapturemodule.o
+# https://github.com/keroserene/go-webrtc/issues/23#issuecomment-175312648
+# dump_syms_regtest.o is actually an executable, not an object file. If not
+# excluded, it results in the error:
+# libwebrtc-linux-386-magic.a(dump_syms_regtest.o): unsupported ELF file type 2
+ar crs libwebrtc-magic.a $(find . -name '*.o' -not -name '*.main.o' -not -name 'dump_syms_regtest.o' | sort)
+
+mkdir -p $distdir
+cd $distdir
+mkdir -p include lib
+cp -f $builddir/libwebrtc-magic.a [% c("var/webrtc/lib_path") %]
+INCLUDE_DIR="$PWD/include"
+cd $builddir
+for h in $(find talk/ webrtc/ -type f -name '*.h');
+do
+ mkdir -p "$INCLUDE_DIR/$(dirname $h)"
+ cp -f "$h" "$INCLUDE_DIR/$h"
+done
+
+cd /var/tmp/dist
+[% c('tar', {
+ tar_src => [ project ],
+ tar_args => '-czf ' _ dest_dir _ '/' _ c('filename'),
+ }) %]
diff --git a/projects/webrtc/config b/projects/webrtc/config
new file mode 100644
index 0000000..52618c2
--- /dev/null
+++ b/projects/webrtc/config
@@ -0,0 +1,115 @@
+# vim: filetype=yaml sw=2
+version: '[% c("var/webrtc_tag") %]'
+remote_docker: 1
+filename: 'webrtc-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz'
+
+var:
+ webrtc_tag: f33698296719f956497d2dbff81b5080864a8804
+
+input_files:
+ - project: docker-image
+ - project: webrtc
+ pkg_type: fetch_sources
+ - project: depot_tools
+ name: depot_tools
+ - name: '[% c("var/compiler") %]'
+ project: '[% c("var/compiler") %]'
+ - project: binutils
+ name: binutils
+ enable: '[% c("var/linux") %]'
+
+targets:
+ linux:
+ var:
+ webrtc:
+ os: linux
+ linux-i686:
+ var:
+ dockerbuild: "[% pc('docker-image', 'pre') %]"
+ sort_deps: 0
+ arch_deps:
+ - default-jre-headless
+ - default-jdk
+ - lib32asound2-dev
+ - lib64expat1
+ - libexpat1-dev:i386
+ - libudev-dev:i386
+ - libx11-dev:i386
+ - libxext-dev:i386
+ - libxrandr-dev:i386
+ - lib32stdc++6
+ - hardening-wrapper
+ webrtc:
+ arch: ia32
+ lib_arch: 386
+ lib_path: lib/libwebrtc-linux-386-magic.a
+ linux-x86_64:
+ var:
+ arch_deps:
+ - default-jre-headless
+ - default-jdk
+ - libasound2-dev
+ - libexpat1-dev
+ - libudev-dev
+ - libx11-dev
+ - libxext-dev
+ - libxrandr-dev
+ webrtc:
+ arch: x64
+ lib_arch: amd64
+ lib_path: lib/libwebrtc-linux-amd64-magic.a
+
+steps:
+ fetch_sources:
+ remote_docker: 0
+ filename: 'webrtc-sources-[% c("var/webrtc_tag") %].tar.gz'
+ fetch_sources: |
+ #!/bin/bash
+ [% c("var/set_default_env") -%]
+ # WebRTC is special, having its own build system that brings in lots of Chromium dependencies.
+ # https://webrtc.org/native-code/development/
+ tar xf [% c('input_files_by_name/depot_tools') %]
+ export PATH="$rootdir/depot_tools:$PATH"
+ # GYP_CROSSCOMPILE=1 and GYP_DEFINES="use_x11=0" prevent probing for certain dependencies.
+ # Use --no-history because the whole checkout with history is about 12 GB.
+ export GYP_CROSSCOMPILE=1
+ export GYP_DEFINES="use_x11=0"
+ # JAVA_HOME is needed in a hook for libjingle. The readlink line tries to find the current JRE.
+ # default-java comes from the package default-jdk-headless.
+ export JAVA_HOME=/usr/lib/jvm/default-java
+ clone_dir='[% c("basedir") %]/gclient/webrtc'
+ mkdir -p "$clone_dir"
+ cd "$clone_dir"
+ if [ ! -d "src" ];
+ then
+ # "fetch" is part of depot_tools.
+ #fetch --nohooks --no-history webrtc
+ # FIXME: To avoid the unconditional `gclient sync` in the call to fetch,
+ # we inline the result of a `fetch --dry-run`
+ gclient root
+ gclient config --spec 'solutions = [
+ {
+ "managed": False,
+ "name": "src",
+ "url": "https://chromium.googlesource.com/external/webrtc.git",
+ "custom_deps": {},
+ "deps_file": "DEPS",
+ "safesync_url": "",
+ },
+ ]
+ '
+ gclient sync --nohooks --no-history --with_branch_heads -r [% c("var/webrtc_tag") %]
+ cd src
+ git submodule foreach 'git config -f $toplevel/.git/config submodule.$name.ignore all'
+ git config --add remote.origin.fetch '+refs/tags/*:refs/tags/*'
+ git config diff.ignoreSubmodules all
+ cd ..
+ fi
+ # "gclient" is part of depot_tools. This download takes a long time the first time.
+ gclient sync --no-history --with_branch_heads -r [% c("var/webrtc_tag") %]
+ cd ..
+ tar --exclude .git -czf [% dest_dir _ '/' _ c('filename') %] webrtc
+ input_files:
+ - project: depot_tools
+ name: depot_tools
+ pkg_type: build
diff --git a/rbm.conf b/rbm.conf
index e5c69b0..e206193 100644
--- a/rbm.conf
+++ b/rbm.conf
@@ -105,6 +105,7 @@ targets:
linux-x86_64:
arch: x86_64
var:
+ linux-x86_64: 1
osname: linux-x86_64
deps:
- build-essential
@@ -115,9 +116,12 @@ targets:
- libtool
- zip
- unzip
+ # We only build snowflake on linux-x86_64 for now
+ snowflake: 1
linux-i686:
arch: i686
var:
+ linux-i686: 1
osname: linux-i686
configure_opt: '--host=i686-linux-gnu CFLAGS=-m32 CXXFLAGS=-m32 LDFLAGS=-m32'
pre_pkginst: dpkg --add-architecture i386
More information about the tor-commits
mailing list