commit 1318b6a9ec69b1e776d8ed5e0a99ab20eef3576c Author: Hans-Christoph Steiner <hans@eds.org> Date: Thu Jul 16 14:02:15 2020 +0200 stripped down Android build process for gitlab-ci and Vagrant --- .gitignore | 6 ++++++ .gitlab-ci.yml | 64 +++++++++++++++++++++++++++++++++++++------------------- README.md | 10 +++++++++ Vagrantfile | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 125 insertions(+), 21 deletions(-) diff --git a/.gitignore b/.gitignore index 002f95e..b5474f1 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,9 @@ proxy/proxy probetest/probetest snowflake.log ignore/ + +# from running the vagrant setup +/.vagrant/ +/sdk-tools-linux-*.zip* +/android-ndk-* +/tools/ \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c483873..35caa6c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,18 @@ +.apt-template: &apt-template +- export LC_ALL=C.UTF-8 +- export DEBIAN_FRONTEND=noninteractive +- echo Etc/UTC > /etc/timezone +- echo 'quiet "1";' + 'APT::Install-Recommends "0";' + 'APT::Install-Suggests "0";' + 'APT::Acquire::Retries "20";' + 'APT::Get::Assume-Yes "true";' + 'Dpkg::Use-Pty "0";' + > /etc/apt/apt.conf.d/99gitlab +- apt-get update +- apt-get dist-upgrade + # Set things up to use the OS-native packages for Go. Anything that # is downloaded by go during the `go fmt` stage is not coming from the # Debian/Ubuntu repo. So those would need to be packaged for this to @@ -25,6 +39,7 @@ golang-github-xtaci-smux-dev golang-golang-x-crypto-dev golang-golang-x-net-dev + golang-goptlib-dev golang-golang-x-sys-dev golang-golang-x-text-dev golang-golang-x-xerrors-dev @@ -68,33 +83,45 @@ # -- jobs ------------------------------------------------------------ android: - image: registry.gitlab.com/fdroid/ci-images-client + image: debian:bullseye-backports variables: + ANDROID_HOME: /usr/lib/android-sdk + DEBIAN_FRONTEND: noninteractive GOPATH: "/go" - ANDROID_VERSION: 29 + LANG: C.UTF-8 + PATH: "/go/bin:/usr/lib/go-1.16/bin:/usr/bin:/bin" cache: paths: - .gradle/wrapper - .gradle/caches + <<: *test-template before_script: - - apt-get -qy update - - apt-get -qy install --no-install-recommends + - *apt-template + - apt-get install + android-sdk-platform-23 + android-sdk-platform-tools build-essential + curl + default-jdk-headless + git gnupg + unzip wget - - cd /usr/local - - export gotarball="go1.16.8.linux-amd64.tar.gz" - - wget -q https://dl.google.com/go/${gotarball} - - wget -q https://dl.google.com/go/${gotarball}.asc - - curl https://dl.google.com/linux/linux_signing_key.pub | gpg --import - - gpg --verify ${gotarball}.asc - - echo "f32501aeb8b7b723bc7215f6c373abb6981bbc7e1c7b44e9f07317e1a300dce2 ${gotarball}" | sha256sum -c - - tar -xzf ${gotarball} - - export PATH="/usr/local/go/bin:$GOPATH/bin:$PATH" # putting this in 'variables:' cause weird runner errors - - cd $CI_PROJECT_DIR + - apt-get install -t bullseye-backports golang-1.16 + + - ndk=android-ndk-r21e-linux-x86_64.zip + - wget --continue --no-verbose https://dl.google.com/android/repository/$ndk + - echo "ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e $ndk" > $ndk.sha256 + - sha256sum -c $ndk.sha256 + - unzip -q $ndk + - rm ${ndk}* + - mv android-ndk-* $ANDROID_HOME/ndk-bundle/ + + - chmod -R a+rX $ANDROID_HOME + script: - *go-test - - export GRADLE_USER_HOME=$PWD/.gradle + - export GRADLE_USER_HOME=$CI_PROJECT_DIR/.gradle - go version - go env @@ -102,18 +129,13 @@ android: - go get golang.org/x/mobile/cmd/gobind - go install golang.org/x/mobile/cmd/gobind - go install golang.org/x/mobile/cmd/gomobile - - echo y | $ANDROID_HOME/tools/bin/sdkmanager 'ndk-bundle' > /dev/null - - echo y | $ANDROID_HOME/tools/bin/sdkmanager "platforms;android-${ANDROID_VERSION}" > /dev/null - gomobile init - - git -C $CI_PROJECT_DIR reset --hard - - git -C $CI_PROJECT_DIR clean -fdx - cd $CI_PROJECT_DIR/client # gomobile builds a shared library not a CLI executable - sed -i 's,^package main$,package snowflakeclient,' snowflake.go - go get golang.org/x/mobile/bind - - gomobile bind -v -target=android . - <<: *test-template + - gomobile bind -v -target=android -trimpath . go-1.13: image: golang:1.13-stretch diff --git a/README.md b/README.md index 0278c04..9a1c958 100644 --- a/README.md +++ b/README.md @@ -87,3 +87,13 @@ abundance of ephemeral and short-lived (and special!) volunteer proxies... ### More info and links We have more documentation in the [Snowflake wiki](https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowf...) and at https://snowflake.torproject.org/. + + +##### -- Android AAR Reproducible Build Setup -- + +Using `gomobile` it is possible to build snowflake as shared libraries for all +the architectures supported by Android. This is in the _.gitlab-ci.yml_, which +runs in GitLab CI. It is also possible to run this setup in a Virtual Machine +using [vagrant](https://www.vagrantup.com/). Just run `vagrant up` and it will +create and provision the VM. `vagrant ssh` to get into the VM to use it as a +development environment. diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 0000000..1b538d5 --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,66 @@ +require 'pathname' +require 'tempfile' +require 'yaml' + +srvpath = Pathname.new(File.dirname(__FILE__)).realpath +configfile = YAML.load_file(File.join(srvpath, "/.gitlab-ci.yml")) +remote_url = 'https://git.torproject.org/pluggable-transports/snowflake.git' + +# set up essential environment variables +env = configfile['android']['variables'] +env['CI_PROJECT_DIR'] = '/builds/tpo/anti-censorship/pluggable-transports/snowflake' +env_file = Tempfile.new('env') +File.chmod(0644, env_file.path) +env.each do |k,v| + env_file.write("export #{k}='#{v}'\n") +end +env_file.rewind + +sourcepath = '/etc/profile.d/env.sh' +header = "#!/bin/bash -ex\nsource #{sourcepath}\ncd $CI_PROJECT_DIR\n" + +before_script_file = Tempfile.new('before_script') +File.chmod(0755, before_script_file.path) +before_script_file.write(header) +configfile['android']['before_script'].flatten.each do |line| + before_script_file.write(line) + before_script_file.write("\n") +end +before_script_file.rewind + +script_file = Tempfile.new('script') +File.chmod(0755, script_file.path) +script_file.write(header) +configfile['android']['script'].flatten.each do |line| + script_file.write(line) + script_file.write("\n") +end +script_file.rewind + +Vagrant.configure("2") do |config| + config.vm.box = "debian/bullseye64" + config.vm.synced_folder '.', '/vagrant', disabled: true + config.vm.provision "file", source: env_file.path, destination: 'env.sh' + config.vm.provision :shell, inline: <<-SHELL + set -ex + mv ~vagrant/env.sh #{sourcepath} + source #{sourcepath} + test -d /go || mkdir /go + mkdir -p $(dirname $CI_PROJECT_DIR) + chown -R vagrant.vagrant $(dirname $CI_PROJECT_DIR) + apt-get update + apt-get -qy install --no-install-recommends git + git clone #{remote_url} $CI_PROJECT_DIR + chmod -R a+rX,u+w /go $CI_PROJECT_DIR + chown -R vagrant.vagrant /go $CI_PROJECT_DIR +SHELL + config.vm.provision "file", source: before_script_file.path, destination: 'before_script.sh' + config.vm.provision "file", source: script_file.path, destination: 'script.sh' + config.vm.provision :shell, inline: '/home/vagrant/before_script.sh' + config.vm.provision :shell, privileged: false, inline: '/home/vagrant/script.sh' + + # remove this or comment it out to use VirtualBox instead of libvirt + config.vm.provider :libvirt do |libvirt| + libvirt.memory = 1536 + end +end