commit 2d98c063010fc5b0f8da3e386587a501e27507b9 Author: Nicolas Vigier <boklm@torproject.org> Date: Tue Apr 25 22:06:10 2017 +0200 Bug 21824: use runc instead of docker --- README | 25 ++---- keyring/ubuntu.gpg | Bin 0 -> 32904 bytes projects/binutils/config | 5 +- projects/cmake/config | 6 +- projects/common/runc-config.json | 179 ++++++++++++++++++++++++++++++++++++++ projects/container-image/build | 3 + projects/container-image/config | 86 ++++++++++++++++++ projects/debootstrap-image/build | 3 + projects/debootstrap-image/config | 52 +++++++++++ projects/docker-image/build | 4 - projects/docker-image/config | 51 ----------- projects/ed25519/config | 5 +- projects/firefox/config | 5 +- projects/fonts/config | 5 +- projects/gcc/config | 5 +- projects/gmp/config | 7 +- projects/go-webrtc/config | 5 +- projects/go/config | 5 +- projects/goerrors/config | 5 +- projects/gogb/config | 5 +- projects/goptlib/config | 5 +- projects/goxcrypto/config | 5 +- projects/goxnet/config | 5 +- projects/https-everywhere/config | 9 +- projects/libdmg-hfsplus/config | 5 +- projects/libevent/config | 7 +- projects/llvm/config | 7 +- projects/macosx-toolchain/config | 5 +- projects/meek/config | 6 +- projects/mingw-w64/config | 5 +- projects/nsis/config | 5 +- projects/obfs4/config | 7 +- projects/openssl/config | 7 +- projects/sandbox/config | 8 +- projects/siphash/config | 5 +- projects/snowflake/config | 7 +- projects/tor-browser/config | 8 +- projects/tor-launcher/config | 6 +- projects/tor/config | 5 +- projects/torbutton/config | 6 +- projects/uniuri/config | 5 +- projects/webrtc/config | 10 ++- projects/yasm/config | 6 +- projects/zlib/config | 7 +- rbm | 2 +- rbm.conf | 93 ++++++++++++++++++-- rbm.local.conf.example | 6 -- tools/clean-old | 21 ----- 48 files changed, 549 insertions(+), 185 deletions(-) diff --git a/README b/README index 08904fb..fa968ff 100644 --- a/README +++ b/README @@ -5,13 +5,12 @@ Installing build dependencies ----------------------------- To build Tor Browser, you need a Linux distribution that has support -for Docker (such as Debian jessie, Ubuntu 14.04, Fedora 20, etc ...). -The Docker package is usually named docker.io or docker-io. -On Debian jessie, the docker.io package is available in backports. +for runC (such as Debian jessie, Ubuntu 14.04, Fedora 20, etc ...). +On Debian jessie, the runc package is available in backports. -Your user account should have access to the docker command without using -sudo, so it should be in the docker group. The docker daemon should -also be running. +Your user account should have access sudo access, which is required to +be able to extract container file systems, start containers and copy +files to and from containers. The sources of most components are downloaded using git, which needs to be installed. The sources of webrtc are downloaded using gclient, which @@ -41,7 +40,7 @@ If you are running Debian or Ubuntu, you can install them with: libio-captureoutput-perl libfile-slurp-perl \ libstring-shellquote-perl libsort-versions-perl \ libdigest-sha-perl libdata-uuid-perl libdata-dump-perl \ - libfile-copy-recursive-perl git libgtk2.0-dev curl + libfile-copy-recursive-perl git libgtk2.0-dev curl runc Starting a build @@ -184,18 +183,6 @@ of files and containers that would be removed without doing it, you can use 'make clean-dry-run'. -Multiple build directories on the same host -------------------------------------------- - -You can do multiple builds of Tor Browser in different directories on -the same host. However the docker images namespace is global, so you -may have some conflicts with the same image names used by the -different builds. By default, the docker images are prefixed with -tor-browser_$USER. You can change this prefix by defining the -docker_image_prefix option in rbm.local.conf, using a different prefix -for each of your build directories. - - Common Build Errors ------------------- diff --git a/keyring/ubuntu.gpg b/keyring/ubuntu.gpg new file mode 100644 index 0000000..8b77bf0 Binary files /dev/null and b/keyring/ubuntu.gpg differ diff --git a/projects/binutils/config b/projects/binutils/config index 2975f14..0cb2088 100644 --- a/projects/binutils/config +++ b/projects/binutils/config @@ -1,9 +1,10 @@ # vim: filetype=yaml sw=2 version: 2.24 filename: '[% project %]-[% c("version") %]-[% c("var/build_id") %].tar.gz' -remote_docker: 1 var: configure_opt: '--disable-multilib --enable-gold --enable-deterministic-archives --enable-plugins' + container: + use_container: 1 targets: windows-i686: @@ -15,7 +16,7 @@ input_files: sig_ext: sig file_gpg_id: 1 gpg_keyring: binutils.gpg - - project: docker-image + - project: container-image - filename: enable-reloc-section-ld.patch enable: '[% c("var/windows") %]' - filename: peXXigen.patch diff --git a/projects/cmake/config b/projects/cmake/config index 837d9e1..357370d 100644 --- a/projects/cmake/config +++ b/projects/cmake/config @@ -5,7 +5,9 @@ git_hash: 'v[% c("version") %]' tag_gpg_id: 1 gpg_keyring: cmake.gpg filename: '[% project %]-[% c("version") %]-[% c("var/build_id") %].tar.gz' -remote_docker: 1 +var: + container: + use_container: 1 input_files: - - project: docker-image + - project: container-image diff --git a/projects/common/runc-config.json b/projects/common/runc-config.json new file mode 100644 index 0000000..4c231cd --- /dev/null +++ b/projects/common/runc-config.json @@ -0,0 +1,179 @@ +{ + "ociVersion": "1.0.0-rc1", + "platform": { + "os": "linux", + "arch": "amd64" + }, + "process": { + "terminal": true, + "user": { + "uid": 0, + "gid": 0 + }, + "args": [ + "/rbm/run" + ], + "env": [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "TERM=xterm" + ], + "cwd": "/", + "capabilities": [ + "CAP_AUDIT_WRITE", + "CAP_KILL", + "CAP_NET_BIND_SERVICE", + "CAP_SETGID", + "CAP_SETUID", + "CAP_MKNOD", + "CAP_SYS_CHROOT", +[% IF c("var/container/CAP_SYS_ADMIN") -%] + "CAP_SYS_ADMIN", +[% END -%] + "CAP_FSETID", + "CAP_FOWNER", + "CAP_DAC_OVERRIDE", + "CAP_CHOWN" + ], + "rlimits": [ + { + "type": "RLIMIT_NOFILE", + "hard": 1024, + "soft": 1024 + } + ], + "noNewPrivileges": true + }, + "root": { + "path": "rootfs", + "readonly": false + }, + "hostname": "runc", + "mounts": [ + { + "destination": "/proc", + "type": "proc", + "source": "proc" + }, + { + "type": "bind", + "source": "/etc/resolv.conf", + "destination": "/etc/resolv.conf", + "options": [ + "rbind", + "ro" + ] + }, + { + "destination": "/dev", + "type": "tmpfs", + "source": "tmpfs", + "options": [ + "nosuid", + "strictatime", + "mode=755", + "size=65536k" + ] + }, + { + "destination": "/dev/pts", + "type": "devpts", + "source": "devpts", + "options": [ + "nosuid", + "noexec", + "newinstance", + "ptmxmode=0666", + "mode=0620", + "gid=5" + ] + }, + { + "destination": "/dev/shm", + "type": "tmpfs", + "source": "shm", + "options": [ + "nosuid", + "noexec", + "nodev", + "mode=1777", + "size=65536k" + ] + }, + { + "destination": "/dev/mqueue", + "type": "mqueue", + "source": "mqueue", + "options": [ + "nosuid", + "noexec", + "nodev" + ] + }, + { + "destination": "/sys", + "type": "sysfs", + "source": "sysfs", + "options": [ + "nosuid", + "noexec", + "nodev", + "ro" + ] + }, + { + "destination": "/sys/fs/cgroup", + "type": "cgroup", + "source": "cgroup", + "options": [ + "nosuid", + "noexec", + "nodev", + "relatime", + "ro" + ] + } + ], + "hooks": {}, + "linux": { + "resources": { + "devices": [ + { + "allow": false, + "access": "rwm" + } + ] + }, + "namespaces": [ + { + "type": "pid" + }, + { + "type": "ipc" + }, + { + "type": "uts" + }, + { + "type": "mount" + } + ], + "maskedPaths": [ + "/proc/kcore", + "/proc/latency_stats", + "/proc/timer_stats", + "/proc/sched_debug" + ], + "readonlyPaths": [ + "/proc/asound", + "/proc/bus", + "/proc/fs", + "/proc/irq", + "/proc/sys", + "/proc/sysrq-trigger" + ] + }, + "solaris": { + "cappedCPU": {}, + "cappedMemory": {} + } +} diff --git a/projects/container-image/build b/projects/container-image/build new file mode 100644 index 0000000..c7d1c46 --- /dev/null +++ b/projects/container-image/build @@ -0,0 +1,3 @@ +#!/bin/sh +set -e +# Doing nothing diff --git a/projects/container-image/config b/projects/container-image/config new file mode 100644 index 0000000..c9f377f --- /dev/null +++ b/projects/container-image/config @@ -0,0 +1,86 @@ +# vim: filetype=yaml sw=2 +filename: 'container-image_[% c("var/container/suite") %]-[% c("var/container/arch") %]-[% sha256(c("pre")).substr(0, 12) %].tar.gz' +pkg_type: build + +var: + container: + use_container: 1 + suite: '[% pc(c("origin_project"), "var/container/suite") %]' + arch: '[% pc(c("origin_project"), "var/container/arch") %]' + +lsb_release: + id: Debian + codename: wheezy + release: 7.11 + +pre: | + #!/bin/sh + # [% c('var/container/suite') %] + set -e + [% IF pc(c('origin_project'), 'var/pre_pkginst') -%] + [% pc(c('origin_project'), 'var/pre_pkginst') %] + [% END -%] + [% IF c("var/container/suite") == "precise" -%] + export INITRD=no + mkdir -p /etc/container_environment + echo -n no > /etc/container_environment/INITRD + dpkg-divert --local --rename --add /sbin/initctl + ln -s /bin/true /sbin/initctl + dpkg-divert --local --rename --add /usr/bin/ischroot + ln -sf /bin/true /usr/bin/ischroot + cat >> /etc/apt/sources.list << EOF + deb http://archive.ubuntu.com/ubuntu/ precise-updates main + deb-src http://archive.ubuntu.com/ubuntu/ precise-updates main + + deb http://archive.ubuntu.com/ubuntu/ precise universe + deb-src http://archive.ubuntu.com/ubuntu/ precise universe + deb http://archive.ubuntu.com/ubuntu/ precise-updates universe + deb-src http://archive.ubuntu.com/ubuntu/ precise-updates universe + + deb http://archive.ubuntu.com/ubuntu/ precise-security main + deb-src http://archive.ubuntu.com/ubuntu/ precise-security main + deb http://archive.ubuntu.com/ubuntu/ precise-security universe + deb-src http://archive.ubuntu.com/ubuntu/ precise-security universe + EOF + [% END -%] + apt-get update -y + apt-get upgrade -y + [% + deps = []; + IF pc(c('origin_project'), 'var/deps'); + CALL deps.import(pc(c('origin_project'), 'var/deps')); + END; + IF pc(c('origin_project'), 'var/arch_deps'); + CALL deps.import(pc(c('origin_project'), 'var/arch_deps')); + END; + IF deps.size; + IF pc(c('origin_project'), 'var/sort_deps'); + deps = deps.sort; + END; + FOREACH pkg IN deps; + SET p = tmpl(pkg); + IF p; + GET c('install_package', { pkg_name => p }); + GET "\n"; + END; + END; + END; + -%] + [% IF pc(c('origin_project'), 'var/post_pkginst') -%] + [% pc(c('origin_project'), 'var/post_pkginst') %] + [% END -%] + +remote_get: | + #!/bin/sh + set -e + [% + SET src = shell_quote(c('get_src', { error_if_undef => 1 })); + SET dst = shell_quote(c('get_dst', { error_if_undef => 1 })); + -%] + mkdir -p "[% dst %]" + sudo tar -C "[% c("var/container/dir") %]/rootfs" -czf "[% dst %]/[% c("filename") %]" . + +input_files: + - project: debootstrap-image + target: + - '[% c("var/container/suite") %]-[% c("var/container/arch") %]' diff --git a/projects/debootstrap-image/build b/projects/debootstrap-image/build new file mode 100644 index 0000000..c7d1c46 --- /dev/null +++ b/projects/debootstrap-image/build @@ -0,0 +1,3 @@ +#!/bin/sh +set -e +# Doing nothing diff --git a/projects/debootstrap-image/config b/projects/debootstrap-image/config new file mode 100644 index 0000000..bd204f5 --- /dev/null +++ b/projects/debootstrap-image/config @@ -0,0 +1,52 @@ +# vim: filetype=yaml sw=2 +filename: 'container-image_[% c("var/container/suite") %]-[% c("var/container/arch") %].tar.gz' +pkg_type: build + +var: + ubuntu_version: 17.04 + + container_dir: '[% c("tmp_dir") %]/rbm-containers/[% sha256(c("build_id")) %]' + container_user: rbm + + container: + use_container: 1 + # We need CAP_SYS_ADMIN for debootstrap to work + CAP_SYS_ADMIN: 1 + +pre: | + #!/bin/sh + set -e + apt-get update -y + apt-get install -y debian-archive-keyring ubuntu-keyring debootstrap + debootstrap --arch=[% c("var/container/arch") %] [% c("var/container/debootstrap_opt") %] [% c("var/container/suite") %] base-image [% c("var/container/debootstrap_mirror") %] + tar -C ./base-image -czf [% dest_dir %]/[% c("filename") %] . + +targets: + wheezy-amd64: + var: + container: + suite: wheezy + arch: amd64 + jessie-amd64: + var: + container: + suite: jessie + arch: amd64 + precise-amd64: + var: + container: + suite: precise + arch: amd64 + debootstrap_opt: --keyring=/usr/share/keyrings/ubuntu-archive-removed-keys.gpg + utopic-amd64: + var: + container: + suite: utopic + arch: amd64 + debootstrap_mirror: http://old-releases.ubuntu.com/ubuntu/ + +input_files: + - URL: 'http://cdimage.ubuntu.com/ubuntu-base/releases/[% c("var/ubuntu_version") %]/release/ubuntu-base-[% c("var/ubuntu_version") %]-base-amd64.tar.gz' + filename: 'container-image_ubuntu-base-[% c("var/ubuntu_version") %]-base-amd64.tar.gz' + sha256sum: df2c8fd540e474b8e1e29c0db8ed6b43a932918f1b9a8149bb82104a7c07ba2a + diff --git a/projects/docker-image/build b/projects/docker-image/build deleted file mode 100644 index ced6ad3..0000000 --- a/projects/docker-image/build +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -set -e -echo 1 > [% dest_dir %]/[% c('filename') %] -echo Creating [% dest_dir %]/[% c('filename') %] diff --git a/projects/docker-image/config b/projects/docker-image/config deleted file mode 100644 index 1968b77..0000000 --- a/projects/docker-image/config +++ /dev/null @@ -1,51 +0,0 @@ -# vim: filetype=yaml sw=2 -filename: '[% sha256(c("pre")).substr(0, 12) %]' -remote_docker: 1 -docker_save_image: '[% c("docker_image_prefix") %]:[% c("filename") %]' -pkg_type: build - -docker_image: '[% c("lsb_release/id").lower %]:[% c("lsb_release/release") %]' - -lsb_release: - id: '[% pc(c("origin_project", { no_distro => 1 }), "lsb_release/id", { no_distro => 1 }) %]' - release: '[% pc(c("origin_project", { no_distro => 1 }), "lsb_release/release", { no_distro => 1 }) %]' - codename: '[% pc(c("origin_project", { no_distro => 1 }), "lsb_release/codename", { no_distro => 1 }) %]' - -pre: | - #!/bin/sh - # [% c('docker_image') %] - set -e - [% IF c('lsb_release/release') == '14.10' -%] - sed -i 's/archive\.ubuntu\.com/old-releases.ubuntu.com/' /etc/apt/sources.list - [% END -%] - [% IF pc(c('origin_project'), 'var/pre_pkginst') -%] - [% pc(c('origin_project'), 'var/pre_pkginst') %] - [% END -%] - [% IF c('lsb_release/id') == 'Ubuntu' || c('lsb_release/id') == 'Debian' %] - apt-get update -y - apt-get upgrade -y - [% END %] - [% - deps = []; - IF pc(c('origin_project'), 'var/deps'); - CALL deps.import(pc(c('origin_project'), 'var/deps')); - END; - IF pc(c('origin_project'), 'var/arch_deps'); - CALL deps.import(pc(c('origin_project'), 'var/arch_deps')); - END; - IF deps.size; - IF pc(c('origin_project'), 'var/sort_deps'); - deps = deps.sort; - END; - FOREACH pkg IN deps; - SET p = tmpl(pkg); - IF p; - GET c('install_package', { pkg_name => p }); - GET "\n"; - END; - END; - END; - -%] - [% IF pc(c('origin_project'), 'var/post_pkginst') -%] - [% pc(c('origin_project'), 'var/post_pkginst') %] - [% END -%] diff --git a/projects/ed25519/config b/projects/ed25519/config index 5bab68b..c6790b8 100644 --- a/projects/ed25519/config +++ b/projects/ed25519/config @@ -3,16 +3,17 @@ version: '[% c("abbrev") %]' git_url: https://github.com/agl/ed25519.git git_hash: c4161f4c7483313562781c61b9a20aba73daf9de filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz' -remote_docker: 1 build: '[% c("projects/go/var/build_go_lib") %]' var: + container: + use_container: 1 go_lib: github.com/agl/ed25519 go_lib_install: - github.com/agl/ed25519/extra25519 input_files: - - project: docker-image + - project: container-image - name: go project: go diff --git a/projects/firefox/config b/projects/firefox/config index 2c74e4b..d6a68fe 100644 --- a/projects/firefox/config +++ b/projects/firefox/config @@ -5,7 +5,6 @@ git_hash: 'tor-browser-[% c("var/firefox_version") %]-[% c("var/torbrowser_branc tag_gpg_id: 1 git_url: https://git.torproject.org/tor-browser.git gpg_keyring: torbutton.gpg -remote_docker: 1 var: firefox_version: 52.1.0esr @@ -19,6 +18,8 @@ var: - autoconf2.13 - yasm - python + container: + use_container: 1 targets: nightly: @@ -74,7 +75,7 @@ targets: martools_filename: mar-tools-win32.zip input_files: - - project: docker-image + - project: container-image - name: '[% c("var/compiler") %]' project: '[% c("var/compiler") %]' - filename: get-moz-build-date diff --git a/projects/fonts/config b/projects/fonts/config index 69e16b3..382804d 100644 --- a/projects/fonts/config +++ b/projects/fonts/config @@ -3,8 +3,9 @@ version: '[% c("abbrev") %]' git_url: https://github.com/googlei18n/noto-fonts.git git_hash: 720e34851382ee3c1ef024d8dffb68ffbfb234c2 filename: "[% project %]-[% c('version') %]-[% c('var/build_id') %].tar.gz" -remote_docker: 1 var: + container: + use_container: 1 noto_fonts_hinted: - Arimo-Regular.ttf - Arimo-Bold.ttf @@ -85,7 +86,7 @@ targets: - NotoSansYi-Regular.ttf input_files: - - project: docker-image + - project: container-image - URL: https://github.com/googlei18n/noto-emoji/raw/2f1ffdd6fbbd05d6f382138a3d3adcd... sha256sum: 415dc6290378574135b64c808dc640c1df7531973290c4970c51fdeb849cb0c5 enable: '[% c("var/linux") %]' diff --git a/projects/gcc/config b/projects/gcc/config index 391e453..1acf30a 100644 --- a/projects/gcc/config +++ b/projects/gcc/config @@ -1,8 +1,9 @@ # vim: filetype=yaml sw=2 filename: '[% project %]-[% c("version") %]-[% c("var/build_id") %].tar.gz' version: 5.1.0 -remote_docker: 1 var: + container: + use_container: 1 configure_opt: --enable-multilib --enable-languages=c,c++ --with-arch_32=i686 deps: - build-essential @@ -42,6 +43,6 @@ targets: var: configure_opt: --disable-multilib --enable-languages=c,c++ input_files: + - project: container-image - URL: 'https://ftp.gnu.org/gnu/gcc/gcc-[% c("version") %]/gcc-[% c("version") %].tar.bz2' sha256sum: b7dafdf89cbb0e20333dbf5b5349319ae06e3d1a30bf3515b5488f7e89dca5ad - - project: docker-image diff --git a/projects/gmp/config b/projects/gmp/config index 913f181..41eb630 100644 --- a/projects/gmp/config +++ b/projects/gmp/config @@ -1,7 +1,10 @@ # vim: filetype=yaml sw=2 filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz' version: 5.1.3 -remote_docker: 1 + +var: + container: + use_container: 1 targets: linux: @@ -9,9 +12,9 @@ targets: configure_opt_gmp: --enable-fat input_files: + - project: container-image - name: gmp URL: 'https://ftp.gnu.org/gnu/gmp/gmp-[% c("version") %].tar.bz2' sha256sum: 752079520b4690531171d0f4532e40f08600215feefede70b24fabdc6f1ab160 - name: '[% c("var/compiler") %]' project: '[% c("var/compiler") %]' - - project: docker-image diff --git a/projects/go-webrtc/config b/projects/go-webrtc/config index 3a1a9d4..d7c31d6 100644 --- a/projects/go-webrtc/config +++ b/projects/go-webrtc/config @@ -3,11 +3,12 @@ version: '[% c("abbrev") %]' git_url: https://github.com/keroserene/go-webrtc.git git_hash: ab1b64862e0c4b4182010699911c2c5818f0a101 filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz' -remote_docker: 1 build: '[% c("projects/go/var/build_go_lib") %]' var: + container: + use_container: 1 go_lib: github.com/keroserene/go-webrtc build_go_lib_pre: | [% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %] @@ -43,7 +44,7 @@ targets: - lib32stdc++6 input_files: - - project: docker-image + - project: container-image - name: go project: go - name: webrtc diff --git a/projects/go/config b/projects/go/config index 89b4b45..ef9c411 100644 --- a/projects/go/config +++ b/projects/go/config @@ -1,10 +1,11 @@ # vim: filetype=yaml sw=2 version: 1.7.5 filename: '[% project %]-[% c("version") %]-[% c("var/build_id") %].tar.gz' -remote_docker: 1 var: go14_version: 1.4.3 + container: + use_container: 1 setup: | mkdir -p /var/tmp/dist @@ -69,7 +70,7 @@ targets: GOARCH: 386 input_files: - - project: docker-image + - project: container-image - name: '[% c("var/compiler") %]' project: '[% c("var/compiler") %]' enable: '[% c("var/windows") || c("var/osx") %]' diff --git a/projects/goerrors/config b/projects/goerrors/config index 4451f7b..3c11fab 100644 --- a/projects/goerrors/config +++ b/projects/goerrors/config @@ -3,14 +3,15 @@ version: '[% c("abbrev") %]' git_url: https://github.com/pkg/errors git_hash: 248dadf4e9068a0b3e79f02ed0a610d935de5302 filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz' -remote_docker: 1 build: '[% c("projects/go/var/build_go_lib") %]' var: + container: + use_container: 1 go_lib: github.com/pkg/errors input_files: - - project: docker-image + - project: container-image - name: go project: go diff --git a/projects/gogb/config b/projects/gogb/config index a358819..dcf30f6 100644 --- a/projects/gogb/config +++ b/projects/gogb/config @@ -3,18 +3,19 @@ version: '[% c("abbrev") %]' git_url: https://github.com/constabulary/gb git_hash: 06cc925cce6592e922dcc4839a8b44feb384e71e filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz' -remote_docker: 1 build: '[% c("projects/go/var/build_go_lib") %]' var: + container: + use_container: 1 go_lib: github.com/constabulary/gb go_lib_install: github.com/constabulary/gb/cmd/gb go_lib_deps: - goerrors input_files: - - project: docker-image + - project: container-image - name: go project: go - name: goerrors diff --git a/projects/goptlib/config b/projects/goptlib/config index dd520ec..c083763 100644 --- a/projects/goptlib/config +++ b/projects/goptlib/config @@ -5,14 +5,15 @@ git_hash: '[% c("version") %]' tag_gpg_id: 1 gpg_keyring: goptlib.gpg filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz' -remote_docker: 1 build: '[% c("projects/go/var/build_go_lib") %]' var: + container: + use_container: 1 go_lib: git.torproject.org/pluggable-transports/goptlib.git input_files: - - project: docker-image + - project: container-image - name: go project: go diff --git a/projects/goxcrypto/config b/projects/goxcrypto/config index 8362f28..b51b578 100644 --- a/projects/goxcrypto/config +++ b/projects/goxcrypto/config @@ -3,11 +3,12 @@ version: '[% c("abbrev") %]' git_url: https://go.googlesource.com/crypto git_hash: 4ed45ec682102c643324fae5dff8dab085b6c300 filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz' -remote_docker: 1 build: '[% c("projects/go/var/build_go_lib") %]' var: + container: + use_container: 1 go_lib: golang.org/x/crypto go_lib_install: - golang.org/x/crypto/curve25519 @@ -19,6 +20,6 @@ targets: git_hash: master input_files: - - project: docker-image + - project: container-image - name: go project: go diff --git a/projects/goxnet/config b/projects/goxnet/config index 507f997..ec368a2 100644 --- a/projects/goxnet/config +++ b/projects/goxnet/config @@ -3,11 +3,12 @@ version: '[% c("abbrev") %]' git_url: https://go.googlesource.com/net git_hash: 7dbad50ab5b31073856416cdcfeb2796d682f844 filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz' -remote_docker: 1 build: '[% c("projects/go/var/build_go_lib") %]' var: + container: + use_container: 1 go_lib: golang.org/x/net go_lib_install: - golang.org/x/net/proxy @@ -17,6 +18,6 @@ targets: git_hash: master input_files: - - project: docker-image + - project: container-image - name: go project: go diff --git a/projects/https-everywhere/config b/projects/https-everywhere/config index 3e5100e..f149cac 100644 --- a/projects/https-everywhere/config +++ b/projects/https-everywhere/config @@ -6,9 +6,11 @@ git_submodule: 1 gpg_keyring: https-everywhere.gpg tag_gpg_id: 1 filename: "[% project %]-[% c('version') %]-[% c('var/build_id') %].xpi" -remote_docker: 1 -distribution: Debian-7.11 var: + container: + use_container: 1 + suite: wheezy + arch: amd64 deps: - git - python @@ -19,8 +21,9 @@ var: - rsync - zip - unzip + input_files: - - project: docker-image + - project: container-image targets: nightly: diff --git a/projects/libdmg-hfsplus/config b/projects/libdmg-hfsplus/config index 9071078..67e8287 100644 --- a/projects/libdmg-hfsplus/config +++ b/projects/libdmg-hfsplus/config @@ -3,13 +3,14 @@ version: '[% c("abbrev") %]' git_url: https://github.com/vasi/libdmg-hfsplus git_hash: dfd5e5cc3dc1191e37d3c3a6118975afdd1d7014 filename: '[% project %]-[% c("version") %]-[% c("var/build_id") %].tar.gz' -remote_docker: 1 var: + container: + use_container: 1 deps: - build-essential - cmake - zlib1g-dev - libbz2-dev input_files: + - project: container-image - filename: libdmg.patch - - project: docker-image diff --git a/projects/libevent/config b/projects/libevent/config index 15ada11..46cbb7e 100644 --- a/projects/libevent/config +++ b/projects/libevent/config @@ -5,7 +5,10 @@ git_hash: 'release-[% c("version") %]-stable' tag_gpg_id: 1 gpg_keyring: libevent.gpg filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz' -remote_docker: 1 + +var: + container: + use_container: 1 targets: osx-x86_64: @@ -14,6 +17,6 @@ targets: - faketime input_files: - - project: docker-image + - project: container-image - name: '[% c("var/compiler") %]' project: '[% c("var/compiler") %]' diff --git a/projects/llvm/config b/projects/llvm/config index 81f7aaa..f0a803b 100644 --- a/projects/llvm/config +++ b/projects/llvm/config @@ -1,10 +1,13 @@ # vim: filetype=yaml sw=2 version: 3.8.0 filename: '[% project %]-[% c("version") %]-[% c("var/build_id") %].tar.gz' -remote_docker: 1 + +var: + container: + use_container: 1 input_files: - - project: docker-image + - project: container-image - project: cmake name: cmake - URL: 'http://releases.llvm.org/[% c("version") %]/llvm-[% c("version") %].src.tar.xz' diff --git a/projects/macosx-toolchain/config b/projects/macosx-toolchain/config index b237e4d..55fb554 100644 --- a/projects/macosx-toolchain/config +++ b/projects/macosx-toolchain/config @@ -1,8 +1,9 @@ # vim: filetype=yaml sw=2 filename: '[% project %]-[% c("version") %]-[% c("var/build_id") %].tar.gz' version: 10.7-1 -remote_docker: 1 var: + container: + use_container: 1 setup: | mkdir -p /var/tmp/dist tar -C /var/tmp/dist -xf [% c("compiler_tarfile") %] @@ -15,6 +16,7 @@ var: export LDFLAGS="[% c('var/LDFLAGS') %]" input_files: + - project: container-image - name: llvm project: llvm - name: cctools @@ -23,4 +25,3 @@ input_files: - name: SDK URL: https://people.torproject.org/~mikeperry/mirrors/sources/MacOSX10.7.sdk.tar.... sha256sum: da77bb0003fcca5ea8c4e8cb2da8828ded750c54afdcac29ec6f3b46ad5e3adf - - project: docker-image diff --git a/projects/meek/config b/projects/meek/config index 84ed2bd..46f4ec9 100644 --- a/projects/meek/config +++ b/projects/meek/config @@ -5,10 +5,12 @@ git_hash: '[% c("version") %]' tag_gpg_id: 1 gpg_keyring: meek.gpg filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz' -remote_docker: 1 +var: + container: + use_container: 1 input_files: - - project: docker-image + - project: container-image - name: go project: go - name: goptlib diff --git a/projects/mingw-w64/config b/projects/mingw-w64/config index 060857c..21d498a 100644 --- a/projects/mingw-w64/config +++ b/projects/mingw-w64/config @@ -3,8 +3,9 @@ filename: '[% project %]-[% c("version") %]-[% c("var/build_id") %].tar.gz' git_url: http://git.code.sf.net/p/mingw-w64/mingw-w64 git_hash: 98e5b4930a717eafddd8ca0f0dfeb7c57c6b026a version: '[% c("abbrev") %]' -remote_docker: 1 var: + container: + use_container: 1 gcc_version: 5.1.0 deps: - automake @@ -17,6 +18,7 @@ var: export PATH="/var/tmp/dist/mingw-w64/helpers:/var/tmp/dist/mingw-w64/bin:$PATH" export gcclibs=/var/tmp/dist/mingw-w64/gcclibs input_files: + - project: container-image - URL: 'https://ftp.gnu.org/gnu/gcc/gcc-[% c("var/gcc_version") %]/gcc-[% c("var/gcc_version") %].tar.bz2' sha256sum: b7dafdf89cbb0e20333dbf5b5349319ae06e3d1a30bf3515b5488f7e89dca5ad - name: binutils @@ -24,4 +26,3 @@ input_files: - filename: i686-w64-mingw32-g++ - filename: i686-w64-mingw32-gcc - filename: i686-w64-mingw32-ld - - project: docker-image diff --git a/projects/nsis/config b/projects/nsis/config index 6729101..2812a22 100644 --- a/projects/nsis/config +++ b/projects/nsis/config @@ -1,8 +1,9 @@ # vim: filetype=yaml sw=2 version: 2.51 filename: 'nsis-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz' -remote_docker: 1 var: + container: + use_container: 1 deps: - build-essential - libmpc-dev @@ -12,6 +13,7 @@ var: - xsltproc input_files: + - project: container-image - filename: 'nsis-[% c("version") %].tar.bz2' URL: 'http://downloads.sourceforge.net/nsis/nsis-[% c("version") %]-src.tar.bz2' sha256sum: 43d4c9209847e35eb6e2c7cd5a7586e1445374c056c2c7899e40a080e17a1be7 @@ -21,4 +23,3 @@ input_files: - filename: nsis-missing-unistd-include.patch - name: '[% c("var/compiler") %]' project: '[% c("var/compiler") %]' - - project: docker-image diff --git a/projects/obfs4/config b/projects/obfs4/config index 952b054..44db79d 100644 --- a/projects/obfs4/config +++ b/projects/obfs4/config @@ -5,7 +5,10 @@ git_hash: 'obfs4proxy-[% c("version") %]' tag_gpg_id: 1 gpg_keyring: obfs4.gpg filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz' -remote_docker: 1 + +var: + container: + use_container: 1 targets: nightly: @@ -13,7 +16,7 @@ targets: tag_gpg_id: 0 input_files: - - project: docker-image + - project: container-image - name: go project: go - name: goptlib diff --git a/projects/openssl/config b/projects/openssl/config index 1a2e4dd..8b4ea4f 100644 --- a/projects/openssl/config +++ b/projects/openssl/config @@ -1,7 +1,10 @@ # vim: filetype=yaml sw=2 version: 1.0.2k filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz' -remote_docker: 1 + +var: + container: + use_container: 1 targets: linux-x86_64: @@ -18,7 +21,7 @@ targets: configure_opts: --cross-compile-prefix=x86_64-apple-darwin10- darwin64-x86_64-cc enable-ec_nistp_64_gcc_128 input_files: - - project: docker-image + - project: container-image - name: '[% c("var/compiler") %]' project: '[% c("var/compiler") %]' - URL: 'https://www.openssl.org/source/openssl-[% c("version") %].tar.gz' diff --git a/projects/sandbox/config b/projects/sandbox/config index 218a276..3970322 100644 --- a/projects/sandbox/config +++ b/projects/sandbox/config @@ -5,10 +5,12 @@ git_hash: 'sandboxed-tor-browser-[% c("version") %]' tag_gpg_id: 1 gpg_keyring: obfs4.gpg filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz' -remote_docker: 1 -distribution: Debian-8.7 var: + container: + use_container: 1 + suite: jessie + arch: amd64 deps: - libx11-dev - pkg-config @@ -22,7 +24,7 @@ targets: tag_gpg_id: 0 input_files: - - project: docker-image + - project: container-image - name: go project: go - name: gogb diff --git a/projects/siphash/config b/projects/siphash/config index 0f3f4b5..b2cb2a9 100644 --- a/projects/siphash/config +++ b/projects/siphash/config @@ -3,14 +3,15 @@ version: '[% c("abbrev") %]' git_url: https://github.com/dchest/siphash.git git_hash: 42ba037e748c9062a75e0924705c43b893edefcd filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz' -remote_docker: 1 build: '[% c("projects/go/var/build_go_lib") %]' var: + container: + use_container: 1 go_lib: github.com/dchest/siphash input_files: - - project: docker-image + - project: container-image - name: go project: go diff --git a/projects/snowflake/config b/projects/snowflake/config index 3233ba4..212e86c 100644 --- a/projects/snowflake/config +++ b/projects/snowflake/config @@ -3,7 +3,10 @@ version: '[% c("abbrev") %]' git_url: https://git.torproject.org/pluggable-transports/snowflake.git git_hash: 9f2e9a6ecb696149708716ca06ce842df03cf492 filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz' -remote_docker: 1 + +var: + container: + use_container: 1 targets: linux-i686: @@ -19,7 +22,7 @@ targets: - libx11-dev input_files: - - project: docker-image + - project: container-image - name: go project: go - name: '[% c("var/compiler") %]' diff --git a/projects/tor-browser/config b/projects/tor-browser/config index 87cc6b2..c4c2521 100644 --- a/projects/tor-browser/config +++ b/projects/tor-browser/config @@ -1,9 +1,10 @@ # vim: filetype=yaml sw=2 version: '[% c("var/torbrowser_version") %]' filename: 'tor-browser-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %]' -remote_docker: 1 var: + container: + use_container: 1 ddmg: '[% INCLUDE ddmg.sh %]' targets: @@ -24,12 +25,13 @@ targets: - bzip2 - faketime windows-i686: - distribution: Ubuntu-14.10 var: mar_osname: win32 + container: + suite: utopic input_files: - - project: docker-image + - project: container-image - project: firefox name: firefox - project: tor diff --git a/projects/tor-launcher/config b/projects/tor-launcher/config index 70f8acd..9211844 100644 --- a/projects/tor-launcher/config +++ b/projects/tor-launcher/config @@ -5,9 +5,11 @@ git_hash: '[% c("version") %]' gpg_keyring: torbutton.gpg tag_gpg_id: 1 filename: "[% project %]-[% c('version') %]-[% c('var/build_id') %].xpi" -remote_docker: 1 +var: + container: + use_container: 1 input_files: - - project: docker-image + - project: container-image targets: nightly: diff --git a/projects/tor/config b/projects/tor/config index c8940dd..a9da811 100644 --- a/projects/tor/config +++ b/projects/tor/config @@ -5,9 +5,10 @@ git_hash: 'tor-[% c("version") %]' git_url: https://git.torproject.org/tor.git gpg_keyring: tor.gpg tag_gpg_id: 1 -remote_docker: 1 var: + container: + use_container: 1 deps: - build-essential - automake @@ -50,6 +51,7 @@ targets: flag_mwindows: '' input_files: + - project: container-image - name: openssl project: openssl - name: libevent @@ -59,4 +61,3 @@ input_files: enable: '[% c("var/windows") %]' - name: '[% c("var/compiler") %]' project: '[% c("var/compiler") %]' - - project: docker-image diff --git a/projects/torbutton/config b/projects/torbutton/config index b60e217..41d4d69 100644 --- a/projects/torbutton/config +++ b/projects/torbutton/config @@ -5,9 +5,11 @@ git_hash: '[% c("version") %]' gpg_keyring: torbutton.gpg tag_gpg_id: 1 filename: "[% project %]-[% c('version') %]-[% c('var/build_id') %].xpi" -remote_docker: 1 +var: + container: + use_container: 1 input_files: - - project: docker-image + - project: container-image targets: nightly: diff --git a/projects/uniuri/config b/projects/uniuri/config index 62fa8ef..e4c7294 100644 --- a/projects/uniuri/config +++ b/projects/uniuri/config @@ -3,11 +3,12 @@ version: '[% c("abbrev") %]' git_url: https://github.com/dchest/uniuri git_hash: 8902c56451e9b58ff940bbe5fec35d5f9c04584a filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz' -remote_docker: 1 build: '[% c("projects/go/var/build_go_lib") %]' var: + container: + use_container: 1 go_lib: github.com/dchest/uniuri targets: @@ -15,6 +16,6 @@ targets: git_hash: master input_files: - - project: docker-image + - project: container-image - name: go project: go diff --git a/projects/webrtc/config b/projects/webrtc/config index bfd38c9..d46d821 100644 --- a/projects/webrtc/config +++ b/projects/webrtc/config @@ -1,13 +1,14 @@ # vim: filetype=yaml sw=2 version: '[% c("var/webrtc_tag") %]' -remote_docker: 1 filename: 'webrtc-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz' var: + container: + use_container: 1 webrtc_tag: c279861207c5b15fc51069e96595782350e0ac12 input_files: - - project: docker-image + - project: container-image - project: webrtc pkg_type: fetch_sources - project: depot_tools @@ -27,7 +28,6 @@ targets: os: linux linux-i686: var: - dockerbuild: "[% pc('docker-image', 'pre') %]" sort_deps: 0 arch_deps: - lib32asound2-dev @@ -65,7 +65,9 @@ targets: steps: fetch_sources: - remote_docker: 0 + var: + container: + use_container: 0 filename: 'webrtc-sources-[% c("var/webrtc_tag") %].tar.gz' fetch_sources: | #!/bin/bash diff --git a/projects/yasm/config b/projects/yasm/config index 3d8a28a..12d009b 100644 --- a/projects/yasm/config +++ b/projects/yasm/config @@ -1,10 +1,12 @@ # vim: filetype=yaml sw=2 version: 1.2.0 filename: '[% project %]-[% c("version") %]-[% c("var/build_id") %].tar.gz' -remote_docker: 1 +var: + container: + use_container: 1 input_files: - - project: docker-image + - project: container-image - URL: 'https://www.tortall.net/projects/yasm/releases/yasm-[% c("version") %].tar.gz' name: yasm sha256sum: 768ffab457b90a20a6d895c39749adb547c1b7cb5c108e84b151a838a23ccf31 diff --git a/projects/zlib/config b/projects/zlib/config index 5219559..3ad562c 100644 --- a/projects/zlib/config +++ b/projects/zlib/config @@ -5,9 +5,12 @@ git_hash: 'v[% c("version") %]' git_url: https://github.com/madler/zlib.git gpg_keyring: zlib.gpg tag_gpg_id: 1 -remote_docker: 1 + +var: + container: + use_container: 1 input_files: + - project: container-image - name: '[% c("var/compiler") %]' project: '[% c("var/compiler") %]' - - project: docker-image diff --git a/rbm b/rbm index 3f3886e..106e9b0 160000 --- a/rbm +++ b/rbm @@ -1 +1 @@ -Subproject commit 3f3886e1f210ad2853209c5aecd0951350a6f758 +Subproject commit 106e9b05aeff6309e241a3c9bae1781e0d551e7a diff --git a/rbm.conf b/rbm.conf index 651acfa..722c85a 100644 --- a/rbm.conf +++ b/rbm.conf @@ -20,12 +20,16 @@ var: build_id_txt: | [% c("version") %] [% IF c("git_hash") || c("hg_hash"); GET c("abbrev"); END; %] - [% IF c("remote_docker") -%] - [% c("distribution") %] + [% IF c("var/container/use_container") -%] + [% c("var/container/suite") %] + [% c("var/container/arch") %] [% END -%] input_files: [% c("input_files_id") %] build: [% c("build", { filename => 'f', output_dir => '/out' }) %] + container: + dir: '[% c("tmp_dir") %]/rbm-containers/[% sha256(c("build_id")) %]' + user: rbm input_files_list: | [% FOREACH file IN c("input_files_by_name").keys.sort -%] [% c("input_files_by_name/" _ file) %] @@ -136,19 +140,23 @@ targets: - zip - unzip linux: - distribution: Debian-7.11 var: linux: 1 compiler: gcc + container: + suite: wheezy + arch: amd64 torbrowser-windows-i686: - windows-i686 windows-i686: - distribution: Ubuntu-12.04 arch: i686 var: windows: 1 osname: windows-i686 + container: + suite: precise + arch: amd64 configure_opt: '--host=i686-w64-mingw32 CFLAGS="[% c("var/CFLAGS") %]" LDFLAGS="[% c("var/LDFLAGS") %]"' CFLAGS: '[% c("var/flag_mwindows") %] -fstack-protector-all -Wstack-protector --param ssp-buffer-size=4 -fno-strict-overflow -Wno-missing-field-initializers -Wformat -Wformat-security' LDFLAGS: '[% c("var/flag_mwindows") %] -Wl,--dynamicbase -Wl,--nxcompat -Wl,--enable-reloc-section -lssp -L$gcclibs' @@ -167,11 +175,13 @@ targets: torbrowser-osx-x86_64: - osx-x86_64 osx-x86_64: - distribution: Debian-8.7 arch: x86_64 var: osx: 1 osname: osx-x86_64 + container: + suite: jessie + arch: amd64 compiler: 'macosx-toolchain' configure_opt: '--host=x86_64-apple-darwin10 CC="x86_64-apple-darwin10-clang [% c("var/FLAGS") %]" CXX="x86_64-apple-darwin10-clang++ [% c("var/FLAGS") %]"' FLAGS: "-target x86_64-apple-darwin10 -mlinker-version=136 -B $cctoolsdir -isysroot $sysrootdir" @@ -195,9 +205,6 @@ targets: build_id: 1 -docker_image: '[% pc("docker-image", "docker_save_image") %]' -docker_image_prefix: '[% GET c("var/project_name") ? c("var/project_name") : "rbm-build" %]_[% GET ENV.RBM_BUILDNAME ? ENV.RBM_BUILDNAME : ENV.USER ? ENV.USER : c("uid") %]' - # change the default gpg_wrapper to allow git tag signed using an # expired key. # https://bugs.torproject.org/19737 @@ -218,6 +225,76 @@ gpg_wrapper: | exec [% c('gpg_bin') %] [% c('gpg_args') %] --with-fingerprint [% gpg_kr %] "$@" fi +remote_start: '[% IF c("var/container/use_container") %][% c("runc/remote_start") %][% END %]' +remote_exec: '[% IF c("var/container/use_container") %][% c("runc/remote_exec") %][% END %]' +remote_put: '[% IF c("var/container/use_container") %][% c("runc/remote_put") %][% END %]' +remote_get: '[% IF c("var/container/use_container") %][% c("runc/remote_get") %][% END %]' +remote_finish: '[% IF c("var/container/use_container") %][% c("runc/remote_finish") %][% END %]' + +runc: + remote_start: | + #!/bin/sh + set -e + if [ $(ls -1 '[% c("remote_srcdir", { error_if_undef => 1 }) %]/container-image_'* | wc -l) -ne 1 ] + then + echo "Can't find container image in input files" >&2 + ls -l '[% c("remote_srcdir") %]' >&2 + exit 1 + fi + mkdir -p '[% c("var/container/dir") %]'/rootfs/rbm + sudo tar -C '[% c("var/container/dir") %]'/rootfs -xf $(ls -1 '[% c("remote_srcdir", { error_if_undef => 1 }) %]/container-image_'*) + cat > '[% c("var/container/dir") %]'/config.json << EOF + [% INCLUDE 'runc-config.json' %] + EOF + [% SET user = c("var/container/user") -%] + [% c("remote_exec", { exec_as_root => 1, exec_cmd => 'id ' _ user + _ ' >/dev/null 2>&1 || adduser -m ' _ user _ ' || useradd -m ' _ user }) %] + + remote_exec: | + #!/bin/sh + set -e + mkdir -p '[% c("var/container/dir", { error_if_undef => 1 }) %]'/rootfs/rbm + echo '#!/bin/sh' > '[% c("var/container/dir") %]'/rootfs/rbm/cmd + echo [% shell_quote(c('exec_cmd')) %] >> '[% c("var/container/dir") %]'/rootfs/rbm/cmd + echo '#!/bin/sh' > '[% c("var/container/dir") %]'/rootfs/rbm/run + [% IF c('exec_as_root'); SET user = 'root'; ELSE; SET user = c("var/container/user", { error_if_undef => 1 }); END; %] + echo 'su - [% user %] -c /rbm/cmd' >> '[% c("var/container/dir") %]'/rootfs/rbm/run + chmod +x '[% c("var/container/dir") %]'/rootfs/rbm/cmd + chmod +x '[% c("var/container/dir") %]'/rootfs/rbm/run + sudo runc start -b '[% c("var/container/dir") %]' rbm-[% sha256(c("build_id", { error_if_undef => 1 })) %] + + remote_put: | + #!/bin/sh + set -e + [% + SET src = shell_quote(c('put_src', { error_if_undef => 1 })); + SET dst = shell_quote(c('put_dst', { error_if_undef => 1 })); + -%] + sudo mkdir -p '[% c("var/container/dir") %]'/rootfs/[% dst %] + sudo cp -aP [% src %] '[% c("var/container/dir") %]'/rootfs/[% dst %] + [% c("remote_exec", { exec_as_root => 1, exec_cmd => 'chown -R ' _ c("var/container/user") _ ' ' _ dst }) %] + + remote_get: | + #!/bin/sh + set -e + [% + SET src = shell_quote(c('get_src', { error_if_undef => 1 })); + SET dst = shell_quote(c('get_dst', { error_if_undef => 1 })); + -%] + mkdir -p [% dst %] + srcdir='[% c("var/container/dir", { error_if_undef => 1 }) %]'/rootfs/[% src %] + if [ $(ls -1 "$srcdir"/* 2> /dev/null | wc -l) -gt 0 ] + then + sudo chown $(whoami) "$srcdir"/* + sudo mv -f "$srcdir"/* [% dst %]/ + fi + + remote_finish: | + #!/bin/sh + set -e + sudo rm -Rf '[% c("var/container/dir", { error_if_undef => 1 }) %]'/rootfs '[% c("var/container/dir", { error_if_undef => 1 }) %]'/config.json + rmdir '[% c("var/container/dir") %]' + ENV: TZ: UTC LC_ALL: C diff --git a/rbm.local.conf.example b/rbm.local.conf.example index 78de08f..203ce5f 100644 --- a/rbm.local.conf.example +++ b/rbm.local.conf.example @@ -16,12 +16,6 @@ ### this. #debug: 0 -### If you are doing multiple builds in different directories on the -### same host, you should define docker_image_prefix with a different -### value for each build directory, so that the different builds don't -### use the same docker image names. -#docker_image_prefix: tor-browser_XXXXX - ### The build_log option defines in which file the build logs of each ### component are stored. If you set it to '-' the logs are output on ### stdout and stderr. diff --git a/tools/clean-old b/tools/clean-old index 4d603fc..c7d9e0c 100755 --- a/tools/clean-old +++ b/tools/clean-old @@ -27,24 +27,6 @@ sub clean_file { } } -sub clean_docker_images { - my ($dockerdir, $used_files) = @_; - my $imgprefix = RBM::project_config('docker-image', 'docker_image_prefix'); - my @imgs = read_dir($dockerdir); - foreach my $dockerimage (@imgs) { - next if $used_files->{"$dockerdir/$dockerimage"}; - my $img = "$imgprefix:$dockerimage"; - print "Cleaning docker image $img\n"; - next if $options{'dry-run'}; - my ($out, $err, $success) = capture_exec('docker', 'rmi', '-f', $img); - if (!$success) { - print STDERR "Error removing docker image $img:\n$err\n"; - exit 1; - } - unlink "$dockerdir/$dockerimage"; - } -} - sub get_project_input_files { my ($project, @targets) = @_; print "Getting input files for $project ", join(' ', @targets), "\n"; @@ -108,7 +90,4 @@ foreach my $branch (keys %$clean) { } my %used_files = map { $_ => 1 } @files; my $outdir = $RBM::config->{basedir} . '/out'; -# Don't clean docker-image files yet -$used_files{"$outdir/docker-image"} = 1; clean_file($outdir, \%used_files); -clean_docker_images("$outdir/docker-image", \%used_files);