commit a2f565643d4c34a49712c37e3931dd6f80da7ba1 Author: Nicolas Vigier <boklm@torproject.org> Date: Mon Apr 16 19:30:02 2018 +0200 Bug 25817: add ansible roles for tbb-nightly setup --- README | 43 ++------ tools/ansible/Makefile | 3 + tools/ansible/README | 6 ++ tools/ansible/boklm-tbb-nightly-build.yml | 7 ++ .../boklm-tbb-nightly/tbb-nightly-build.yml | 3 + tools/ansible/inventory | 4 + .../roles/tbb-nightly-build/defaults/main.yml | 11 ++ .../roles/tbb-nightly-build/handlers/main.yml | 5 + .../ansible/roles/tbb-nightly-build/tasks/main.yml | 84 +++++++++++++++ .../roles/tbb-nightly-build/templates/nginx.conf | 16 +++ .../tbb-nightly-build/templates/rbm.local.conf | 116 +++++++++++++++++++++ .../tbb-nightly-build/templates/start-tbb-nightly | 5 + .../tbb-nightly-build/templates/testsuite-config | 27 +++++ .../tbb-nightly-build/templates/www-index.html | 10 ++ 14 files changed, 306 insertions(+), 34 deletions(-) diff --git a/README b/README index d498c05..a20b659 100644 --- a/README +++ b/README @@ -138,43 +138,18 @@ Automated builds using tbb-testsuite ------------------------------------ The Tor Browser testsuite scripts can also be used to do nightly builds -and publish the build logs. +and publish the build logs. The recommended way to do that is to use +the ansible roles from the tools/ansible directory. See next section +for details. -If you want to do that, start by cloning the git repository: - $ git clone https://git.torproject.org/boklm/tor-browser-bundle-testsuite.git +Using ansible to set up a nightly build machine +----------------------------------------------- -Install some dependencies: - - # apt-get install -y libdata-dump-perl libfile-slurp-perl \ - libio-captureoutput-perl perlmagick libjson-perl \ - libwww-perl liblwp-protocol-https-perl libtemplate-perl \ - libyaml-syck-perl libdatetime-perl \ - libemail-sender-perl libemail-simple-perl libfile-type-perl \ - libipc-run-perl libxml-libxml-perl - -Copy the config/tor-browser_build-boklm file and edit it: - - $ cd tor-browser-bundle-testsuite - $ cp config/tor-browser_build-boklm config/tor-browser_build-$user - $ vim config/tor-browser_build-$user - -Change the publish_dir and publish_url options. The publish_dir option is -the local directory where the builds will be stored. The publish_url -option is the public URL where the builds will be available. - -Copy the tools/tor-browser-builds-boklm file and edit it to change the ---config= option: - - $ cp tools/tor-browser-builds-boklm tools/tor-browser-builds-$user - $ vim tools/tor-browser-builds-$user - -You can now run ./tools/tor-browser-builds-$user to start the build, and -add it to your crontab. - -The html build reports will be available in the reports/ directory, and -the build files in the tor-browser-builds/ directory (unless you changed -the publish_dir option). +The directory tools/ansible contains some ansible roles to set up a +nightly build machine. You can look at the playbook defined in +boklm-tbb-nightly-build.yml and variables in group_vars/boklm-tbb-nightly/ +for an example of how it can be used. Signing builds diff --git a/tools/ansible/Makefile b/tools/ansible/Makefile index 608f932..72deb5b 100644 --- a/tools/ansible/Makefile +++ b/tools/ansible/Makefile @@ -3,3 +3,6 @@ ansible-tbb-build: fpcentral: ANSIBLE_CONFIG='$(@D)/ansible-fpcentral.cfg' ansible-playbook -i inventory --ask-become-pass fpcentral.yml + +boklm-tbb-nightly-build: + ansible-playbook -i inventory boklm-tbb-nightly-build.yml diff --git a/tools/ansible/README b/tools/ansible/README index 6b2e6a4..dc69ad7 100644 --- a/tools/ansible/README +++ b/tools/ansible/README @@ -16,3 +16,9 @@ fpcentral: you need to be in the fpcentral tpo ldap group. Your ldap password will be asked, to sudo to the fpcentral user. +boklm-tbb-nightly-build: + This target is used by boklm to deploy a nightly build machine. If + anybody else wants to set up their own nightly builds, it is possible to + do it by adding a new host to the inventory file and making copies of + group_vars/boklm-tbb-nightly/ and boklm-tbb-nightly-build.yml. + diff --git a/tools/ansible/boklm-tbb-nightly-build.yml b/tools/ansible/boklm-tbb-nightly-build.yml new file mode 100644 index 0000000..cc37e23 --- /dev/null +++ b/tools/ansible/boklm-tbb-nightly-build.yml @@ -0,0 +1,7 @@ +--- + +- hosts: boklm-tbb-nightly-build + roles: + - role: tbb-builder + - role: tbb-nightly-build + - role: unattended-upgrades diff --git a/tools/ansible/group_vars/boklm-tbb-nightly/tbb-nightly-build.yml b/tools/ansible/group_vars/boklm-tbb-nightly/tbb-nightly-build.yml new file mode 100644 index 0000000..ebaadfe --- /dev/null +++ b/tools/ansible/group_vars/boklm-tbb-nightly/tbb-nightly-build.yml @@ -0,0 +1,3 @@ +--- +nightly_build_hostname: f4amtbsowhix7rrf.onion +nightly_build_url: 'http://{{ nightly_build_hostname }}' diff --git a/tools/ansible/inventory b/tools/ansible/inventory index 32a5805..fc25842 100644 --- a/tools/ansible/inventory +++ b/tools/ansible/inventory @@ -1,8 +1,12 @@ build-sunet-a ansible_ssh_user=root ansible_ssh_host=build-sunet-a.torproject.net fpcentral ansible_become=True ansible_become_method=sudo ansible_become_user=fpcentral ansible_ssh_host=forrestii.torproject.org allow_world_readable_tmpfiles=True +boklm-tbb-nightly-build ansible_ssh_user=root ansible_become_method=su [tbb-build] build-sunet-a [fpcentral] fpcentral + +[boklm-tbb-nightly] +boklm-tbb-nightly-build diff --git a/tools/ansible/roles/tbb-nightly-build/defaults/main.yml b/tools/ansible/roles/tbb-nightly-build/defaults/main.yml new file mode 100644 index 0000000..d84a161 --- /dev/null +++ b/tools/ansible/roles/tbb-nightly-build/defaults/main.yml @@ -0,0 +1,11 @@ +--- +nightly_build_user: tbb-nightly +nightly_build_cron_hour: 2 +nightly_build_cron_minute: 20 +nightly_build_keep_builds: 2 +testsuite_dir: "/home/{{ nightly_build_user }}/tbb-testsuite" +testsuite_git_url: https://git.torproject.org/boklm/tor-browser-bundle-testsuite.git +testsuite_git_commit: 348ad855711382089c4fbf1badfec58e31a6c148 +nightly_build_wwwdir: "/home/{{ nightly_build_user }}/www" +nightly_build_nginx_enable: true +nightly_build_nginx_listen: 127.0.0.1:80 diff --git a/tools/ansible/roles/tbb-nightly-build/handlers/main.yml b/tools/ansible/roles/tbb-nightly-build/handlers/main.yml new file mode 100644 index 0000000..3350d73 --- /dev/null +++ b/tools/ansible/roles/tbb-nightly-build/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: restart nginx + service: + name: nginx + state: restarted diff --git a/tools/ansible/roles/tbb-nightly-build/tasks/main.yml b/tools/ansible/roles/tbb-nightly-build/tasks/main.yml new file mode 100644 index 0000000..64d31a9 --- /dev/null +++ b/tools/ansible/roles/tbb-nightly-build/tasks/main.yml @@ -0,0 +1,84 @@ +--- +- name: create tbb-nightly user + user: + name: "{{ nightly_build_user }}" + comment: "Tor Browser Nightly Builds" + groups: tbb-build + createhome: yes + home: "/home/{{ nightly_build_user }}" + +- name: clone tor browser testsuite + become: yes + become_user: "{{ nightly_build_user }}" + git: + repo: "{{ testsuite_git_url }}" + dest: "{{ testsuite_dir }}" + version: "{{ testsuite_git_commit }}" + +- name: install testsuite dependencies + command: "{{ testsuite_dir }}/install-deps" + +- name: add testsuite config + template: + src: testsuite-config + dest: "{{ testsuite_dir }}/config/tbb-nightly" + mode: 0644 + owner: "{{ nightly_build_user }}" + +- name: create rbm-config directory + file: + path: '{{ testsuite_dir }}/rbm-config/' + state: directory + owner: '{{ nightly_build_user }}' + mode: 0755 + +- name: add rbm config + template: + src: rbm.local.conf + dest: "{{ testsuite_dir }}/rbm-config/tbb-nightly.rbm.local.conf" + mode: 0644 + owner: "{{ nightly_build_user }}" + +- name: add start-tbb-nightly script + template: + src: start-tbb-nightly + dest: "/home/{{ nightly_build_user }}/start-tbb-nightly" + mode: 0755 + owner: "{{ nightly_build_user }}" + +- name: add cron to start nighly build + cron: + name: tbb-nightly-build + user: "{{ nightly_build_user }}" + hour: "{{ nightly_build_cron_hour }}" + minute: "{{ nightly_build_cron_minute }}" + job: "/home/{{ nightly_build_user }}/start-tbb-nightly" + +- name: create wwwdir + file: + path: '{{ nightly_build_wwwdir }}' + state: directory + owner: '{{ nightly_build_user }}' + mode: 0755 + +- name: add index page + template: + src: www-index.html + dest: "{{ nightly_build_wwwdir }}/index.html" + mode: 0644 + owner: "{{ nightly_build_user }}" + +- name: install nginx + apt: + name: nginx + state: present + when: nightly_build_nginx_enable + +- name: add nginx config + template: + src: nginx.conf + dest: /etc/nginx/sites-enabled/tbb-nightly-build + mode: 0644 + notify: + - restart nginx + when: nightly_build_nginx_enable diff --git a/tools/ansible/roles/tbb-nightly-build/templates/nginx.conf b/tools/ansible/roles/tbb-nightly-build/templates/nginx.conf new file mode 100644 index 0000000..9d44bec --- /dev/null +++ b/tools/ansible/roles/tbb-nightly-build/templates/nginx.conf @@ -0,0 +1,16 @@ +server { + listen {{ nightly_build_nginx_listen }}; + server_name {{ nightly_build_hostname }}; + root {{ nightly_build_wwwdir }}; + index index.html; + location / { + try_files $uri $uri/ =404; + } + location /reports { + alias {{ testsuite_dir }}/reports; + } + location /tor-browser-builds { + alias {{ testsuite_dir }}/tor-browser-builds; + autoindex on; + } +} diff --git a/tools/ansible/roles/tbb-nightly-build/templates/rbm.local.conf b/tools/ansible/roles/tbb-nightly-build/templates/rbm.local.conf new file mode 100644 index 0000000..fcd6b34 --- /dev/null +++ b/tools/ansible/roles/tbb-nightly-build/templates/rbm.local.conf @@ -0,0 +1,116 @@ +--- +### This file is used to override options from rbm.conf to adapt them +### to your local setup. +### +### Copy this file as rbm.local.conf to enable it, and uncomment the +### options you want to modify. + + +### The tmp_dir option defines where temporary files are stored. The +### builds are made from this directory, so using a directory on a fast +### disk can improve build time. By default we are using a tmp directory +### under the tor-browser-build directory. +#tmp_dir: /tmp + +### The debug option defines whether a debugging shell should be opened +### automatically in the build directory/container in case of build +### failure. If you are doing automated builds, you might want to disable +### this. +#debug: 0 + +### The build_log option defines in which file the build logs of each +### component are stored. If you set it to '-' the logs are output on +### stdout and stderr. +#build_log: '-' + +### By default, the logs from previous builds are kept in the log files. +### If you set build_log_append to 0, then previous logs are cleaned +### when starting a new build. +#build_log_append: 0 + +buildconf: + buildconf: 1 + + ### The buildconf/num_procs option can be used to select the number of + ### build processes to run simultaneously. You can also use the + ### RBM_NUM_PROCS environment variable. The default is 4. + #num_procs: 8 + + ### The buildconf/git_signtag_opt option is useful when you tag a release. + ### It is used to set the 'git tag' argument to select the keyid for + ### signing the tag. + #git_signtag_opt: '-u keyid' + +var: + local_conf: 1 + + ### The var/sign_build option defines if you want to sign the + ### sha256sums-unsigned-build.txt file with gpg. + #sign_build: 1 + + ### The var/sign_build_gpg_opts option can be used to define some gpg + ### options to select the key to use to sign the sha256sums-unsigned-build.txt + ### file. + #sign_build_gpg_opts: '--local-user XXXXXXXX' + + ### The clean configuration is used by the cleaning script to find the + ### branches and build targets you are using, to compute the list of + ### files that should be kept. + ### + ### If you only do alpha builds for all platforms, you can use the + ### following configuration: + clean: + HEAD: + - project: release + target: + - alpha + - torbrowser-all + - project: release + target: + - nightly + - torbrowser-all + # + ### If you are doing 'release' builds in the maint-7.0 branch and + ### 'alpha' builds in the master branch, you can use the following + ### configuration: + #clean: + # master: + # - project: release + # target: + # - alpha + # - torbrowser-all + # maint-7.0: + # - project: release + # target: + # - release + # - torbrowser-all + +targets: + + ### testbuild is based on alpha by default. Uncomment this if you want it + ### to be based on nightly. + #torbrowser-testbuild: + # - testbuild + # - nightly + + testbuild: + var: + testbuild: 1 + + ### Uncomment this if you want to create mar files in your test build. + #build_mar: 1 + + nightly: + + ### By default 'fetch' is set to 1 for nightly builds, meaning that new + ### commits will be fetched automatically during each build. You can + ### disable this during development if you want to do rebuilds to test + ### a specific change, but don't want rebuilds caused by unrelated + ### changes, or if you want to decide at which point new commits are + ### fetched. When 'fetch' is set to 'if_needed', new commits will only + ### be fetched if the selected commit (or branch, or tag) is not present, + ### which means that existing branches won't be updated. In that case + ### you can fetch new commits by running 'make fetch'. + fetch: 'if_needed' + +# vim: filetype=yaml sw=2 diff --git a/tools/ansible/roles/tbb-nightly-build/templates/start-tbb-nightly b/tools/ansible/roles/tbb-nightly-build/templates/start-tbb-nightly new file mode 100644 index 0000000..3c1847d --- /dev/null +++ b/tools/ansible/roles/tbb-nightly-build/templates/start-tbb-nightly @@ -0,0 +1,5 @@ +#!/bin/sh +cd {{ testsuite_dir }} +export RBM_NO_DEBUG=1 +./tbb-testsuite --config=tbb-nightly "$@" +./tools/prune-old-builds/prune-old-builds --prefix '' --days {{ nightly_build_keep_builds }} ./tor-browser-builds diff --git a/tools/ansible/roles/tbb-nightly-build/templates/testsuite-config b/tools/ansible/roles/tbb-nightly-build/templates/testsuite-config new file mode 100644 index 0000000..b537f56 --- /dev/null +++ b/tools/ansible/roles/tbb-nightly-build/templates/testsuite-config @@ -0,0 +1,27 @@ +# vim: filetype=perl expandtab +use strict; +use FindBin; +use DateTime; +use TBBTestSuite::TestSuite::TorBrowserBuild; + +my $date = DateTime->now->ymd; +my $name = "tor-browser-$date"; + +if (-d "$options->{'reports-dir'}/r/$name") { + print "Doing nothing: $name already done\n"; + return ( args => [] ); +} + +my $testsuite = TBBTestSuite::TestSuite::TorBrowserBuild->new({ + publish_dir => "$FindBin::Bin/tor-browser-builds/$date", + publish_url => "{{ nightly_build_url }}/tor-browser-builds/$date", + rbm_local_conf => "$FindBin::Bin/rbm-config/tbb-nightly.rbm.local.conf", + make_clean => 1, + }); + +my %res = ( + name => $name, + args => [ $testsuite ], + tags => [ 'nightly' ], +); +%res; diff --git a/tools/ansible/roles/tbb-nightly-build/templates/www-index.html b/tools/ansible/roles/tbb-nightly-build/templates/www-index.html new file mode 100644 index 0000000..05050d8 --- /dev/null +++ b/tools/ansible/roles/tbb-nightly-build/templates/www-index.html @@ -0,0 +1,10 @@ +<html> +<head> + <title>Tor Browser Nightly Builds</title> +</head> +<body> + <h1>Tor Browser Nightly Builds</title> + <a href="reports/"><h3>Build Reports</h3></a> + <a href="tor-browser-builds/"><h3>Build files</h3></a> +</body> +</html>