[tor-commits] [tor-browser-build/master] Bug 25817: add ansible roles for tbb-nightly setup

gk at torproject.org gk at torproject.org
Tue May 8 06:56:48 UTC 2018


commit a2f565643d4c34a49712c37e3931dd6f80da7ba1
Author: Nicolas Vigier <boklm at torproject.org>
Date:   Mon Apr 16 19:30:02 2018 +0200

    Bug 25817: add ansible roles for tbb-nightly setup
---
 README                                             |  43 ++------
 tools/ansible/Makefile                             |   3 +
 tools/ansible/README                               |   6 ++
 tools/ansible/boklm-tbb-nightly-build.yml          |   7 ++
 .../boklm-tbb-nightly/tbb-nightly-build.yml        |   3 +
 tools/ansible/inventory                            |   4 +
 .../roles/tbb-nightly-build/defaults/main.yml      |  11 ++
 .../roles/tbb-nightly-build/handlers/main.yml      |   5 +
 .../ansible/roles/tbb-nightly-build/tasks/main.yml |  84 +++++++++++++++
 .../roles/tbb-nightly-build/templates/nginx.conf   |  16 +++
 .../tbb-nightly-build/templates/rbm.local.conf     | 116 +++++++++++++++++++++
 .../tbb-nightly-build/templates/start-tbb-nightly  |   5 +
 .../tbb-nightly-build/templates/testsuite-config   |  27 +++++
 .../tbb-nightly-build/templates/www-index.html     |  10 ++
 14 files changed, 306 insertions(+), 34 deletions(-)

diff --git a/README b/README
index d498c05..a20b659 100644
--- a/README
+++ b/README
@@ -138,43 +138,18 @@ Automated builds using tbb-testsuite
 ------------------------------------
 
 The Tor Browser testsuite scripts can also be used to do nightly builds
-and publish the build logs.
+and publish the build logs. The recommended way to do that is to use
+the ansible roles from the tools/ansible directory. See next section
+for details.
 
-If you want to do that, start by cloning the git repository:
 
-   $ git clone https://git.torproject.org/boklm/tor-browser-bundle-testsuite.git
+Using ansible to set up a nightly build machine
+-----------------------------------------------
 
-Install some dependencies:
-
-   # apt-get install -y libdata-dump-perl libfile-slurp-perl \
-               libio-captureoutput-perl perlmagick libjson-perl \
-               libwww-perl liblwp-protocol-https-perl libtemplate-perl \
-               libyaml-syck-perl libdatetime-perl \
-               libemail-sender-perl libemail-simple-perl libfile-type-perl \
-               libipc-run-perl libxml-libxml-perl
-
-Copy the config/tor-browser_build-boklm file and edit it:
-
-   $ cd tor-browser-bundle-testsuite
-   $ cp config/tor-browser_build-boklm config/tor-browser_build-$user
-   $ vim config/tor-browser_build-$user
-
-Change the publish_dir and publish_url options. The publish_dir option is
-the local directory where the builds will be stored. The publish_url
-option is the public URL where the builds will be available.
-
-Copy the tools/tor-browser-builds-boklm file and edit it to change the
---config= option:
-
-   $ cp tools/tor-browser-builds-boklm tools/tor-browser-builds-$user
-   $ vim tools/tor-browser-builds-$user
-
-You can now run ./tools/tor-browser-builds-$user to start the build, and
-add it to your crontab.
-
-The html build reports will be available in the reports/ directory, and
-the build files in the tor-browser-builds/ directory (unless you changed
-the publish_dir option).
+The directory tools/ansible contains some ansible roles to set up a
+nightly build machine. You can look at the playbook defined in
+boklm-tbb-nightly-build.yml and variables in group_vars/boklm-tbb-nightly/
+for an example of how it can be used.
 
 
 Signing builds
diff --git a/tools/ansible/Makefile b/tools/ansible/Makefile
index 608f932..72deb5b 100644
--- a/tools/ansible/Makefile
+++ b/tools/ansible/Makefile
@@ -3,3 +3,6 @@ ansible-tbb-build:
 
 fpcentral:
 	ANSIBLE_CONFIG='$(@D)/ansible-fpcentral.cfg' ansible-playbook -i inventory --ask-become-pass fpcentral.yml
+
+boklm-tbb-nightly-build:
+	ansible-playbook -i inventory boklm-tbb-nightly-build.yml
diff --git a/tools/ansible/README b/tools/ansible/README
index 6b2e6a4..dc69ad7 100644
--- a/tools/ansible/README
+++ b/tools/ansible/README
@@ -16,3 +16,9 @@ fpcentral:
   you need to be in the fpcentral tpo ldap group. Your ldap password will
   be asked, to sudo to the fpcentral user.
 
+boklm-tbb-nightly-build:
+  This target is used by boklm to deploy a nightly build machine. If
+  anybody else wants to set up their own nightly builds, it is possible to
+  do it by adding a new host to the inventory file and making copies of
+  group_vars/boklm-tbb-nightly/ and boklm-tbb-nightly-build.yml.
+
diff --git a/tools/ansible/boklm-tbb-nightly-build.yml b/tools/ansible/boklm-tbb-nightly-build.yml
new file mode 100644
index 0000000..cc37e23
--- /dev/null
+++ b/tools/ansible/boklm-tbb-nightly-build.yml
@@ -0,0 +1,7 @@
+---
+
+- hosts: boklm-tbb-nightly-build
+  roles:
+      - role: tbb-builder
+      - role: tbb-nightly-build
+      - role: unattended-upgrades
diff --git a/tools/ansible/group_vars/boklm-tbb-nightly/tbb-nightly-build.yml b/tools/ansible/group_vars/boklm-tbb-nightly/tbb-nightly-build.yml
new file mode 100644
index 0000000..ebaadfe
--- /dev/null
+++ b/tools/ansible/group_vars/boklm-tbb-nightly/tbb-nightly-build.yml
@@ -0,0 +1,3 @@
+---
+nightly_build_hostname: f4amtbsowhix7rrf.onion
+nightly_build_url: 'http://{{ nightly_build_hostname }}'
diff --git a/tools/ansible/inventory b/tools/ansible/inventory
index 32a5805..fc25842 100644
--- a/tools/ansible/inventory
+++ b/tools/ansible/inventory
@@ -1,8 +1,12 @@
 build-sunet-a ansible_ssh_user=root ansible_ssh_host=build-sunet-a.torproject.net
 fpcentral ansible_become=True ansible_become_method=sudo ansible_become_user=fpcentral ansible_ssh_host=forrestii.torproject.org allow_world_readable_tmpfiles=True
+boklm-tbb-nightly-build ansible_ssh_user=root ansible_become_method=su
 
 [tbb-build]
 build-sunet-a
 
 [fpcentral]
 fpcentral
+
+[boklm-tbb-nightly]
+boklm-tbb-nightly-build
diff --git a/tools/ansible/roles/tbb-nightly-build/defaults/main.yml b/tools/ansible/roles/tbb-nightly-build/defaults/main.yml
new file mode 100644
index 0000000..d84a161
--- /dev/null
+++ b/tools/ansible/roles/tbb-nightly-build/defaults/main.yml
@@ -0,0 +1,11 @@
+---
+nightly_build_user: tbb-nightly
+nightly_build_cron_hour: 2
+nightly_build_cron_minute: 20
+nightly_build_keep_builds: 2
+testsuite_dir: "/home/{{ nightly_build_user }}/tbb-testsuite"
+testsuite_git_url: https://git.torproject.org/boklm/tor-browser-bundle-testsuite.git
+testsuite_git_commit: 348ad855711382089c4fbf1badfec58e31a6c148
+nightly_build_wwwdir: "/home/{{ nightly_build_user }}/www"
+nightly_build_nginx_enable: true
+nightly_build_nginx_listen: 127.0.0.1:80
diff --git a/tools/ansible/roles/tbb-nightly-build/handlers/main.yml b/tools/ansible/roles/tbb-nightly-build/handlers/main.yml
new file mode 100644
index 0000000..3350d73
--- /dev/null
+++ b/tools/ansible/roles/tbb-nightly-build/handlers/main.yml
@@ -0,0 +1,5 @@
+---
+- name: restart nginx
+  service:
+      name: nginx
+      state: restarted
diff --git a/tools/ansible/roles/tbb-nightly-build/tasks/main.yml b/tools/ansible/roles/tbb-nightly-build/tasks/main.yml
new file mode 100644
index 0000000..64d31a9
--- /dev/null
+++ b/tools/ansible/roles/tbb-nightly-build/tasks/main.yml
@@ -0,0 +1,84 @@
+---
+- name: create tbb-nightly user
+  user:
+      name: "{{ nightly_build_user }}"
+      comment: "Tor Browser Nightly Builds"
+      groups: tbb-build
+      createhome: yes
+      home: "/home/{{ nightly_build_user }}"
+
+- name: clone tor browser testsuite
+  become: yes
+  become_user: "{{ nightly_build_user }}"
+  git:
+      repo: "{{ testsuite_git_url }}"
+      dest: "{{ testsuite_dir }}"
+      version: "{{ testsuite_git_commit }}"
+
+- name: install testsuite dependencies
+  command: "{{ testsuite_dir }}/install-deps"
+
+- name: add testsuite config
+  template:
+      src: testsuite-config
+      dest: "{{ testsuite_dir }}/config/tbb-nightly"
+      mode: 0644
+      owner: "{{ nightly_build_user }}"
+
+- name: create rbm-config directory
+  file:
+      path: '{{ testsuite_dir }}/rbm-config/'
+      state: directory
+      owner: '{{ nightly_build_user }}'
+      mode: 0755
+
+- name: add rbm config
+  template:
+      src: rbm.local.conf
+      dest: "{{ testsuite_dir }}/rbm-config/tbb-nightly.rbm.local.conf"
+      mode: 0644
+      owner: "{{ nightly_build_user }}"
+
+- name: add start-tbb-nightly script
+  template:
+      src: start-tbb-nightly
+      dest: "/home/{{ nightly_build_user }}/start-tbb-nightly"
+      mode: 0755
+      owner: "{{ nightly_build_user }}"
+
+- name: add cron to start nighly build
+  cron:
+      name: tbb-nightly-build
+      user: "{{ nightly_build_user }}"
+      hour: "{{ nightly_build_cron_hour }}"
+      minute: "{{ nightly_build_cron_minute }}"
+      job: "/home/{{ nightly_build_user }}/start-tbb-nightly"
+
+- name: create wwwdir
+  file:
+      path: '{{ nightly_build_wwwdir }}'
+      state: directory
+      owner: '{{ nightly_build_user }}'
+      mode: 0755
+
+- name: add index page
+  template:
+      src: www-index.html
+      dest: "{{ nightly_build_wwwdir }}/index.html"
+      mode: 0644
+      owner: "{{ nightly_build_user }}"
+
+- name: install nginx
+  apt:
+      name: nginx
+      state: present
+  when: nightly_build_nginx_enable
+
+- name: add nginx config
+  template:
+      src: nginx.conf
+      dest: /etc/nginx/sites-enabled/tbb-nightly-build
+      mode: 0644
+  notify:
+      - restart nginx
+  when: nightly_build_nginx_enable
diff --git a/tools/ansible/roles/tbb-nightly-build/templates/nginx.conf b/tools/ansible/roles/tbb-nightly-build/templates/nginx.conf
new file mode 100644
index 0000000..9d44bec
--- /dev/null
+++ b/tools/ansible/roles/tbb-nightly-build/templates/nginx.conf
@@ -0,0 +1,16 @@
+server {
+    listen {{ nightly_build_nginx_listen }};
+    server_name {{ nightly_build_hostname }};
+    root {{ nightly_build_wwwdir }};
+    index index.html;
+    location / {
+        try_files $uri $uri/ =404;
+    }
+    location /reports {
+        alias {{ testsuite_dir }}/reports;
+    }
+    location /tor-browser-builds {
+        alias {{ testsuite_dir }}/tor-browser-builds;
+        autoindex on;
+    }
+}
diff --git a/tools/ansible/roles/tbb-nightly-build/templates/rbm.local.conf b/tools/ansible/roles/tbb-nightly-build/templates/rbm.local.conf
new file mode 100644
index 0000000..fcd6b34
--- /dev/null
+++ b/tools/ansible/roles/tbb-nightly-build/templates/rbm.local.conf
@@ -0,0 +1,116 @@
+---
+### This file is used to override options from rbm.conf to adapt them
+### to your local setup.
+###
+### Copy this file as rbm.local.conf to enable it, and uncomment the
+### options you want to modify.
+
+
+### The tmp_dir option defines where temporary files are stored. The
+### builds are made from this directory, so using a directory on a fast
+### disk can improve build time. By default we are using a tmp directory
+### under the tor-browser-build directory.
+#tmp_dir: /tmp
+
+### The debug option defines whether a debugging shell should be opened
+### automatically in the build directory/container in case of build
+### failure. If you are doing automated builds, you might want to disable
+### this.
+#debug: 0
+
+### The build_log option defines in which file the build logs of each
+### component are stored. If you set it to '-' the logs are output on
+### stdout and stderr.
+#build_log: '-'
+
+### By default, the logs from previous builds are kept in the log files.
+### If you set build_log_append to 0, then previous logs are cleaned
+### when starting a new build.
+#build_log_append: 0
+
+buildconf:
+  buildconf: 1
+
+  ### The buildconf/num_procs option can be used to select the number of
+  ### build processes to run simultaneously. You can also use the
+  ### RBM_NUM_PROCS environment variable. The default is 4.
+  #num_procs: 8
+
+  ### The buildconf/git_signtag_opt option is useful when you tag a release.
+  ### It is used to set the 'git tag' argument to select the keyid for
+  ### signing the tag.
+  #git_signtag_opt: '-u keyid'
+
+var:
+  local_conf: 1
+
+  ### The var/sign_build option defines if you want to sign the
+  ### sha256sums-unsigned-build.txt file with gpg.
+  #sign_build: 1
+
+  ### The var/sign_build_gpg_opts option can be used to define some gpg
+  ### options to select the key to use to sign the sha256sums-unsigned-build.txt
+  ### file.
+  #sign_build_gpg_opts: '--local-user XXXXXXXX'
+
+  ### The clean configuration is used by the cleaning script to find the
+  ### branches and build targets you are using, to compute the list of
+  ### files that should be kept.
+  ###
+  ### If you only do alpha builds for all platforms, you can use the
+  ### following configuration:
+  clean:
+    HEAD:
+      - project: release
+        target:
+          - alpha
+          - torbrowser-all
+      - project: release
+        target:
+          - nightly
+          - torbrowser-all
+  #
+  ### If you are doing 'release' builds in the maint-7.0 branch and
+  ### 'alpha' builds in the master branch, you can use the following
+  ### configuration:
+  #clean:
+  #  master:
+  #    - project: release
+  #      target:
+  #        - alpha
+  #        - torbrowser-all
+  #  maint-7.0:
+  #    - project: release
+  #      target:
+  #        - release
+  #        - torbrowser-all
+
+targets:
+
+  ### testbuild is based on alpha by default. Uncomment this if you want it
+  ### to be based on nightly.
+  #torbrowser-testbuild:
+  #  - testbuild
+  #  - nightly
+
+  testbuild:
+    var:
+      testbuild: 1
+
+      ### Uncomment this if you want to create mar files in your test build.
+      #build_mar: 1
+
+  nightly:
+
+    ### By default 'fetch' is set to 1 for nightly builds, meaning that new
+    ### commits will be fetched automatically during each build. You can
+    ### disable this during development if you want to do rebuilds to test
+    ### a specific change, but don't want rebuilds caused by unrelated
+    ### changes, or if you want to decide at which point new commits are
+    ### fetched. When 'fetch' is set to 'if_needed', new commits will only
+    ### be fetched if the selected commit (or branch, or tag) is not present,
+    ### which means that existing branches won't be updated. In that case
+    ### you can fetch new commits by running 'make fetch'.
+    fetch: 'if_needed'
+
+# vim: filetype=yaml sw=2
diff --git a/tools/ansible/roles/tbb-nightly-build/templates/start-tbb-nightly b/tools/ansible/roles/tbb-nightly-build/templates/start-tbb-nightly
new file mode 100644
index 0000000..3c1847d
--- /dev/null
+++ b/tools/ansible/roles/tbb-nightly-build/templates/start-tbb-nightly
@@ -0,0 +1,5 @@
+#!/bin/sh
+cd {{ testsuite_dir }}
+export RBM_NO_DEBUG=1
+./tbb-testsuite --config=tbb-nightly "$@"
+./tools/prune-old-builds/prune-old-builds --prefix '' --days {{ nightly_build_keep_builds }} ./tor-browser-builds
diff --git a/tools/ansible/roles/tbb-nightly-build/templates/testsuite-config b/tools/ansible/roles/tbb-nightly-build/templates/testsuite-config
new file mode 100644
index 0000000..b537f56
--- /dev/null
+++ b/tools/ansible/roles/tbb-nightly-build/templates/testsuite-config
@@ -0,0 +1,27 @@
+# vim: filetype=perl expandtab
+use strict;
+use FindBin;
+use DateTime;
+use TBBTestSuite::TestSuite::TorBrowserBuild;
+
+my $date = DateTime->now->ymd;
+my $name = "tor-browser-$date";
+
+if (-d "$options->{'reports-dir'}/r/$name") {
+    print "Doing nothing: $name already done\n";
+    return ( args => [] );
+}
+
+my $testsuite = TBBTestSuite::TestSuite::TorBrowserBuild->new({
+        publish_dir => "$FindBin::Bin/tor-browser-builds/$date",
+        publish_url => "{{ nightly_build_url }}/tor-browser-builds/$date",
+        rbm_local_conf => "$FindBin::Bin/rbm-config/tbb-nightly.rbm.local.conf",
+        make_clean => 1,
+    });
+
+my %res = (
+    name => $name,
+    args => [ $testsuite ],
+    tags => [ 'nightly' ],
+);
+%res;
diff --git a/tools/ansible/roles/tbb-nightly-build/templates/www-index.html b/tools/ansible/roles/tbb-nightly-build/templates/www-index.html
new file mode 100644
index 0000000..05050d8
--- /dev/null
+++ b/tools/ansible/roles/tbb-nightly-build/templates/www-index.html
@@ -0,0 +1,10 @@
+<html>
+<head>
+    <title>Tor Browser Nightly Builds</title>
+</head>
+<body>
+    <h1>Tor Browser Nightly Builds</title>
+    <a href="reports/"><h3>Build Reports</h3></a>
+    <a href="tor-browser-builds/"><h3>Build files</h3></a>
+</body>
+</html>



More information about the tor-commits mailing list