[tor-commits] [metrics-cloud/master] onionoo-backend: Complete stack for AWS test/dev instance

[irl at torproject.org]

commit b9aaf7c37b48da4581769495a52479288afaf42a
Author: Iain R. Learmonth <irl at fsfe.org>
Date:   Tue Oct 29 15:02:38 2019 +0000

    onionoo-backend: Complete stack for AWS test/dev instance
---
 ansible/ansible.cfg                              |  2 +
 ansible/onionoo-backends-aws.yml                 | 15 +++++
 ansible/roles/onionoo-backend-sys/tasks/main.yml | 76 ++++++++++++++++++++++++
 ansible/roles/onionoo-backend/tasks/main.yml     | 11 ++++
 cloudformation/onionoo-dev.yml                   | 39 ++++++++++++
 5 files changed, 143 insertions(+)

diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg
new file mode 100644
index 0000000..bf578e2
--- /dev/null
+++ b/ansible/ansible.cfg
@@ -0,0 +1,2 @@
+[defaults]
+allow_world_readable_tmpfiles = true
diff --git a/ansible/onionoo-backends-aws.yml b/ansible/onionoo-backends-aws.yml
new file mode 100644
index 0000000..6e525dd
--- /dev/null
+++ b/ansible/onionoo-backends-aws.yml
@@ -0,0 +1,15 @@
+---
+- hosts: onionoo-backends
+  user: admin
+  vars:
+    onionoo_version: 7.0-1.21.0
+  pre_tasks:
+    - name: get sudo password
+      local_action: shell pass Tor/sudo/onionoo
+      register: pass_output
+      changed_when: False
+    - name: store as ansible become password
+      set_fact: ansible_become_password="{{ pass_output.stdout_lines[0] }}"
+  roles:
+    - onionoo-backend-sys
+    - onionoo-backend
diff --git a/ansible/roles/onionoo-backend-sys/tasks/main.yml b/ansible/roles/onionoo-backend-sys/tasks/main.yml
new file mode 100644
index 0000000..10989aa
--- /dev/null
+++ b/ansible/roles/onionoo-backend-sys/tasks/main.yml
@@ -0,0 +1,76 @@
+---
+- name: install xfsprogs
+  apt:
+    update_cache: yes
+    name: xfsprogs
+    state: present
+  become: yes
+- name: install jdk
+  apt:
+    update_cache: yes
+    name: default-jdk
+    state: present
+  become: yes
+- name: format service volume
+  filesystem:
+    fstype: xfs
+    device: /dev/xvdb
+  become: yes
+- name: add /srv to fstab
+  mount:
+    src: /dev/xvdb
+    path: /srv
+    fstype: xfs
+    dump: 0
+    passno: 2
+    state: mounted
+  become: yes
+- name: create service directory
+  file:
+    path: /srv/onionoo.torproject.org
+    state: directory
+  become: yes
+- name: link /home in /srv
+  file:
+    src: /home
+    dest: /srv/home
+    state: link
+  become: yes
+- name: link home directories /home
+  file:
+    src: "{{ item.src }}"
+    dest: "{{ item.dest }}"
+    state: link
+    force: yes
+  with_items:
+    - { src: /home/onionoo, dest: /srv/onionoo.torproject.org/home }
+    - { src: /home/onionoo-unpriv, dest: /srv/onionoo.torproject.org/home-unpriv }
+  become: yes
+- name: create onionoo account
+  user:
+    name: onionoo
+    comment: "Onionoo Service User"
+    uid: 1547
+    state: present
+  become: yes
+- name: create runtime directory
+  file:
+    path: /srv/onionoo.torproject.org/onionoo
+    owner: onionoo
+    group: onionoo
+    mode: 0755
+    state: directory
+  become: yes
+- name: create onionoo-unpriv account
+  user:
+    name: onionoo-unpriv
+    comment: "Onionoo Unpriv Service User"
+    uid: 2075
+    state: present
+  become: yes
+- name: enable lingering for onionoo and onionoo-unpriv
+  shell: "loginctl enable-linger {{ item }}"
+  with_items:
+    - onionoo
+    - onionoo-unpriv
+  become: yes
diff --git a/ansible/roles/onionoo-backend/tasks/main.yml b/ansible/roles/onionoo-backend/tasks/main.yml
index a0c76fd..0d80b29 100644
--- a/ansible/roles/onionoo-backend/tasks/main.yml
+++ b/ansible/roles/onionoo-backend/tasks/main.yml
@@ -132,6 +132,17 @@
     daemon_reload: yes
   become: true
   become_user: onionoo-unpriv
+- name: check for out directory
+  stat:
+    path: "/srv/onionoo.torproject.org/onionoo/out"
+  register: out_stat
+- name: populate out directory with a single run
+  shell: java -Xmx4g -Dsun.net.client.defaultConnectTimeout=60000 -Dsun.net.client.defaultReadTimeout=60000 -DLOGBASE=logs -cp onionoo-{{ onionoo_version }}.jar org.torproject.metrics.onionoo.cron.Main --single-run
+  args:
+    chdir: /srv/onionoo.torproject.org/onionoo
+  when: out_stat.stat.exists == False
+  become: yes
+  become_user: onionoo
 - name: start onionoo service
   systemd:
     scope: user
diff --git a/cloudformation/onionoo-dev.yml b/cloudformation/onionoo-dev.yml
new file mode 100644
index 0000000..a635466
--- /dev/null
+++ b/cloudformation/onionoo-dev.yml
@@ -0,0 +1,39 @@
+---
+# CloudFormation Stack for Onionoo development instance
+# This stack will only deploy on us-east-1 and will deploy in the Metrics VPC
+# aws cloudformation deploy --region us-east-1 --stack-name `whoami`-onionoo-dev --template-file onionoo-dev.yml --parameter-overrides myKeyPair="irl yubikey 4"
+AWSTemplateFormatVersion: 2010-09-09
+Parameters:
+  myKeyPair:
+    Description: Amazon EC2 Key Pair
+    Type: "AWS::EC2::KeyPair::KeyName"
+Resources:
+  Instance:
+    Type: AWS::EC2::Instance
+    Properties:
+      AvailabilityZone: us-east-1a
+      ImageId: ami-01db78123b2b99496
+      InstanceType: t2.large
+      SubnetId:
+        Fn::ImportValue: !Sub 'MetricsSubnet'
+      KeyName: !Ref myKeyPair
+      SecurityGroupIds:
+        - Fn::ImportValue: !Sub 'MetricsInternetSecurityGroup'
+        - Fn::ImportValue: !Sub 'MetricsPingableSecurityGroup'
+        - Fn::ImportValue: !Sub 'MetricsHTTPASecurityGroup'
+  ServiceVolume:
+    Type: AWS::EC2::Volume
+    Properties: 
+      AvailabilityZone: us-east-1a
+      Size: 15
+      VolumeType: gp2
+  ServiceVolumeAttachment:
+    Type: AWS::EC2::VolumeAttachment
+    Properties:
+      Device: /dev/sdb
+      InstanceId: !Ref Instance
+      VolumeId: !Ref ServiceVolume
+Outputs:
+  PublicIp:
+    Description: "Instance public IP"
+    Value: !GetAtt Instance.PublicIp