[tor-commits] [metrics-cloud/master] Initial CloudFormation template and Ansible for exit scanner

irl at torproject.org irl at torproject.org
Thu Feb 20 14:16:45 UTC 2020


commit c4aaea174394a8ba612ca898a443fc07c4813bbf
Author: Iain R. Learmonth <irl at torproject.org>
Date:   Thu Feb 20 14:11:21 2020 +0000

    Initial CloudFormation template and Ansible for exit scanner
    
    The exitmap module used for the exit scanner is maintained as part of
    metrics-cloud. If it were rewritten to be less of a hack, it might be
    imported into the upstream exitmap repository.
---
 ansible/exit-scanners-aws.yml                      |   9 ++
 ansible/roles/exit-scanner-sys/tasks/main.yml      |  83 ++++++++++++++++
 ansible/roles/exit-scanner/files/exitscan.py       | 105 +++++++++++++++++++++
 .../roles/exit-scanner/files/exitscanner.service   |  10 ++
 ansible/roles/exit-scanner/files/ipscan.py         |  95 +++++++++++++++++++
 ansible/roles/exit-scanner/tasks/main.yml          |  53 +++++++++++
 cloudformation/exit-scanner-dev.yml                |  27 ++++++
 7 files changed, 382 insertions(+)

diff --git a/ansible/exit-scanners-aws.yml b/ansible/exit-scanners-aws.yml
new file mode 100644
index 0000000..72ce0c6
--- /dev/null
+++ b/ansible/exit-scanners-aws.yml
@@ -0,0 +1,9 @@
+---
+- hosts: exit-scanners
+  user: admin
+  vars:
+    onionoo_version: 7.0-1.21.0
+  roles:
+    - tor-client
+    - exit-scanner-sys
+    - exit-scanner
diff --git a/ansible/roles/exit-scanner-sys/tasks/main.yml b/ansible/roles/exit-scanner-sys/tasks/main.yml
new file mode 100644
index 0000000..78916d8
--- /dev/null
+++ b/ansible/roles/exit-scanner-sys/tasks/main.yml
@@ -0,0 +1,83 @@
+---
+- name: disable system tor
+  systemd:
+    name: tor.service
+    enabled: false
+    state: stopped
+  become: true
+- name: install stem for py2 from backports
+  apt:
+    pkg: python-stem
+    state: latest
+    default_release: buster-backports
+  become: true
+- name: install stem for py3 from backports
+  apt:
+    pkg: python3-stem
+    state: latest
+    default_release: buster-backports
+  become: true
+- name: install exitmap requirements
+  apt:
+    pkg:
+      - git
+      - python-dnspython
+    update_cache: yes
+  become: yes
+- name: create check account
+  user:
+    name: check
+    comment: "Check Service User"
+    #uid: 1547
+    state: present
+  become: yes
+- name: create tordnsel account
+  user:
+    name: tordnsel
+    comment: "Exit Scanner Service User"
+    #uid: 1547
+    state: present
+  become: yes
+- name: create service directory
+  file:
+    path: /srv/exitscanner.torproject.org
+    state: directory
+  become: yes
+- name: link /home in /srv
+  file:
+    src: /home
+    dest: /srv/home
+    state: link
+  become: yes
+- name: link home directories /home
+  file:
+    src: "{{ item.src }}"
+    dest: "{{ item.dest }}"
+    state: link
+    force: yes
+  with_items:
+    - { src: /home/tordnsel, dest: /srv/exitscanner.torproject.org/home }
+    - { src: /home/check, dest: /srv/exitscanner.torproject.org/check-home }
+  become: yes
+- name: create exit scanner runtime directory
+  file:
+    path: /srv/exitscanner.torproject.org/exitscanner
+    owner: tordnsel
+    group: tordnsel
+    mode: 0755
+    state: directory
+  become: yes
+- name: create check runtime directory
+  file:
+    path: /srv/exitscanner.torproject.org/check
+    owner: check
+    group: check
+    mode: 0755
+    state: directory
+  become: yes
+- name: enable lingering for service users
+  shell: "loginctl enable-linger {{ item }}"
+  with_items:
+    - tordnsel
+    - check
+  become: yes
diff --git a/ansible/roles/exit-scanner/files/exitscan.py b/ansible/roles/exit-scanner/files/exitscan.py
new file mode 100644
index 0000000..14c0b17
--- /dev/null
+++ b/ansible/roles/exit-scanner/files/exitscan.py
@@ -0,0 +1,105 @@
+
+import collections
+import datetime
+import glob
+import json
+import os
+import os.path
+import re
+import subprocess
+
+import stem.descriptor
+
+fortyeighthoursago = datetime.datetime.utcnow() - datetime.timedelta(hours=48)
+
+Measurement = collections.namedtuple("Measurement", ["address", "date"])
+exits = dict()
+
+
+def merge_addresses(fp, new):
+    addresses = exits[fp].exit_addresses
+    addresses.extend(new)
+    addresses.sort(key=lambda x: x[1], reverse=True)
+    uniq_addresses = []
+    while len(uniq_addresses) < len(addresses):
+        if addresses[len(uniq_addresses)][0] in uniq_addresses:
+            addresses.remove(addresses[len(uniq_addresses)])
+            continue
+        uniq_addresses.append(addresses[len(uniq_addresses)][0])
+    return [
+        a for a in addresses
+        if a[1] > fortyeighthoursago
+    ]
+
+
+def merge(desc):
+    if desc.fingerprint not in exits:
+        exits[desc.fingerprint] = desc
+        return
+    fp = desc.fingerprint
+    exits[fp].published = max(exits[fp].published, desc.published)
+    exits[fp].last_status = max(exits[fp].last_status, desc.last_status)
+    exits[fp].exit_addresses = merge_addresses(fp, desc.exit_addresses)
+
+
+def run():
+    exit_lists = list(glob.iglob('lists/2*')) # fix this glob before 23:59 on 31st Dec 2999
+
+    # Import latest exit list from disc
+    if exit_lists:
+        latest_exit_list = max(exit_lists, key=os.path.getctime)
+        for desc in stem.descriptor.parse_file(latest_exit_list,
+                                               descriptor_type="tordnsel 1.0"):
+            merge(desc)
+
+    # Import new measurements
+    with subprocess.Popen(["./bin/exitmap", "ipscan", "-o", "/dev/stdout"],
+                          cwd="/srv/exitscanner.torproject.org/exitscanner/exitmap",
+                          stdout=subprocess.PIPE,
+                          encoding='utf-8') as p:
+        for line in p.stdout:
+            print(line)
+            result = re.match(
+                r"^([0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}),[0-9]{3} modules\.ipscan \[INFO\] (\{.*\})$",
+                line)
+            if result:
+                print(result)
+                check_result = json.loads(result.group(2))
+                desc = stem.descriptor.tordnsel.TorDNSEL("", False)
+                desc.fingerprint = check_result["Fingerprint"]
+                desc.last_status = datetime.datetime.utcnow().replace(minute=0, second=0, microsecond=0)
+                desc.published = datetime.datetime.strptime(
+                    check_result["DescPublished"], "%Y-%m-%dT%H:%M:%S")
+                desc.exit_addresses = [
+                    (check_result["IP"],
+                     datetime.datetime.strptime(result.group(1),
+                                                "%Y-%m-%d %H:%M:%S"))
+                ]
+                merge(desc)
+
+    # Format exit list filename
+    now = datetime.datetime.utcnow()
+    filename = (f"{now.year}-{now.month:02d}-"
+                f"{now.day:02d}-{now.hour:02d}-"
+                f"{now.minute:02d}-{now.second:02d}")
+
+    # Format an exit list
+    with open(f"lists/{filename}", "w") as out:
+        for desc in exits.values():
+            if desc.exit_addresses:
+                out.write(f"ExitNode {desc.fingerprint}\n")
+                out.write(f"Published {desc.published}\n")
+                out.write(f"LastStatus {desc.last_status}\n")
+                for a in desc.exit_addresses:
+                    out.write(f"ExitAddress {a[0]} {a[1]}\n")
+
+    # Provide the snapshot emulation
+    os.unlink("lists/latest")
+    os.symlink(os.path.abspath(f"lists/{filename}"), "lists/latest")
+
+if __name__ == "__main__":
+    while True:
+        start = datetime.datetime.utcnow()
+        run()
+        while datetime.datetime.utcnow() < start + datetime.timedelta(minutes=40):
+            pass
diff --git a/ansible/roles/exit-scanner/files/exitscanner.service b/ansible/roles/exit-scanner/files/exitscanner.service
new file mode 100644
index 0000000..012d8b7
--- /dev/null
+++ b/ansible/roles/exit-scanner/files/exitscanner.service
@@ -0,0 +1,10 @@
+[Unit]
+Description=Exit Scanner
+
+[Service]
+Type=simple
+WorkingDirectory=/srv/exitscanner.torproject.org/exitscanner
+ExecStart=/usr/bin/python3 /srv/exitscanner.torproject.org/exitscanner/exitscan.py
+
+[Install]
+WantedBy=default.target
diff --git a/ansible/roles/exit-scanner/files/ipscan.py b/ansible/roles/exit-scanner/files/ipscan.py
new file mode 100644
index 0000000..d59ce4c
--- /dev/null
+++ b/ansible/roles/exit-scanner/files/ipscan.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python2
+
+# Copyright 2013-2017 Philipp Winter <phw at nymity.ch>
+#
+# This file is part of exitmap.
+#
+# exitmap is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# exitmap is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with exitmap.  If not, see <http://www.gnu.org/licenses/>.
+
+"""
+Module to detect false negatives for <https://check.torproject.org>.
+"""
+
+import sys
+import json
+import logging
+try:
+    import urllib2
+except ImportError:
+    import urllib.request as urllib2
+
+from util import exiturl
+
+import stem.descriptor.server_descriptor as descriptor
+
+log = logging.getLogger(__name__)
+
+# exitmap needs this variable to figure out which relays can exit to the given
+# destination(s).
+
+destinations = [("check.torproject.org", 443)]
+
+
+def fetch_page(exit_desc):
+    """
+    Fetch check.torproject.org and see if we are using Tor.
+    """
+
+    data = None
+    url = exiturl(exit_desc.fingerprint)
+
+    try:
+        data = urllib2.urlopen("https://check.torproject.org/api/ip",
+                               timeout=10).read()
+    except Exception as err:
+        log.debug("urllib2.urlopen says: %s" % err)
+        return
+
+    if not data:
+        return
+
+    try:
+        check_answer = json.loads(data)
+    except ValueError as err:
+        log.warning("Couldn't parse JSON over relay %s: %s" % (url, data))
+        return
+
+    check_answer["DescPublished"] = exit_desc.published.isoformat()
+    check_answer["Fingerprint"] = exit_desc.fingerprint
+
+    log.info(json.dumps(check_answer))
+
+def probe(exit_desc, run_python_over_tor, run_cmd_over_tor, **kwargs):
+    """
+    Probe the given exit relay and look for check.tp.o false negatives.
+    """
+
+    run_python_over_tor(fetch_page, exit_desc)
+
+
+def main():
+    """
+    Entry point when invoked over the command line.
+    """
+
+    desc = descriptor.ServerDescriptor("")
+    desc.fingerprint = "bogus"
+    desc.address = "0.0.0.0"
+    fetch_page(desc)
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/ansible/roles/exit-scanner/tasks/main.yml b/ansible/roles/exit-scanner/tasks/main.yml
new file mode 100644
index 0000000..d80edb5
--- /dev/null
+++ b/ansible/roles/exit-scanner/tasks/main.yml
@@ -0,0 +1,53 @@
+---
+- name: clone the sources
+  git:
+    repo: https://github.com/NullHypothesis/exitmap.git
+    dest: /srv/exitscanner.torproject.org/exitscanner/exitmap
+  become: true
+  become_user: tordnsel
+- name: install the ipscan module
+  copy:
+    src: ipscan.py
+    dest: /srv/exitscanner.torproject.org/exitscanner/exitmap/src/modules/ipscan.py
+    mode: 0755
+  become: true
+  become_user: tordnsel
+- name: install the exit scanner script
+  copy:
+    src: exitscan.py
+    dest: /srv/exitscanner.torproject.org/exitscanner/exitscan.py
+    mode: 0755
+  become: true
+  become_user: tordnsel
+- name: create systemd user directory for exitscanner
+  file:
+    path: /srv/exitscanner.torproject.org/home/.config/systemd/user
+    state: directory
+  become: true
+  become_user: tordnsel
+- name: create exit lists directory
+  file:
+    path: /srv/exitscanner.torproject.org/exitscanner/lists
+    state: directory
+  become: true
+  become_user: tordnsel
+- name: install exit scanner service file
+  copy:
+    src: exitscanner.service
+    dest: "/srv/exitscanner.torproject.org/home/.config/systemd/user/exitscanner.service"
+  become: true
+  become_user: tordnsel
+- name: reload systemd daemon
+  systemd:
+    scope: user
+    daemon_reload: yes
+  become: true
+  become_user: tordnsel
+- name: enable and start exitscanner service
+  systemd:
+    scope: user
+    name: exitscanner
+    state: started
+    enabled: yes
+  become: yes
+  become_user: tordnsel
diff --git a/cloudformation/exit-scanner-dev.yml b/cloudformation/exit-scanner-dev.yml
new file mode 100644
index 0000000..2ee4259
--- /dev/null
+++ b/cloudformation/exit-scanner-dev.yml
@@ -0,0 +1,27 @@
+---
+# CloudFormation Stack for Exit Scanner development instance
+# This stack will only deploy on us-east-1 and will deploy in the Metrics VPC
+# aws cloudformation deploy --region us-east-1 --stack-name `whoami`-exit-scanner-dev --template-file exit-scanner-dev.yml --parameter-overrides myKeyPair="$(./identify_user.sh)"
+AWSTemplateFormatVersion: 2010-09-09
+Parameters:
+  myKeyPair:
+    Description: Amazon EC2 Key Pair
+    Type: "AWS::EC2::KeyPair::KeyName"
+Resources:
+  Instance:
+    Type: AWS::EC2::Instance
+    Properties:
+      AvailabilityZone: us-east-1a
+      ImageId: ami-01db78123b2b99496
+      InstanceType: t2.large
+      SubnetId:
+        Fn::ImportValue: 'MetricsSubnet'
+      KeyName: !Ref myKeyPair
+      SecurityGroupIds:
+        - Fn::ImportValue: 'MetricsInternetSecurityGroup'
+        - Fn::ImportValue: 'MetricsPingableSecurityGroup'
+        - Fn::ImportValue: 'MetricsHTTPSSecurityGroup'
+Outputs:
+  PublicIp:
+    Description: "Instance public IP"
+    Value: !GetAtt Instance.PublicIp



More information about the tor-commits mailing list