commit c4aaea174394a8ba612ca898a443fc07c4813bbf Author: Iain R. Learmonth <irl@torproject.org> Date: Thu Feb 20 14:11:21 2020 +0000 Initial CloudFormation template and Ansible for exit scanner The exitmap module used for the exit scanner is maintained as part of metrics-cloud. If it were rewritten to be less of a hack, it might be imported into the upstream exitmap repository. --- ansible/exit-scanners-aws.yml | 9 ++ ansible/roles/exit-scanner-sys/tasks/main.yml | 83 ++++++++++++++++ ansible/roles/exit-scanner/files/exitscan.py | 105 +++++++++++++++++++++ .../roles/exit-scanner/files/exitscanner.service | 10 ++ ansible/roles/exit-scanner/files/ipscan.py | 95 +++++++++++++++++++ ansible/roles/exit-scanner/tasks/main.yml | 53 +++++++++++ cloudformation/exit-scanner-dev.yml | 27 ++++++ 7 files changed, 382 insertions(+) diff --git a/ansible/exit-scanners-aws.yml b/ansible/exit-scanners-aws.yml new file mode 100644 index 0000000..72ce0c6 --- /dev/null +++ b/ansible/exit-scanners-aws.yml @@ -0,0 +1,9 @@ +--- +- hosts: exit-scanners + user: admin + vars: + onionoo_version: 7.0-1.21.0 + roles: + - tor-client + - exit-scanner-sys + - exit-scanner diff --git a/ansible/roles/exit-scanner-sys/tasks/main.yml b/ansible/roles/exit-scanner-sys/tasks/main.yml new file mode 100644 index 0000000..78916d8 --- /dev/null +++ b/ansible/roles/exit-scanner-sys/tasks/main.yml @@ -0,0 +1,83 @@ +--- +- name: disable system tor + systemd: + name: tor.service + enabled: false + state: stopped + become: true +- name: install stem for py2 from backports + apt: + pkg: python-stem + state: latest + default_release: buster-backports + become: true +- name: install stem for py3 from backports + apt: + pkg: python3-stem + state: latest + default_release: buster-backports + become: true +- name: install exitmap requirements + apt: + pkg: + - git + - python-dnspython + update_cache: yes + become: yes +- name: create check account + user: + name: check + comment: "Check Service User" + #uid: 1547 + state: present + become: yes +- name: create tordnsel account + user: + name: tordnsel + comment: "Exit Scanner Service User" + #uid: 1547 + state: present + become: yes +- name: create service directory + file: + path: /srv/exitscanner.torproject.org + state: directory + become: yes +- name: link /home in /srv + file: + src: /home + dest: /srv/home + state: link + become: yes +- name: link home directories /home + file: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + state: link + force: yes + with_items: + - { src: /home/tordnsel, dest: /srv/exitscanner.torproject.org/home } + - { src: /home/check, dest: /srv/exitscanner.torproject.org/check-home } + become: yes +- name: create exit scanner runtime directory + file: + path: /srv/exitscanner.torproject.org/exitscanner + owner: tordnsel + group: tordnsel + mode: 0755 + state: directory + become: yes +- name: create check runtime directory + file: + path: /srv/exitscanner.torproject.org/check + owner: check + group: check + mode: 0755 + state: directory + become: yes +- name: enable lingering for service users + shell: "loginctl enable-linger {{ item }}" + with_items: + - tordnsel + - check + become: yes diff --git a/ansible/roles/exit-scanner/files/exitscan.py b/ansible/roles/exit-scanner/files/exitscan.py new file mode 100644 index 0000000..14c0b17 --- /dev/null +++ b/ansible/roles/exit-scanner/files/exitscan.py @@ -0,0 +1,105 @@ + +import collections +import datetime +import glob +import json +import os +import os.path +import re +import subprocess + +import stem.descriptor + +fortyeighthoursago = datetime.datetime.utcnow() - datetime.timedelta(hours=48) + +Measurement = collections.namedtuple("Measurement", ["address", "date"]) +exits = dict() + + +def merge_addresses(fp, new): + addresses = exits[fp].exit_addresses + addresses.extend(new) + addresses.sort(key=lambda x: x[1], reverse=True) + uniq_addresses = [] + while len(uniq_addresses) < len(addresses): + if addresses[len(uniq_addresses)][0] in uniq_addresses: + addresses.remove(addresses[len(uniq_addresses)]) + continue + uniq_addresses.append(addresses[len(uniq_addresses)][0]) + return [ + a for a in addresses + if a[1] > fortyeighthoursago + ] + + +def merge(desc): + if desc.fingerprint not in exits: + exits[desc.fingerprint] = desc + return + fp = desc.fingerprint + exits[fp].published = max(exits[fp].published, desc.published) + exits[fp].last_status = max(exits[fp].last_status, desc.last_status) + exits[fp].exit_addresses = merge_addresses(fp, desc.exit_addresses) + + +def run(): + exit_lists = list(glob.iglob('lists/2*')) # fix this glob before 23:59 on 31st Dec 2999 + + # Import latest exit list from disc + if exit_lists: + latest_exit_list = max(exit_lists, key=os.path.getctime) + for desc in stem.descriptor.parse_file(latest_exit_list, + descriptor_type="tordnsel 1.0"): + merge(desc) + + # Import new measurements + with subprocess.Popen(["./bin/exitmap", "ipscan", "-o", "/dev/stdout"], + cwd="/srv/exitscanner.torproject.org/exitscanner/exitmap", + stdout=subprocess.PIPE, + encoding='utf-8') as p: + for line in p.stdout: + print(line) + result = re.match( + r"^([0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}),[0-9]{3} modules\.ipscan \[INFO\] (\{.*\})$", + line) + if result: + print(result) + check_result = json.loads(result.group(2)) + desc = stem.descriptor.tordnsel.TorDNSEL("", False) + desc.fingerprint = check_result["Fingerprint"] + desc.last_status = datetime.datetime.utcnow().replace(minute=0, second=0, microsecond=0) + desc.published = datetime.datetime.strptime( + check_result["DescPublished"], "%Y-%m-%dT%H:%M:%S") + desc.exit_addresses = [ + (check_result["IP"], + datetime.datetime.strptime(result.group(1), + "%Y-%m-%d %H:%M:%S")) + ] + merge(desc) + + # Format exit list filename + now = datetime.datetime.utcnow() + filename = (f"{now.year}-{now.month:02d}-" + f"{now.day:02d}-{now.hour:02d}-" + f"{now.minute:02d}-{now.second:02d}") + + # Format an exit list + with open(f"lists/{filename}", "w") as out: + for desc in exits.values(): + if desc.exit_addresses: + out.write(f"ExitNode {desc.fingerprint}\n") + out.write(f"Published {desc.published}\n") + out.write(f"LastStatus {desc.last_status}\n") + for a in desc.exit_addresses: + out.write(f"ExitAddress {a[0]} {a[1]}\n") + + # Provide the snapshot emulation + os.unlink("lists/latest") + os.symlink(os.path.abspath(f"lists/{filename}"), "lists/latest") + +if __name__ == "__main__": + while True: + start = datetime.datetime.utcnow() + run() + while datetime.datetime.utcnow() < start + datetime.timedelta(minutes=40): + pass diff --git a/ansible/roles/exit-scanner/files/exitscanner.service b/ansible/roles/exit-scanner/files/exitscanner.service new file mode 100644 index 0000000..012d8b7 --- /dev/null +++ b/ansible/roles/exit-scanner/files/exitscanner.service @@ -0,0 +1,10 @@ +[Unit] +Description=Exit Scanner + +[Service] +Type=simple +WorkingDirectory=/srv/exitscanner.torproject.org/exitscanner +ExecStart=/usr/bin/python3 /srv/exitscanner.torproject.org/exitscanner/exitscan.py + +[Install] +WantedBy=default.target diff --git a/ansible/roles/exit-scanner/files/ipscan.py b/ansible/roles/exit-scanner/files/ipscan.py new file mode 100644 index 0000000..d59ce4c --- /dev/null +++ b/ansible/roles/exit-scanner/files/ipscan.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python2 + +# Copyright 2013-2017 Philipp Winter <phw@nymity.ch> +# +# This file is part of exitmap. +# +# exitmap is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# exitmap is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with exitmap. If not, see <http://www.gnu.org/licenses/>. + +""" +Module to detect false negatives for <https://check.torproject.org>. +""" + +import sys +import json +import logging +try: + import urllib2 +except ImportError: + import urllib.request as urllib2 + +from util import exiturl + +import stem.descriptor.server_descriptor as descriptor + +log = logging.getLogger(__name__) + +# exitmap needs this variable to figure out which relays can exit to the given +# destination(s). + +destinations = [("check.torproject.org", 443)] + + +def fetch_page(exit_desc): + """ + Fetch check.torproject.org and see if we are using Tor. + """ + + data = None + url = exiturl(exit_desc.fingerprint) + + try: + data = urllib2.urlopen("https://check.torproject.org/api/ip", + timeout=10).read() + except Exception as err: + log.debug("urllib2.urlopen says: %s" % err) + return + + if not data: + return + + try: + check_answer = json.loads(data) + except ValueError as err: + log.warning("Couldn't parse JSON over relay %s: %s" % (url, data)) + return + + check_answer["DescPublished"] = exit_desc.published.isoformat() + check_answer["Fingerprint"] = exit_desc.fingerprint + + log.info(json.dumps(check_answer)) + +def probe(exit_desc, run_python_over_tor, run_cmd_over_tor, **kwargs): + """ + Probe the given exit relay and look for check.tp.o false negatives. + """ + + run_python_over_tor(fetch_page, exit_desc) + + +def main(): + """ + Entry point when invoked over the command line. + """ + + desc = descriptor.ServerDescriptor("") + desc.fingerprint = "bogus" + desc.address = "0.0.0.0" + fetch_page(desc) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ansible/roles/exit-scanner/tasks/main.yml b/ansible/roles/exit-scanner/tasks/main.yml new file mode 100644 index 0000000..d80edb5 --- /dev/null +++ b/ansible/roles/exit-scanner/tasks/main.yml @@ -0,0 +1,53 @@ +--- +- name: clone the sources + git: + repo: https://github.com/NullHypothesis/exitmap.git + dest: /srv/exitscanner.torproject.org/exitscanner/exitmap + become: true + become_user: tordnsel +- name: install the ipscan module + copy: + src: ipscan.py + dest: /srv/exitscanner.torproject.org/exitscanner/exitmap/src/modules/ipscan.py + mode: 0755 + become: true + become_user: tordnsel +- name: install the exit scanner script + copy: + src: exitscan.py + dest: /srv/exitscanner.torproject.org/exitscanner/exitscan.py + mode: 0755 + become: true + become_user: tordnsel +- name: create systemd user directory for exitscanner + file: + path: /srv/exitscanner.torproject.org/home/.config/systemd/user + state: directory + become: true + become_user: tordnsel +- name: create exit lists directory + file: + path: /srv/exitscanner.torproject.org/exitscanner/lists + state: directory + become: true + become_user: tordnsel +- name: install exit scanner service file + copy: + src: exitscanner.service + dest: "/srv/exitscanner.torproject.org/home/.config/systemd/user/exitscanner.service" + become: true + become_user: tordnsel +- name: reload systemd daemon + systemd: + scope: user + daemon_reload: yes + become: true + become_user: tordnsel +- name: enable and start exitscanner service + systemd: + scope: user + name: exitscanner + state: started + enabled: yes + become: yes + become_user: tordnsel diff --git a/cloudformation/exit-scanner-dev.yml b/cloudformation/exit-scanner-dev.yml new file mode 100644 index 0000000..2ee4259 --- /dev/null +++ b/cloudformation/exit-scanner-dev.yml @@ -0,0 +1,27 @@ +--- +# CloudFormation Stack for Exit Scanner development instance +# This stack will only deploy on us-east-1 and will deploy in the Metrics VPC +# aws cloudformation deploy --region us-east-1 --stack-name `whoami`-exit-scanner-dev --template-file exit-scanner-dev.yml --parameter-overrides myKeyPair="$(./identify_user.sh)" +AWSTemplateFormatVersion: 2010-09-09 +Parameters: + myKeyPair: + Description: Amazon EC2 Key Pair + Type: "AWS::EC2::KeyPair::KeyName" +Resources: + Instance: + Type: AWS::EC2::Instance + Properties: + AvailabilityZone: us-east-1a + ImageId: ami-01db78123b2b99496 + InstanceType: t2.large + SubnetId: + Fn::ImportValue: 'MetricsSubnet' + KeyName: !Ref myKeyPair + SecurityGroupIds: + - Fn::ImportValue: 'MetricsInternetSecurityGroup' + - Fn::ImportValue: 'MetricsPingableSecurityGroup' + - Fn::ImportValue: 'MetricsHTTPSSecurityGroup' +Outputs: + PublicIp: + Description: "Instance public IP" + Value: !GetAtt Instance.PublicIp