commit 26e45154a20650fa8d2ae2c2891f7aa864d7d750 Author: Lunar <lunar@torproject.org> Date: Fri Jul 18 15:54:13 2014 +0200 Add scripts to monitor changes to RT articles --- rt-articles/README | 69 +++++++++++++++++++++++++++++++++++++++ rt-articles/dump_rt_articles | 69 +++++++++++++++++++++++++++++++++++++++ rt-articles/push_rt_articles | 11 +++++++ rt-articles/receive_rt_articles | 19 +++++++++++ 4 files changed, 168 insertions(+) diff --git a/rt-articles/README b/rt-articles/README new file mode 100644 index 0000000..7fb39c2 --- /dev/null +++ b/rt-articles/README @@ -0,0 +1,69 @@ +This is a set of script to extract RT articles from the database and +track when they change. + +Setup notes +=========== + +Two systems are involved. rude.torproject.org which hosts RT and +perdulce.torproject.org which hosts people.torproject.org. We are going +to allow a very restricted push of data from the formor to the latter. + +On rude.torproject.org +---------------------- + +Create SSH key: + + ssh-keygen -t rsa -b 4096 -N '' -f .ssh/id_rsa_rt_articles + +Configure SSH (through `~/.ssh/config`): + + Host perdulce.torproject.org + BatchMode yes + IdentityFile ~/.ssh/id_rsa_rt_articles + +Add cron job: + + 17,47 * * * * /srv/rtstuff/support-tools/rt-articles/push_rt_articles + +On perdulce.torproject.org +-------------------------- + +Install the public key from rude. To do that, use email interface of +ud-ldap: <https://db.debian.org/doc-mail.html>. *But replace +`debian.org` with `torproject.org`!* + + ssh-rsa AAAA…== usual key + allowed_hosts=perdulce.debian.org from="rude.torproject.org",command="~/bin/receive_rt_articles",no-pty,no-port-forwarding ssh-rsa AAAA…== key from rude + +Create the Git repository: + + cd public_html + mkdir rt-articles.git && cd rt-articles.git + git init --bare + vi description + mv hooks/post-update.sample hooks/post-update + chmod +x hooks/post-update + +Configure the email notification: + + echo 'sh /usr/share/git-core/contrib/hooks/post-receive-email' >> hooks/post-receive + chmod +x hooks/post-receive + git config hooks.mailinglist lunar@torproject.org + git config hooks.envelopesender lunar+rt-articles@torproject.org + git config hooks.emailprefix '[RT Articles] ' + git config hooks.diffopts '--stat --summary --find-copies-harder -p' + +Do the initial clone: + + cd $HOME + git clone public_html/rt-articles.git + +Also clone the script repository: + + cd $HOME + git clone https://git.torproject.org/support-tools.git + +And add the symlink: + + mkdir -p bin + ln -s ../support-tools/receive_rt_articles bin diff --git a/rt-articles/dump_rt_articles b/rt-articles/dump_rt_articles new file mode 100755 index 0000000..b47bf2f --- /dev/null +++ b/rt-articles/dump_rt_articles @@ -0,0 +1,69 @@ +#!/usr/bin/python +# +# This program is free software. It comes without any warranty, to +# the extent permitted by applicable law. You can redistribute it +# and/or modify it under the terms of the Do What The Fuck You Want +# To Public License, Version 2, as published by Sam Hocevar. See +# http://sam.zoy.org/wtfpl/COPYING for more details. +# +# This script will dump all RT articles to a given directory. +# Ideally, the directory should be empty. + +import psycopg2 +import re +import sys +import os +import os.path + +RT_CONNINFO = "host=drobovi.torproject.org sslmode=require user=rtreader dbname=rt" + +SELECT_TRANSACTIONS_QUERY = """ +SELECT classes.name AS class, + articles.name AS title, + CASE WHEN objectcustomfieldvalues.content != '' THEN objectcustomfieldvalues.content + ELSE objectcustomfieldvalues.largecontent + END AS content, + objectcustomfieldvalues.lastupdated, + articles.id + FROM classes, articles, objectcustomfieldvalues + WHERE articles.class = classes.id + AND objectcustomfieldvalues.objecttype = 'RT::Article' + AND objectcustomfieldvalues.objectid = articles.id + AND objectcustomfieldvalues.id = ( + SELECT objectcustomfieldvalues.id + FROM objectcustomfieldvalues + WHERE objectcustomfieldvalues.objectid = articles.id + AND objectcustomfieldvalues.disabled = 0 + ORDER BY objectcustomfieldvalues.lastupdated DESC + LIMIT 1) + ORDER BY classes.id, articles.id +""" + +def slugify(title): + return re.sub(r'\W+', '-', title) + +def dump_rt_articles(con, path): + cur = con.cursor() + cur.execute(SELECT_TRANSACTIONS_QUERY) + for article_class, title, content, last_updated, article_id in cur: + class_slug = slugify(article_class) + class_dir = os.path.join(path, class_slug) + if not os.path.isdir(class_dir): + os.mkdir(class_dir) + filename = os.path.join(class_dir, "%03d_%s" % (article_id, slugify(title))) + with open(filename, 'w') as f: + f.write('Title: %s\n' % title) + f.write('Last-updated: %s\n\n' % last_updated) + f.write(content) + +if __name__ == '__main__': + if len(sys.argv) != 2: + print >>sys.stderr, "Usage: %s DIR" % (sys.argv[0]) + sys.exit(1) + path = sys.argv[1] + if not os.path.isdir(path): + print >>sys.stderr, "Usage: %s DIR" % (sys.argv[0]) + sys.exit(2) + con = psycopg2.connect(RT_CONNINFO) + dump_rt_articles(con, path) + con.close() diff --git a/rt-articles/push_rt_articles b/rt-articles/push_rt_articles new file mode 100755 index 0000000..a48b801 --- /dev/null +++ b/rt-articles/push_rt_articles @@ -0,0 +1,11 @@ +#!/bin/sh + +# To be run on rude.torproject.org from a cronjob + +set -e + +DIR=$(mktemp -d) +trap "rm -r '$DIR'" EXIT + +$(dirname "$0")/dump_rt_articles "$DIR" +tar -C "$DIR" -zcf - . | ssh perdulce.torproject.org bin/receive_rt_articles diff --git a/rt-articles/receive_rt_articles b/rt-articles/receive_rt_articles new file mode 100755 index 0000000..9e1fe68 --- /dev/null +++ b/rt-articles/receive_rt_articles @@ -0,0 +1,19 @@ +#!/bin/sh + +# To be run from perdule.torproject.org through a restricted SSH command. +# stdin will be a gzipped tarball with the articles + +# Must be a Git repository +DEST="$HOME/rt-articles" + +set -e + +( +cd "$DEST" +git rm --quiet --force -r . || true +tar -zxf - +git add . +git diff --cached --quiet || + git commit --quiet --author='An automated script <lunar+rt-articles@torproject.org>' -m "Update RT articles" +git push --quiet origin master +)