[tor-commits] [support-tools/master] Add scripts to monitor changes to RT articles
lunar at torproject.org
lunar at torproject.org
Fri Jul 18 13:55:34 UTC 2014
commit 26e45154a20650fa8d2ae2c2891f7aa864d7d750
Author: Lunar <lunar at torproject.org>
Date: Fri Jul 18 15:54:13 2014 +0200
Add scripts to monitor changes to RT articles
---
rt-articles/README | 69 +++++++++++++++++++++++++++++++++++++++
rt-articles/dump_rt_articles | 69 +++++++++++++++++++++++++++++++++++++++
rt-articles/push_rt_articles | 11 +++++++
rt-articles/receive_rt_articles | 19 +++++++++++
4 files changed, 168 insertions(+)
diff --git a/rt-articles/README b/rt-articles/README
new file mode 100644
index 0000000..7fb39c2
--- /dev/null
+++ b/rt-articles/README
@@ -0,0 +1,69 @@
+This is a set of script to extract RT articles from the database and
+track when they change.
+
+Setup notes
+===========
+
+Two systems are involved. rude.torproject.org which hosts RT and
+perdulce.torproject.org which hosts people.torproject.org. We are going
+to allow a very restricted push of data from the formor to the latter.
+
+On rude.torproject.org
+----------------------
+
+Create SSH key:
+
+ ssh-keygen -t rsa -b 4096 -N '' -f .ssh/id_rsa_rt_articles
+
+Configure SSH (through `~/.ssh/config`):
+
+ Host perdulce.torproject.org
+ BatchMode yes
+ IdentityFile ~/.ssh/id_rsa_rt_articles
+
+Add cron job:
+
+ 17,47 * * * * /srv/rtstuff/support-tools/rt-articles/push_rt_articles
+
+On perdulce.torproject.org
+--------------------------
+
+Install the public key from rude. To do that, use email interface of
+ud-ldap: <https://db.debian.org/doc-mail.html>. *But replace
+`debian.org` with `torproject.org`!*
+
+ ssh-rsa AAAA…== usual key
+ allowed_hosts=perdulce.debian.org from="rude.torproject.org",command="~/bin/receive_rt_articles",no-pty,no-port-forwarding ssh-rsa AAAA…== key from rude
+
+Create the Git repository:
+
+ cd public_html
+ mkdir rt-articles.git && cd rt-articles.git
+ git init --bare
+ vi description
+ mv hooks/post-update.sample hooks/post-update
+ chmod +x hooks/post-update
+
+Configure the email notification:
+
+ echo 'sh /usr/share/git-core/contrib/hooks/post-receive-email' >> hooks/post-receive
+ chmod +x hooks/post-receive
+ git config hooks.mailinglist lunar at torproject.org
+ git config hooks.envelopesender lunar+rt-articles at torproject.org
+ git config hooks.emailprefix '[RT Articles] '
+ git config hooks.diffopts '--stat --summary --find-copies-harder -p'
+
+Do the initial clone:
+
+ cd $HOME
+ git clone public_html/rt-articles.git
+
+Also clone the script repository:
+
+ cd $HOME
+ git clone https://git.torproject.org/support-tools.git
+
+And add the symlink:
+
+ mkdir -p bin
+ ln -s ../support-tools/receive_rt_articles bin
diff --git a/rt-articles/dump_rt_articles b/rt-articles/dump_rt_articles
new file mode 100755
index 0000000..b47bf2f
--- /dev/null
+++ b/rt-articles/dump_rt_articles
@@ -0,0 +1,69 @@
+#!/usr/bin/python
+#
+# This program is free software. It comes without any warranty, to
+# the extent permitted by applicable law. You can redistribute it
+# and/or modify it under the terms of the Do What The Fuck You Want
+# To Public License, Version 2, as published by Sam Hocevar. See
+# http://sam.zoy.org/wtfpl/COPYING for more details.
+#
+# This script will dump all RT articles to a given directory.
+# Ideally, the directory should be empty.
+
+import psycopg2
+import re
+import sys
+import os
+import os.path
+
+RT_CONNINFO = "host=drobovi.torproject.org sslmode=require user=rtreader dbname=rt"
+
+SELECT_TRANSACTIONS_QUERY = """
+SELECT classes.name AS class,
+ articles.name AS title,
+ CASE WHEN objectcustomfieldvalues.content != '' THEN objectcustomfieldvalues.content
+ ELSE objectcustomfieldvalues.largecontent
+ END AS content,
+ objectcustomfieldvalues.lastupdated,
+ articles.id
+ FROM classes, articles, objectcustomfieldvalues
+ WHERE articles.class = classes.id
+ AND objectcustomfieldvalues.objecttype = 'RT::Article'
+ AND objectcustomfieldvalues.objectid = articles.id
+ AND objectcustomfieldvalues.id = (
+ SELECT objectcustomfieldvalues.id
+ FROM objectcustomfieldvalues
+ WHERE objectcustomfieldvalues.objectid = articles.id
+ AND objectcustomfieldvalues.disabled = 0
+ ORDER BY objectcustomfieldvalues.lastupdated DESC
+ LIMIT 1)
+ ORDER BY classes.id, articles.id
+"""
+
+def slugify(title):
+ return re.sub(r'\W+', '-', title)
+
+def dump_rt_articles(con, path):
+ cur = con.cursor()
+ cur.execute(SELECT_TRANSACTIONS_QUERY)
+ for article_class, title, content, last_updated, article_id in cur:
+ class_slug = slugify(article_class)
+ class_dir = os.path.join(path, class_slug)
+ if not os.path.isdir(class_dir):
+ os.mkdir(class_dir)
+ filename = os.path.join(class_dir, "%03d_%s" % (article_id, slugify(title)))
+ with open(filename, 'w') as f:
+ f.write('Title: %s\n' % title)
+ f.write('Last-updated: %s\n\n' % last_updated)
+ f.write(content)
+
+if __name__ == '__main__':
+ if len(sys.argv) != 2:
+ print >>sys.stderr, "Usage: %s DIR" % (sys.argv[0])
+ sys.exit(1)
+ path = sys.argv[1]
+ if not os.path.isdir(path):
+ print >>sys.stderr, "Usage: %s DIR" % (sys.argv[0])
+ sys.exit(2)
+ con = psycopg2.connect(RT_CONNINFO)
+ dump_rt_articles(con, path)
+ con.close()
diff --git a/rt-articles/push_rt_articles b/rt-articles/push_rt_articles
new file mode 100755
index 0000000..a48b801
--- /dev/null
+++ b/rt-articles/push_rt_articles
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+# To be run on rude.torproject.org from a cronjob
+
+set -e
+
+DIR=$(mktemp -d)
+trap "rm -r '$DIR'" EXIT
+
+$(dirname "$0")/dump_rt_articles "$DIR"
+tar -C "$DIR" -zcf - . | ssh perdulce.torproject.org bin/receive_rt_articles
diff --git a/rt-articles/receive_rt_articles b/rt-articles/receive_rt_articles
new file mode 100755
index 0000000..9e1fe68
--- /dev/null
+++ b/rt-articles/receive_rt_articles
@@ -0,0 +1,19 @@
+#!/bin/sh
+
+# To be run from perdule.torproject.org through a restricted SSH command.
+# stdin will be a gzipped tarball with the articles
+
+# Must be a Git repository
+DEST="$HOME/rt-articles"
+
+set -e
+
+(
+cd "$DEST"
+git rm --quiet --force -r . || true
+tar -zxf -
+git add .
+git diff --cached --quiet ||
+ git commit --quiet --author='An automated script <lunar+rt-articles at torproject.org>' -m "Update RT articles"
+git push --quiet origin master
+)
More information about the tor-commits
mailing list