commit d7c1caadb5af4431ad8bafb54bbb1fb716b88a68 Author: Damian Johnson <atagar@torproject.org> Date: Sun Feb 11 19:15:45 2018 -0800 Dynamically get package versions from the wiki The biggest time sync for me to keep this wiki up to date is bumping versions in this DocTor script. It's brittle, but instead having this script get the current versions on the wiki first so all I need to do to update is fix the wiki. --- package_versions.py | 161 +++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 121 insertions(+), 40 deletions(-) diff --git a/package_versions.py b/package_versions.py index 6c6e3f6..aa10bad 100755 --- a/package_versions.py +++ b/package_versions.py @@ -26,57 +26,58 @@ NETBSD_VERSION = 'CURRENT, <b>Version: </b>([0-9\.]+),' COLUMN = '| %-10s | %-10s | %-10s | %-50s |' DIV = '+%s+%s+%s+%s+' % ('-' * 12, '-' * 12, '-' * 12, '-' * 52) +TRAC_URL = 'https://trac.torproject.org/projects/tor/wiki/doc/packages' -Package = collections.namedtuple('Package', ['platform', 'url', 'version', 'regex']) +Package = collections.namedtuple('Package', ['platform', 'url', 'regex']) PACKAGES = [ ('tor', [ - Package('mac', 'https://raw.githubusercontent.com/Homebrew/homebrew-core/master/Formula/tor....', '0.3.2.9', 'tor-([0-9\.]+).tar.gz'), - Package('debian', 'https://packages.debian.org/sid/tor', '0.3.2.9', DEBIAN_VERSION), - Package('fedora', 'https://apps.fedoraproject.org/packages/tor', '0.3.2.9', FEDORA_VERSION), - Package('gentoo', 'https://packages.gentoo.org/packages/net-vpn/tor', '0.3.2.9', None), - Package('archlinux', 'https://www.archlinux.org/packages/community/x86_64/tor/', '0.3.2.9', ARCH_LINUX_VERSION), - Package('slackware', 'https://slackbuilds.org/repository/14.2/network/tor/', '0.3.2.9', 'tor-([0-9\.]+).tar.gz'), - Package('freebsd', 'https://www.freshports.org/security/tor/', '0.3.2.9', FREEBSD_VERSION), - Package('openbsd', 'https://cvsweb.openbsd.org/cgi-bin/cvsweb/ports/net/tor/Makefile?rev=HEAD&co...', '0.3.2.9', OPENBSD_DIST_VERSION), - Package('netbsd', 'http://pkgsrc.se/net/tor', '0.3.2.9', NETBSD_VERSION), + Package('mac', 'https://raw.githubusercontent.com/Homebrew/homebrew-core/master/Formula/tor....', 'tor-([0-9\.]+).tar.gz'), + Package('debian', 'https://packages.debian.org/sid/tor', DEBIAN_VERSION), + Package('fedora', 'https://apps.fedoraproject.org/packages/tor', FEDORA_VERSION), + Package('gentoo', 'https://packages.gentoo.org/packages/net-vpn/tor', None), + Package('archlinux', 'https://www.archlinux.org/packages/community/x86_64/tor/', ARCH_LINUX_VERSION), + Package('slackware', 'https://slackbuilds.org/repository/14.2/network/tor/', 'tor-([0-9\.]+).tar.gz'), + Package('freebsd', 'https://www.freshports.org/security/tor/', FREEBSD_VERSION), + Package('openbsd', 'https://cvsweb.openbsd.org/cgi-bin/cvsweb/ports/net/tor/Makefile?rev=HEAD&co...', OPENBSD_DIST_VERSION), + Package('netbsd', 'http://pkgsrc.se/net/tor', NETBSD_VERSION), ]), ('nyx', [ - Package('gentoo', 'https://packages.gentoo.org/packages/net-misc/nyx', '2.0.4', None), - Package('archlinux', 'https://aur.archlinux.org/packages/nyx/', '2.0.4', AUR_VERSION), - Package('slackware', 'https://slackbuilds.org/repository/14.2/python/nyx/', '2.0.4', 'nyx-([0-9\.]+).tar.gz'), - Package('freebsd', 'https://www.freshports.org/security/nyx/', '2.0.4', FREEBSD_VERSION), - Package('openbsd', 'https://cvsweb.openbsd.org/cgi-bin/cvsweb/ports/net/nyx/Makefile?rev=HEAD&co...', '2.0.4', OPENBSD_EGG_VERSION), + Package('gentoo', 'https://packages.gentoo.org/packages/net-misc/nyx', None), + Package('archlinux', 'https://aur.archlinux.org/packages/nyx/', AUR_VERSION), + Package('slackware', 'https://slackbuilds.org/repository/14.2/python/nyx/', 'nyx-([0-9\.]+).tar.gz'), + Package('freebsd', 'https://www.freshports.org/security/nyx/', FREEBSD_VERSION), + Package('openbsd', 'https://cvsweb.openbsd.org/cgi-bin/cvsweb/ports/net/nyx/Makefile?rev=HEAD&co...', OPENBSD_EGG_VERSION), ]), ('stem', [ - Package('debian', 'https://packages.debian.org/sid/python-stem', '1.6.0', DEBIAN_VERSION), - Package('fedora', 'https://apps.fedoraproject.org/packages/python-stem', '1.6.0', FEDORA_VERSION), - Package('gentoo', 'https://packages.gentoo.org/packages/net-libs/stem', '1.6.0', None), - Package('archlinux', 'https://aur.archlinux.org/packages/stem/', '1.6.0', AUR_VERSION), - Package('slackware', 'https://slackbuilds.org/repository/14.2/python/stem/', '1.6.0', 'stem-([0-9\.]+).tar.gz'), - Package('freebsd', 'https://www.freshports.org/security/py-stem/', '1.6.0', FREEBSD_VERSION), - Package('openbsd', 'https://cvsweb.openbsd.org/cgi-bin/cvsweb/ports/net/py-stem/Makefile?rev=HEA...', '1.6.0', OPENBSD_EGG_VERSION), + Package('debian', 'https://packages.debian.org/sid/python-stem', DEBIAN_VERSION), + Package('fedora', 'https://apps.fedoraproject.org/packages/python-stem', FEDORA_VERSION), + Package('gentoo', 'https://packages.gentoo.org/packages/net-libs/stem', None), + Package('archlinux', 'https://aur.archlinux.org/packages/stem/', AUR_VERSION), + Package('slackware', 'https://slackbuilds.org/repository/14.2/python/stem/', 'stem-([0-9\.]+).tar.gz'), + Package('freebsd', 'https://www.freshports.org/security/py-stem/', FREEBSD_VERSION), + Package('openbsd', 'https://cvsweb.openbsd.org/cgi-bin/cvsweb/ports/net/py-stem/Makefile?rev=HEA...', OPENBSD_EGG_VERSION), ]), ('txtorcon', [ - Package('debian', 'https://packages.debian.org/sid/python-txtorcon', '0.19.3', DEBIAN_VERSION), - Package('gentoo', 'https://packages.gentoo.org/packages/dev-python/txtorcon', '0.19.3', None), - Package('slackware', 'https://slackbuilds.org/repository/14.2/python/txtorcon/', '0.19.3', 'txtorcon-([0-9\.]+).tar.gz'), - Package('freebsd', 'https://www.freshports.org/security/py-txtorcon/', '0.19.3', FREEBSD_VERSION), + Package('debian', 'https://packages.debian.org/sid/python-txtorcon', DEBIAN_VERSION), + Package('gentoo', 'https://packages.gentoo.org/packages/dev-python/txtorcon', None), + Package('slackware', 'https://slackbuilds.org/repository/14.2/python/txtorcon/', 'txtorcon-([0-9\.]+).tar.gz'), + Package('freebsd', 'https://www.freshports.org/security/py-txtorcon/', FREEBSD_VERSION), ]), ('torsocks', [ - Package('mac', 'https://raw.githubusercontent.com/Homebrew/homebrew-core/master/Formula/tors...', '2.2.0', ':tag => "v([0-9\.]+)",'), - Package('debian', 'https://packages.debian.org/sid/torsocks', '2.2.0', DEBIAN_VERSION), - Package('fedora', 'https://apps.fedoraproject.org/packages/torsocks', '2.1.0', FEDORA_VERSION), - Package('gentoo', 'https://packages.gentoo.org/packages/net-proxy/torsocks', '2.2.0', None), - Package('archlinux', 'https://www.archlinux.org/packages/community/x86_64/torsocks/', '2.2.0', ARCH_LINUX_VERSION), - Package('slackware', 'https://slackbuilds.org/repository/14.2/network/torsocks/', '2.2.0', 'torsocks \(([0-9\.]+)\) </h2>'), - Package('freebsd', 'https://www.freshports.org/net/torsocks/', '2.2.0', 'SHA256 \(dgoulet-torsocks-v([0-9\.]+)_GH0.tar.gz\)'), - Package('openbsd', 'https://cvsweb.openbsd.org/cgi-bin/cvsweb/ports/net/torsocks/Makefile?rev=HE...', '1.2', OPENBSD_DIST_VERSION), + Package('mac', 'https://raw.githubusercontent.com/Homebrew/homebrew-core/master/Formula/tors...', ':tag => "v([0-9\.]+)",'), + Package('debian', 'https://packages.debian.org/sid/torsocks', DEBIAN_VERSION), + Package('fedora', 'https://apps.fedoraproject.org/packages/torsocks', FEDORA_VERSION), + Package('gentoo', 'https://packages.gentoo.org/packages/net-proxy/torsocks', None), + Package('archlinux', 'https://www.archlinux.org/packages/community/x86_64/torsocks/', ARCH_LINUX_VERSION), + Package('slackware', 'https://slackbuilds.org/repository/14.2/network/torsocks/', 'torsocks \(([0-9\.]+)\) </h2>'), + Package('freebsd', 'https://www.freshports.org/net/torsocks/', 'SHA256 \(dgoulet-torsocks-v([0-9\.]+)_GH0.tar.gz\)'), + Package('openbsd', 'https://cvsweb.openbsd.org/cgi-bin/cvsweb/ports/net/torsocks/Makefile?rev=HE...', OPENBSD_DIST_VERSION), ]), ('ooni probe', [ - Package('mac', 'https://raw.githubusercontent.com/Homebrew/homebrew-core/master/Formula/ooni...', '2.2.0', 'ooniprobe-([0-9\.]+).tar.gz'), - Package('debian', 'https://packages.debian.org/sid/ooniprobe', '2.2.0', DEBIAN_VERSION), - Package('archlinux', 'https://aur.archlinux.org/packages/ooniprobe/', '1.2.2', AUR_VERSION), + Package('mac', 'https://raw.githubusercontent.com/Homebrew/homebrew-core/master/Formula/ooni...', 'ooniprobe-([0-9\.]+).tar.gz'), + Package('debian', 'https://packages.debian.org/sid/ooniprobe', DEBIAN_VERSION), + Package('archlinux', 'https://aur.archlinux.org/packages/ooniprobe/', AUR_VERSION), ]), ] @@ -94,6 +95,76 @@ def fetch_url(url): raise IOError(str(exc)) +def wiki_package_versions(): + # Provides versions present on the wiki of the form... + # + # {project => {platform => version}} + # + # Unfortunately the wiki table lacks good handles to match against so this is + # gonna be very, very brittle. That's fine though - this is just an effort + # saving measure for me anyway. ;P + + request = fetch_url(TRAC_URL) + version_entries = [] + + for line in request.splitlines(): + m = re.search('<b>Version:</b> <a href=".*">(.*)</a>', line) + + if m: + version_entries.append(m.group(1)) + + if len(version_entries) != 38: + raise IOError('Table on %s no longer matches what this daemon expects (had %i entries)' % (TRAC_URL, len(version_entries))) + + return { + 'tor': { + 'mac': version_entries[0], + 'debian': version_entries[1], + 'fedora': version_entries[2], + 'gentoo': version_entries[3], + 'archlinux': version_entries[4], + 'slackware': version_entries[5], + 'freebsd': version_entries[6], + 'openbsd': version_entries[7], + 'netbsd': version_entries[8], + }, 'nyx': { + 'fedora': version_entries[9], + 'gentoo': version_entries[10], + 'archlinux': version_entries[11], + 'slackware': version_entries[12], + 'freebsd': version_entries[13], + 'openbsd': version_entries[14], + }, 'stem': { + 'debian': version_entries[15], + 'fedora': version_entries[16], + 'gentoo': version_entries[17], + 'archlinux': version_entries[18], + 'slackware': version_entries[19], + 'freebsd': version_entries[20], + 'openbsd': version_entries[21], + }, 'txtorcon': { + 'debian': version_entries[22], + 'gentoo': version_entries[23], + 'archlinux': version_entries[24], + 'slackware': version_entries[25], + 'freebsd': version_entries[26], + }, 'torsocks': { + 'mac': version_entries[27], + 'debian': version_entries[28], + 'fedora': version_entries[29], + 'gentoo': version_entries[30], + 'archlinux': version_entries[31], + 'slackware': version_entries[32], + 'freebsd': version_entries[33], + 'openbsd': version_entries[34], + }, 'ooni probe': { + 'mac': version_entries[35], + 'debian': version_entries[36], + 'archlinux': version_entries[37], + } + } + + def gentoo_version(request): # Unlike other platforms gentoo lists all package versions, so we # need to figure out what's the latest. @@ -118,6 +189,11 @@ def email_content(): lines.append(DIV) lines.append(COLUMN % ('Project', 'Platform', 'Version', 'Status')) + try: + wiki_versions = wiki_package_versions() + except IOError as exc: + return str(exc), True + has_issue = False for project, packages in PACKAGES: @@ -125,6 +201,11 @@ def email_content(): for package in packages: try: + wiki_version = wiki_versions[project][package.platform] + except KeyError: + return 'Failed to get wiki version for %s on %s' % (project, package.platform), True + + try: request = fetch_url(package.url) if package.platform == 'gentoo': @@ -136,10 +217,10 @@ def email_content(): if not current_version: msg = 'unable to determine current version' has_issue = True - elif current_version == package.version: + elif current_version == wiki_version: msg = 'up to date' else: - msg = 'current version is %s but wiki has %s' % (current_version, package.version) + msg = 'current version is %s but wiki has %s' % (current_version, wiki_version) has_issue = True except IOError as exc: msg = 'unable to retrieve current version: %s' % exc @@ -150,7 +231,7 @@ def email_content(): if package.platform == 'gentoo': has_issue = True - lines.append(COLUMN % (project, package.platform, package.version, msg)) + lines.append(COLUMN % (project, package.platform, wiki_version, msg)) lines.append(DIV) return '\n'.join(lines), has_issue