[tor-commits] [doctor/master] Dynamically get package versions from the wiki

atagar at torproject.org atagar at torproject.org
Mon Feb 12 03:22:52 UTC 2018


commit d7c1caadb5af4431ad8bafb54bbb1fb716b88a68
Author: Damian Johnson <atagar at torproject.org>
Date:   Sun Feb 11 19:15:45 2018 -0800

    Dynamically get package versions from the wiki
    
    The biggest time sync for me to keep this wiki up to date is bumping versions
    in this DocTor script. It's brittle, but instead having this script get the
    current versions on the wiki first so all I need to do to update is fix the
    wiki.
---
 package_versions.py | 161 +++++++++++++++++++++++++++++++++++++++-------------
 1 file changed, 121 insertions(+), 40 deletions(-)

diff --git a/package_versions.py b/package_versions.py
index 6c6e3f6..aa10bad 100755
--- a/package_versions.py
+++ b/package_versions.py
@@ -26,57 +26,58 @@ NETBSD_VERSION = 'CURRENT, <b>Version: </b>([0-9\.]+),'
 
 COLUMN = '| %-10s | %-10s | %-10s | %-50s |'
 DIV = '+%s+%s+%s+%s+' % ('-' * 12, '-' * 12, '-' * 12, '-' * 52)
+TRAC_URL = 'https://trac.torproject.org/projects/tor/wiki/doc/packages'
 
-Package = collections.namedtuple('Package', ['platform', 'url', 'version', 'regex'])
+Package = collections.namedtuple('Package', ['platform', 'url', 'regex'])
 
 PACKAGES = [
   ('tor', [
-    Package('mac', 'https://raw.githubusercontent.com/Homebrew/homebrew-core/master/Formula/tor.rb', '0.3.2.9', 'tor-([0-9\.]+).tar.gz'),
-    Package('debian', 'https://packages.debian.org/sid/tor', '0.3.2.9', DEBIAN_VERSION),
-    Package('fedora', 'https://apps.fedoraproject.org/packages/tor', '0.3.2.9', FEDORA_VERSION),
-    Package('gentoo', 'https://packages.gentoo.org/packages/net-vpn/tor', '0.3.2.9', None),
-    Package('archlinux', 'https://www.archlinux.org/packages/community/x86_64/tor/', '0.3.2.9', ARCH_LINUX_VERSION),
-    Package('slackware', 'https://slackbuilds.org/repository/14.2/network/tor/', '0.3.2.9', 'tor-([0-9\.]+).tar.gz'),
-    Package('freebsd', 'https://www.freshports.org/security/tor/', '0.3.2.9', FREEBSD_VERSION),
-    Package('openbsd', 'https://cvsweb.openbsd.org/cgi-bin/cvsweb/ports/net/tor/Makefile?rev=HEAD&content-type=text/x-cvsweb-markup', '0.3.2.9', OPENBSD_DIST_VERSION),
-    Package('netbsd', 'http://pkgsrc.se/net/tor', '0.3.2.9', NETBSD_VERSION),
+    Package('mac', 'https://raw.githubusercontent.com/Homebrew/homebrew-core/master/Formula/tor.rb', 'tor-([0-9\.]+).tar.gz'),
+    Package('debian', 'https://packages.debian.org/sid/tor', DEBIAN_VERSION),
+    Package('fedora', 'https://apps.fedoraproject.org/packages/tor', FEDORA_VERSION),
+    Package('gentoo', 'https://packages.gentoo.org/packages/net-vpn/tor', None),
+    Package('archlinux', 'https://www.archlinux.org/packages/community/x86_64/tor/', ARCH_LINUX_VERSION),
+    Package('slackware', 'https://slackbuilds.org/repository/14.2/network/tor/', 'tor-([0-9\.]+).tar.gz'),
+    Package('freebsd', 'https://www.freshports.org/security/tor/', FREEBSD_VERSION),
+    Package('openbsd', 'https://cvsweb.openbsd.org/cgi-bin/cvsweb/ports/net/tor/Makefile?rev=HEAD&content-type=text/x-cvsweb-markup', OPENBSD_DIST_VERSION),
+    Package('netbsd', 'http://pkgsrc.se/net/tor', NETBSD_VERSION),
   ]),
   ('nyx', [
-    Package('gentoo', 'https://packages.gentoo.org/packages/net-misc/nyx', '2.0.4', None),
-    Package('archlinux', 'https://aur.archlinux.org/packages/nyx/', '2.0.4', AUR_VERSION),
-    Package('slackware', 'https://slackbuilds.org/repository/14.2/python/nyx/', '2.0.4', 'nyx-([0-9\.]+).tar.gz'),
-    Package('freebsd', 'https://www.freshports.org/security/nyx/', '2.0.4', FREEBSD_VERSION),
-    Package('openbsd', 'https://cvsweb.openbsd.org/cgi-bin/cvsweb/ports/net/nyx/Makefile?rev=HEAD&content-type=text/x-cvsweb-markup', '2.0.4', OPENBSD_EGG_VERSION),
+    Package('gentoo', 'https://packages.gentoo.org/packages/net-misc/nyx', None),
+    Package('archlinux', 'https://aur.archlinux.org/packages/nyx/', AUR_VERSION),
+    Package('slackware', 'https://slackbuilds.org/repository/14.2/python/nyx/', 'nyx-([0-9\.]+).tar.gz'),
+    Package('freebsd', 'https://www.freshports.org/security/nyx/', FREEBSD_VERSION),
+    Package('openbsd', 'https://cvsweb.openbsd.org/cgi-bin/cvsweb/ports/net/nyx/Makefile?rev=HEAD&content-type=text/x-cvsweb-markup', OPENBSD_EGG_VERSION),
   ]),
   ('stem', [
-    Package('debian', 'https://packages.debian.org/sid/python-stem', '1.6.0', DEBIAN_VERSION),
-    Package('fedora', 'https://apps.fedoraproject.org/packages/python-stem', '1.6.0', FEDORA_VERSION),
-    Package('gentoo', 'https://packages.gentoo.org/packages/net-libs/stem', '1.6.0', None),
-    Package('archlinux', 'https://aur.archlinux.org/packages/stem/', '1.6.0', AUR_VERSION),
-    Package('slackware', 'https://slackbuilds.org/repository/14.2/python/stem/', '1.6.0', 'stem-([0-9\.]+).tar.gz'),
-    Package('freebsd', 'https://www.freshports.org/security/py-stem/', '1.6.0', FREEBSD_VERSION),
-    Package('openbsd', 'https://cvsweb.openbsd.org/cgi-bin/cvsweb/ports/net/py-stem/Makefile?rev=HEAD&content-type=text/x-cvsweb-markup', '1.6.0', OPENBSD_EGG_VERSION),
+    Package('debian', 'https://packages.debian.org/sid/python-stem', DEBIAN_VERSION),
+    Package('fedora', 'https://apps.fedoraproject.org/packages/python-stem', FEDORA_VERSION),
+    Package('gentoo', 'https://packages.gentoo.org/packages/net-libs/stem', None),
+    Package('archlinux', 'https://aur.archlinux.org/packages/stem/', AUR_VERSION),
+    Package('slackware', 'https://slackbuilds.org/repository/14.2/python/stem/', 'stem-([0-9\.]+).tar.gz'),
+    Package('freebsd', 'https://www.freshports.org/security/py-stem/', FREEBSD_VERSION),
+    Package('openbsd', 'https://cvsweb.openbsd.org/cgi-bin/cvsweb/ports/net/py-stem/Makefile?rev=HEAD&content-type=text/x-cvsweb-markup', OPENBSD_EGG_VERSION),
   ]),
   ('txtorcon', [
-    Package('debian', 'https://packages.debian.org/sid/python-txtorcon', '0.19.3', DEBIAN_VERSION),
-    Package('gentoo', 'https://packages.gentoo.org/packages/dev-python/txtorcon', '0.19.3', None),
-    Package('slackware', 'https://slackbuilds.org/repository/14.2/python/txtorcon/', '0.19.3', 'txtorcon-([0-9\.]+).tar.gz'),
-    Package('freebsd', 'https://www.freshports.org/security/py-txtorcon/', '0.19.3', FREEBSD_VERSION),
+    Package('debian', 'https://packages.debian.org/sid/python-txtorcon', DEBIAN_VERSION),
+    Package('gentoo', 'https://packages.gentoo.org/packages/dev-python/txtorcon', None),
+    Package('slackware', 'https://slackbuilds.org/repository/14.2/python/txtorcon/', 'txtorcon-([0-9\.]+).tar.gz'),
+    Package('freebsd', 'https://www.freshports.org/security/py-txtorcon/', FREEBSD_VERSION),
   ]),
   ('torsocks', [
-    Package('mac', 'https://raw.githubusercontent.com/Homebrew/homebrew-core/master/Formula/torsocks.rb', '2.2.0', ':tag => "v([0-9\.]+)",'),
-    Package('debian', 'https://packages.debian.org/sid/torsocks', '2.2.0', DEBIAN_VERSION),
-    Package('fedora', 'https://apps.fedoraproject.org/packages/torsocks', '2.1.0', FEDORA_VERSION),
-    Package('gentoo', 'https://packages.gentoo.org/packages/net-proxy/torsocks', '2.2.0', None),
-    Package('archlinux', 'https://www.archlinux.org/packages/community/x86_64/torsocks/', '2.2.0', ARCH_LINUX_VERSION),
-    Package('slackware', 'https://slackbuilds.org/repository/14.2/network/torsocks/', '2.2.0', 'torsocks \(([0-9\.]+)\)    </h2>'),
-    Package('freebsd', 'https://www.freshports.org/net/torsocks/', '2.2.0', 'SHA256 \(dgoulet-torsocks-v([0-9\.]+)_GH0.tar.gz\)'),
-    Package('openbsd', 'https://cvsweb.openbsd.org/cgi-bin/cvsweb/ports/net/torsocks/Makefile?rev=HEAD&content-type=text/x-cvsweb-markup', '1.2', OPENBSD_DIST_VERSION),
+    Package('mac', 'https://raw.githubusercontent.com/Homebrew/homebrew-core/master/Formula/torsocks.rb', ':tag => "v([0-9\.]+)",'),
+    Package('debian', 'https://packages.debian.org/sid/torsocks', DEBIAN_VERSION),
+    Package('fedora', 'https://apps.fedoraproject.org/packages/torsocks', FEDORA_VERSION),
+    Package('gentoo', 'https://packages.gentoo.org/packages/net-proxy/torsocks', None),
+    Package('archlinux', 'https://www.archlinux.org/packages/community/x86_64/torsocks/', ARCH_LINUX_VERSION),
+    Package('slackware', 'https://slackbuilds.org/repository/14.2/network/torsocks/', 'torsocks \(([0-9\.]+)\)    </h2>'),
+    Package('freebsd', 'https://www.freshports.org/net/torsocks/', 'SHA256 \(dgoulet-torsocks-v([0-9\.]+)_GH0.tar.gz\)'),
+    Package('openbsd', 'https://cvsweb.openbsd.org/cgi-bin/cvsweb/ports/net/torsocks/Makefile?rev=HEAD&content-type=text/x-cvsweb-markup', OPENBSD_DIST_VERSION),
   ]),
   ('ooni probe', [
-    Package('mac', 'https://raw.githubusercontent.com/Homebrew/homebrew-core/master/Formula/ooniprobe.rb', '2.2.0', 'ooniprobe-([0-9\.]+).tar.gz'),
-    Package('debian', 'https://packages.debian.org/sid/ooniprobe', '2.2.0', DEBIAN_VERSION),
-    Package('archlinux', 'https://aur.archlinux.org/packages/ooniprobe/', '1.2.2', AUR_VERSION),
+    Package('mac', 'https://raw.githubusercontent.com/Homebrew/homebrew-core/master/Formula/ooniprobe.rb', 'ooniprobe-([0-9\.]+).tar.gz'),
+    Package('debian', 'https://packages.debian.org/sid/ooniprobe', DEBIAN_VERSION),
+    Package('archlinux', 'https://aur.archlinux.org/packages/ooniprobe/', AUR_VERSION),
   ]),
 ]
 
@@ -94,6 +95,76 @@ def fetch_url(url):
         raise IOError(str(exc))
 
 
+def wiki_package_versions():
+  # Provides versions present on the wiki of the form...
+  #
+  #   {project => {platform => version}}
+  #
+  # Unfortunately the wiki table lacks good handles to match against so this is
+  # gonna be very, very brittle. That's fine though - this is just an effort
+  # saving measure for me anyway. ;P
+
+  request = fetch_url(TRAC_URL)
+  version_entries = []
+
+  for line in request.splitlines():
+    m = re.search('<b>Version:</b> <a href=".*">(.*)</a>', line)
+
+    if m:
+      version_entries.append(m.group(1))
+
+  if len(version_entries) != 38:
+    raise IOError('Table on %s no longer matches what this daemon expects (had %i entries)' % (TRAC_URL, len(version_entries)))
+
+  return {
+    'tor': {
+      'mac': version_entries[0],
+      'debian': version_entries[1],
+      'fedora': version_entries[2],
+      'gentoo': version_entries[3],
+      'archlinux': version_entries[4],
+      'slackware': version_entries[5],
+      'freebsd': version_entries[6],
+      'openbsd': version_entries[7],
+      'netbsd': version_entries[8],
+    }, 'nyx': {
+      'fedora': version_entries[9],
+      'gentoo': version_entries[10],
+      'archlinux': version_entries[11],
+      'slackware': version_entries[12],
+      'freebsd': version_entries[13],
+      'openbsd': version_entries[14],
+    }, 'stem': {
+      'debian': version_entries[15],
+      'fedora': version_entries[16],
+      'gentoo': version_entries[17],
+      'archlinux': version_entries[18],
+      'slackware': version_entries[19],
+      'freebsd': version_entries[20],
+      'openbsd': version_entries[21],
+    }, 'txtorcon': {
+      'debian': version_entries[22],
+      'gentoo': version_entries[23],
+      'archlinux': version_entries[24],
+      'slackware': version_entries[25],
+      'freebsd': version_entries[26],
+    }, 'torsocks': {
+      'mac': version_entries[27],
+      'debian': version_entries[28],
+      'fedora': version_entries[29],
+      'gentoo': version_entries[30],
+      'archlinux': version_entries[31],
+      'slackware': version_entries[32],
+      'freebsd': version_entries[33],
+      'openbsd': version_entries[34],
+    }, 'ooni probe': {
+      'mac': version_entries[35],
+      'debian': version_entries[36],
+      'archlinux': version_entries[37],
+    }
+  }
+
+
 def gentoo_version(request):
   # Unlike other platforms gentoo lists all package versions, so we
   # need to figure out what's the latest.
@@ -118,6 +189,11 @@ def email_content():
   lines.append(DIV)
   lines.append(COLUMN % ('Project', 'Platform', 'Version', 'Status'))
 
+  try:
+    wiki_versions = wiki_package_versions()
+  except IOError as exc:
+    return str(exc), True
+
   has_issue = False
 
   for project, packages in PACKAGES:
@@ -125,6 +201,11 @@ def email_content():
 
     for package in packages:
       try:
+        wiki_version = wiki_versions[project][package.platform]
+      except KeyError:
+        return 'Failed to get wiki version for %s on %s' % (project, package.platform), True
+
+      try:
         request = fetch_url(package.url)
 
         if package.platform == 'gentoo':
@@ -136,10 +217,10 @@ def email_content():
         if not current_version:
           msg = 'unable to determine current version'
           has_issue = True
-        elif current_version == package.version:
+        elif current_version == wiki_version:
           msg = 'up to date'
         else:
-          msg = 'current version is %s but wiki has %s' % (current_version, package.version)
+          msg = 'current version is %s but wiki has %s' % (current_version, wiki_version)
           has_issue = True
       except IOError as exc:
         msg = 'unable to retrieve current version: %s' % exc
@@ -150,7 +231,7 @@ def email_content():
         if package.platform == 'gentoo':
           has_issue = True
 
-      lines.append(COLUMN % (project, package.platform, package.version, msg))
+      lines.append(COLUMN % (project, package.platform, wiki_version, msg))
 
   lines.append(DIV)
   return '\n'.join(lines), has_issue





More information about the tor-commits mailing list