[or-cvs] r17085: {updater} Rename glider to thandy, based on discussions on #nottor. Pl (in updater/trunk: . lib lib/thandy specs)
nickm at seul.org
nickm at seul.org
Tue Oct 14 05:10:31 UTC 2008
Author: nickm
Date: 2008-10-14 01:10:30 -0400 (Tue, 14 Oct 2008)
New Revision: 17085
Added:
updater/trunk/lib/thandy/
updater/trunk/lib/thandy/ClientCLI.py
updater/trunk/lib/thandy/ServerCLI.py
updater/trunk/lib/thandy/SignerCLI.py
updater/trunk/lib/thandy/__init__.py
updater/trunk/lib/thandy/checkJson.py
updater/trunk/lib/thandy/download.py
updater/trunk/lib/thandy/formats.py
updater/trunk/lib/thandy/keys.py
updater/trunk/lib/thandy/master_keys.py
updater/trunk/lib/thandy/repository.py
updater/trunk/lib/thandy/tests.py
updater/trunk/lib/thandy/util.py
updater/trunk/specs/thandy-spec.txt
Removed:
updater/trunk/glider/
updater/trunk/lib/glider/
updater/trunk/lib/thandy/__init__.py
updater/trunk/lib/thandy/formats.py
updater/trunk/lib/thandy/keys.py
updater/trunk/lib/thandy/repository.py
updater/trunk/lib/thandy/tests.py
updater/trunk/specs/glider-spec.txt
Modified:
updater/trunk/Makefile
Log:
Rename glider to thandy, based on discussions on #nottor. Please let me know ASAP if there is another program Thandy, or if it means something rude, or whatever.
Modified: updater/trunk/Makefile
===================================================================
--- updater/trunk/Makefile 2008-10-14 05:04:40 UTC (rev 17084)
+++ updater/trunk/Makefile 2008-10-14 05:10:30 UTC (rev 17085)
@@ -3,4 +3,4 @@
test:
#python -m sexp.tests
- python -m glider.tests
+ python -m thandy.tests
Copied: updater/trunk/lib/thandy (from rev 17049, updater/trunk/lib/glider)
Copied: updater/trunk/lib/thandy/ClientCLI.py (from rev 17084, updater/trunk/lib/glider/ClientCLI.py)
===================================================================
--- updater/trunk/lib/thandy/ClientCLI.py (rev 0)
+++ updater/trunk/lib/thandy/ClientCLI.py 2008-10-14 05:10:30 UTC (rev 17085)
@@ -0,0 +1,66 @@
+
+import os
+import sys
+import getopt
+
+import thandy.util
+import thandy.repository
+import thandy.download
+
+def update(args):
+ repoRoot = thandy.util.userFilename("cache")
+ options, args = getopt.getopt(args, "", [ "repo=", "no-download" ])
+ download = True
+
+ for o, v in options:
+ if o == '--repo':
+ repoRoot = v
+ elif o == "--no-download":
+ download = False
+
+ repo = thandy.repository.LocalRepository(repoRoot)
+
+ files = repo.getFilesToUpdate(trackingBundles=args)
+
+ if not download:
+ return
+
+ mirrorlist = repo.getMirrorlistFile().get()
+
+ downloader = thandy.download.Downloads()
+ downloader.start()
+
+ for f in files:
+ # XXXX Use hash.
+ dj = thandy.download.DownloadJob(f, repo.getFilename(f),
+ mirrorlist)
+ downloader.addDownloadJob(dj)
+ # XXXX replace file in repository if ok; reload; see what changed.
+
+ # Wait for in-progress jobs
+
+# Check my repository
+
+# Tell me what I need to download
+
+# Download stuff
+
+# Tell me what to install.
+
+def usage():
+ print "Known commands:"
+ print " update [--repo=repository] [--no-download]"
+ sys.exit(1)
+
+def main():
+ if len(sys.argv) < 2:
+ usage()
+ cmd = sys.argv[1]
+ args = sys.argv[2:]
+ if cmd in [ "update" ]:
+ globals()[cmd](args)
+ else:
+ usage()
+
+if __name__ == '__main__':
+ main()
Copied: updater/trunk/lib/thandy/ServerCLI.py (from rev 17084, updater/trunk/lib/glider/ServerCLI.py)
===================================================================
--- updater/trunk/lib/thandy/ServerCLI.py (rev 0)
+++ updater/trunk/lib/thandy/ServerCLI.py 2008-10-14 05:10:30 UTC (rev 17085)
@@ -0,0 +1,187 @@
+
+import os
+import sys
+import getopt
+import time
+
+import simplejson
+
+import thandy.formats
+import thandy.util
+import thandy.keys
+
+def tstamp():
+ return time.strftime("%Y%m%d_%H%M%S", time.localtime())
+
+def snarf(fname):
+ f = open(fname, 'rb')
+ try:
+ return f.read()
+ finally:
+ f.close()
+
+def snarfObj(fname):
+ f = open(fname, 'r')
+ try:
+ return simplejson.load(f)
+ finally:
+ f.close()
+
+def insert(args):
+ repo = os.environ.get("THANDY_MASTER_REPO")
+ backupDir = thandy.util.userFilename("old_files")
+ checkSigs = True
+
+ options, args = getopt.getopt(args, "", ["repo=", "no-check"])
+ for o,v in options:
+ if o == "--repo":
+ repo = v
+ elif o == "--no-check":
+ checkSigs = False
+
+ if not repo:
+ print "No repository specified."
+ usage()
+ if not os.path.exists(repo):
+ print "No such repository as %r"%repo
+ usage()
+
+ if not os.path.exists(backupDir):
+ os.makedirs(backupDir, 0700)
+
+ if checkSigs:
+ keys = thandy.util.getKeylist(os.path.join(repo, "meta/keys.txt"))
+ else:
+ keys = None
+
+ n_ok = 0
+ for fn in args:
+ print "Loading %s..."%fn
+ try:
+ content = snarf(fn)
+ except OSError, e:
+ print "Couldn't open %s: %s"%(fn, e)
+ continue
+
+ try:
+ obj = simplejson.loads(content)
+ except ValueError, e:
+ print "Couldn't decode %s: %s"%(fn, e)
+ continue
+
+ try:
+ ss, r, path = thandy.formats.checkSignedObj(obj, keys)
+ except thandy.FormatException, e:
+ print "Bad format on %s: %s"%(fn, e)
+ continue
+ if checkSigs and not ss.isValid():
+ print "Not enough valid signatures on %s"%fn
+ continue
+
+ print " Looks okay. It goes in %s"%path
+ assert path.startswith("/")
+ targetPath = os.path.join(repo, path[1:])
+ if os.path.exists(targetPath):
+ oldContents = snarf(targetPath)
+ if oldContents == content:
+ print " File unchanged!"
+ n_ok += 1
+ continue
+
+ baseFname = "%s_%s" % (tstamp(), os.path.split(path)[1])
+ backupFname = os.path.join(backupDir, baseFname)
+ print " Copying old file to %s"%backupFname
+ thandy.util.replaceFile(backupFname, oldContents)
+
+ parentDir = os.path.split(targetPath)[0]
+ if not os.path.exists(parentDir):
+ print " Making %s"%parentDir
+ os.makedirs(parentDir, 0755)
+ print " Replacing file..."
+ thandy.util.replaceFile(targetPath, content)
+ print " Done."
+ n_ok += 1
+ if n_ok != len(args):
+ sys.exit(1)
+
+def timestamp(args):
+ repo = os.environ.get("THANDY_MASTER_REPO")
+ ts_keyfile = thandy.util.userFilename("timestamp_key")
+
+ options, args = getopt.getopt(args, "", ["repo=", "ts-key="])
+ for o,v in options:
+ if o == "--repo":
+ repo = v
+ elif o == "--ts-key":
+ ts_keyfile = v
+
+ if repo == None:
+ print "No repository specified."
+ usage()
+ if not os.path.exists(repo):
+ print "No such repository as %r"%repo
+ usage()
+
+ tsFname = os.path.join(repo, "meta/timestamp.txt")
+
+ try:
+ mObj = snarfObj(os.path.join(repo, "meta/mirrors.txt"))
+ except OSError:
+ print "No mirror list!"
+ sys.exit(1)
+ try:
+ kObj = snarfObj(os.path.join(repo, "meta/keys.txt"))
+ except OSError:
+ print "No key list!"
+ sys.exit(1)
+
+ bundles = []
+ for dirpath, dirname, fns in os.walk(os.path.join(repo, "bundleinfo")):
+ for fn in fns:
+ try:
+ bObj = snarfObj(fn)
+ except (ValueError, OSError), e:
+ print "(Couldn't read bundle-like %s)"%fn
+ continue
+ try:
+ _, r, _ = thandy.formats.checkSignedObj(bObj)
+ except thandy.FormatException, e:
+ print "Problem reading object from %s"%fn
+ continue
+ if r != "bundle":
+ print "%s was not a good bundle"%fn
+ continue
+ bundles.append(bObj['signed'])
+
+ timestamp = thandy.formats.makeTimestampObj(
+ mObj['signed'], kObj['signed'], bundles)
+ signable = thandy.formats.makeSignable(timestamp)
+
+ keydb = thandy.formats.Keylist()
+ #XXXX Still a roundabout way to do this.
+ keylist = thandy.formats.makeKeylistObj(ts_keyfile, True)
+ keydb.addFromKeylist(keylist)
+ for k in keydb.iterkeys():
+ thandy.formats.sign(signable, k)
+
+ content = simplejson.dumps(signable, sort_keys=True)
+ thandy.util.replaceFile(tsFname, content)
+
+def usage():
+ print "Known commands:"
+ print " insert [--no-check] [--repo=repository] file ..."
+ print " timestamp [--repo=repository]"
+ sys.exit(1)
+
+def main():
+ if len(sys.argv) < 2:
+ usage()
+ cmd = sys.argv[1]
+ args = sys.argv[2:]
+ if cmd in [ "insert", "timestamp" ]:
+ globals()[cmd](args)
+ else:
+ usage()
+
+if __name__ == '__main__':
+ main()
Copied: updater/trunk/lib/thandy/SignerCLI.py (from rev 17084, updater/trunk/lib/glider/SignerCLI.py)
===================================================================
--- updater/trunk/lib/thandy/SignerCLI.py (rev 0)
+++ updater/trunk/lib/thandy/SignerCLI.py 2008-10-14 05:10:30 UTC (rev 17085)
@@ -0,0 +1,313 @@
+
+import os
+import getopt
+import sys
+import logging
+import simplejson
+
+import thandy.keys
+import thandy.formats
+
+def getKeyStore():
+ return thandy.keys.KeyStore(thandy.util.userFilename("secret_keys"))
+
+def dumpKey(key, indent=0):
+ i = " "*indent
+ print "%s%s"%(i, key.getKeyID())
+ for r, p in key.getRoles():
+ print " %s%s\t%s"%(i, r, p)
+
+def getKey(ks, keyid=None, role=None, path=None):
+ if keyid is not None:
+ keys = ks.getKeysFuzzy(keyid)
+ if None not in (role, path):
+ keys = [ k for k in keys if k.hasRole(role, path) ]
+ elif None not in (role, path):
+ keys = ks.getKeysByRole(role, path)
+ else:
+ assert False
+ if len(keys) < 1:
+ print "No such key.\nI wanted",
+ if keyid: print "keyid='%s...'"%keyid,
+ if None not in (role, path): print "role=%s, path=%s"%(role,path),
+ print
+ print "I only know about:"
+ for k in ks.iterkeys():
+ dumpKey(k)
+ sys.exit(1)
+ elif len(keys) > 1:
+ print "Multiple keys match. Possibilities are:"
+ for k in keys:
+ dumpKey(k)
+ sys.exit(1)
+ else:
+ return keys[0]
+
+# ------------------------------
+
+def makepackage(args):
+ options, args = getopt.getopt(args, "", "keyid=")
+ keyid = None
+ for o,v in options:
+ if o == "--keyid":
+ keyid = v
+
+ if len(args) < 2:
+ usage()
+
+ configFile = args[0]
+ dataFile = args[1]
+ print "Generating package."
+ package = thandy.formats.makePackageObj(configFile, dataFile)
+ relpath = package['location']
+ print "need a key with role matching [package %s]"%relpath
+ ks = getKeyStore()
+ ks.load()
+ key = getKey(ks, keyid=keyid, role='package', path=relpath)
+ signable = thandy.formats.makeSignable(package)
+ thandy.formats.sign(signable, key)
+
+ if 1:
+ ss, r, p = thandy.formats.checkSignedObj(signable, ks)
+ assert ss.isValid()
+
+ location = os.path.split(package['location'])[-1]
+ print "Writing signed package to %s"%location
+ f = open(location, 'w')
+ simplejson.dump(signable, f, indent=1)
+ f.close()
+
+def makebundle(args):
+ options, args = getopt.getopt(args, "", "keyid=")
+ keyid = None
+ for o,v in options:
+ if o == "--keyid":
+ keyid = v
+
+ if len(args) < 2:
+ usage()
+
+ configFile = args[0]
+ packages = {}
+ for pkgFile in args[1:]:
+ print "Loading", pkgFile
+ f = open(pkgFile, 'r')
+ p = simplejson.load(f)
+ f.close()
+ _, r, _ = thandy.formats.checkSignedObj(p)
+ if r != 'package':
+ print pkgFile, "was not a package"
+ packages[p['signed']['location']] = p
+
+ def getHash(path):
+ p = packages[path]
+ return thandy.formats.getDigest(p['signed'])
+
+ bundleObj = thandy.formats.makeBundleObj(configFile, getHash)
+ signable = thandy.formats.makeSignable(bundleObj)
+
+ ks = getKeyStore()
+ ks.load()
+ key = getKey(ks, keyid=keyid, role="bundle", path=bundleObj['location'])
+ thandy.formats.sign(signable, key)
+
+ if 1:
+ ss, r, p = thandy.formats.checkSignedObj(signable, ks)
+ assert ss.isValid()
+
+ location = os.path.split(bundleObj['location'])[-1]
+ print "Writing signed bundle to %s"%location
+ f = open(location, 'w')
+ simplejson.dump(signable, f, indent=1)
+ f.close()
+
+# ------------------------------
+def makekeylist(args):
+ options, args = getopt.getopt(args, "", "keyid=")
+ keyid = None
+ for o,v in options:
+ if o == "--keyid":
+ keyid = v
+
+ if len(args) < 1:
+ usage()
+
+ keylist = thandy.formats.makeKeylistObj(args[0])
+ signable = thandy.formats.makeSignable(keylist)
+
+ ks = getKeyStore()
+ ks.load()
+ key = getKey(ks, keyid=keyid, role="master", path="/meta/keys.txt")
+ thandy.formats.sign(signable, key)
+
+ if 1:
+ ss, r, p = thandy.formats.checkSignedObj(signable, ks)
+ assert ss.isValid()
+
+ print "writing signed keylist to keys.txt"
+ thandy.util.replaceFile("keys.txt",
+ simplejson.dumps(signable, indent=1, sort_keys=True),
+ textMode=True)
+
+def signkeylist(args):
+ if len(args) != 1:
+ usage()
+
+ keylist = simplejson.load(open(args[0], 'r'))
+ thandy.formats.SIGNED_SCHEMA.checkMatch(keylist)
+ thandy.formats.KEYLIST_SCHEMA.checkMatch(keylist['signed'])
+
+ ks = getKeyStore()
+ ks.load()
+ keys = ks.getKeysByRole("master", "/meta/keys.txt")
+ for k in keys:
+ thandy.formats.sign(keylist, k)
+
+ print "writing signed keylist to keys.txt"
+ thandy.util.replaceFile("keys.txt",
+ simplejson.dumps(keylist, indent=1, sort_keys=True),
+ textMode=True)
+
+def makemirrorlist(args):
+ options, args = getopt.getopt(args, "", "keyid=")
+ keyid = None
+ for o,v in options:
+ if o == "--keyid":
+ keyid = v
+
+ if len(args) < 1:
+ usage()
+
+ mirrorlist = thandy.formats.makeMirrorListObj(args[0])
+ signable = thandy.formats.makeSignable(mirrorlist)
+
+ ks = getKeyStore()
+ ks.load()
+ key = getKey(ks, keyid=keyid, role='mirrors', path="/meta/mirrors.txt")
+ thandy.formats.sign(signable, key)
+
+ if 1:
+ ss, r, p = thandy.formats.checkSignedObj(signable, ks)
+ assert ss.isValid()
+
+ print "writing signed mirrorlist to mirrors.txt"
+ thandy.util.replaceFile("mirrors.txt",
+ simplejson.dumps(signable, indent=1, sort_keys=True),
+ textMode=True)
+
+# ------------------------------
+
+def keygen(args):
+ k = getKeyStore()
+ k.load()
+ print "Generating key. This will be slow."
+ key = thandy.keys.RSAKey.generate()
+ print "Generated new key: %s" % key.getKeyID()
+ k.addKey(key)
+ k.save()
+
+def listkeys(args):
+ k = getKeyStore()
+ k.load()
+ for k in k.iterkeys():
+ print k.getKeyID()
+ for r, p in k.getRoles():
+ print " ", r, p
+
+def addrole(args):
+ if len(args) < 3:
+ usage()
+ ks = getKeyStore()
+ ks.load()
+ k = getKey(ks, args[0])
+ r = args[1]
+ if r not in thandy.formats.ALL_ROLES:
+ print "Unrecognized role %r. Known roles are %s"%(
+ r,", ".join(thandy.format.ALL_ROLES))
+ sys.exit(1)
+ p = args[2]
+ k.addRole(r, p)
+ ks.save()
+
+def delrole(args):
+ if len(args) < 3:
+ usage()
+ ks = getKeyStore()
+ ks.load()
+ k = getKey(ks, args[0])
+ r = args[1]
+ if r not in thandy.formats.ALL_ROLES:
+ print "Unrecognized role %r. Known roles are %s"%(
+ r,", ".join(thandy.format.ALL_ROLES))
+ sys.exit(1)
+ p = args[2]
+
+ #XXXX rep.
+ origLen = len(k._roles)
+ k._roles = [ (role,path) for role,path in k._roles
+ if (role,path) != (r,p) ]
+ removed = origLen - len(k._roles)
+ print removed, "roles removed"
+ if removed:
+ ks.save()
+
+def chpass(args):
+ ks = getKeyStore()
+ print "Old password."
+ ks.load()
+ print "New password."
+ ks.clearPassword()
+ ks.save()
+
+def dumpkey(args):
+ options, args = getopt.getopt(args, "", ["include-secret", "passwd="])
+
+ includeSecret = False
+ for o,v in options:
+ if o == '--include-secret':
+ includeSecret = True
+ else:
+ print "Unexpected %r"%o
+
+ ks = getKeyStore()
+ ks.load()
+
+ keys = []
+ if len(args):
+ keys = [ getKey(ks, a) for a in args ]
+ else:
+ keys = list(ks.iterkeys())
+
+ for k in keys:
+ data = k.format(private=includeSecret, includeRoles=True)
+ print "Key(", simplejson.dumps(data, indent=2), ")"
+
+def usage():
+ print "Known commands:"
+ print " keygen"
+ print " listkeys"
+ print " chpass"
+ print " addrole keyid role path"
+ print " delrole keyid role path"
+ print " dumpkey [--include-secret] keyid"
+ print " makepackage config datafile"
+ print " makebundle config packagefile ..."
+ print " signkeylist keylist"
+ print " makekeylist keylist"
+ print " makemirrorlist config"
+ sys.exit(1)
+
+def main():
+ if len(sys.argv) < 2:
+ usage()
+ cmd = sys.argv[1]
+ args = sys.argv[2:]
+ if cmd in [ "keygen", "listkeys", "addrole", "delrole", "chpass",
+ "dumpkey", "makepackage", "makebundle", "signkeylist",
+ "makekeylist", "signkeylist", "makemirrorlist", ]:
+ globals()[cmd](args)
+ else:
+ usage()
+
+if __name__ == '__main__':
+ main()
Deleted: updater/trunk/lib/thandy/__init__.py
===================================================================
--- updater/trunk/lib/glider/__init__.py 2008-10-06 10:23:52 UTC (rev 17049)
+++ updater/trunk/lib/thandy/__init__.py 2008-10-14 05:10:30 UTC (rev 17085)
@@ -1,3 +0,0 @@
-
-__all__ = [ 'formats' ]
-
Copied: updater/trunk/lib/thandy/__init__.py (from rev 17084, updater/trunk/lib/glider/__init__.py)
===================================================================
--- updater/trunk/lib/thandy/__init__.py (rev 0)
+++ updater/trunk/lib/thandy/__init__.py 2008-10-14 05:10:30 UTC (rev 17085)
@@ -0,0 +1,35 @@
+
+__all__ = [ 'formats' ]
+
+_BaseException = Exception
+
+class Exception(_BaseException):
+ pass
+
+class FormatException(Exception):
+ pass
+
+class UnknownFormat(FormatException):
+ pass
+
+class BadSignature(Exception):
+ pass
+
+class BadPassword(Exception):
+ pass
+
+class InternalError(Exception):
+ pass
+
+class RepoError(InternalError):
+ pass
+
+class CryptoError(Exception):
+ pass
+
+class PubkeyFormatException(FormatException):
+ pass
+
+class UnknownMethod(CryptoError):
+ pass
+
Copied: updater/trunk/lib/thandy/checkJson.py (from rev 17084, updater/trunk/lib/glider/checkJson.py)
===================================================================
--- updater/trunk/lib/thandy/checkJson.py (rev 0)
+++ updater/trunk/lib/thandy/checkJson.py 2008-10-14 05:10:30 UTC (rev 17085)
@@ -0,0 +1,274 @@
+
+import re
+import sys
+
+import thandy
+
+class Schema:
+ def matches(self, obj):
+ try:
+ self.checkMatch(obj)
+ except thandy.FormatException:
+ return False
+ else:
+ return True
+
+ def checkMatch(self, obj):
+ raise NotImplemented()
+
+class Any(Schema):
+ """
+ >>> s = Any()
+ >>> s.matches("A String")
+ True
+ >>> s.matches([1, "list"])
+ True
+ """
+ def checkMatch(self, obj):
+ pass
+
+class RE(Schema):
+ """
+ >>> s = RE("h.*d")
+ >>> s.matches("hello world")
+ True
+ >>> s.matches("Hello World")
+ False
+ >>> s.matches("hello world!")
+ False
+ >>> s.matches([33, "Hello"])
+ False
+ """
+ def __init__(self, pat=None, modifiers=0, reObj=None, reName="pattern"):
+ if not reObj:
+ if not pat.endswith("$"):
+ pat += "$"
+ reObj = re.compile(pat, modifiers)
+ self._re = reObj
+ self._reName = reName
+ def checkMatch(self, obj):
+ if not isinstance(obj, basestring) or not self._re.match(obj):
+ raise thandy.FormatException("%r did not match %s"
+ %(obj,self._reName))
+
+class Str(Schema):
+ """
+ >>> s = Str("Hi")
+ >>> s.matches("Hi")
+ True
+ >>> s.matches("Not hi")
+ False
+ """
+ def __init__(self, val):
+ self._str = val
+ def checkMatch(self, obj):
+ if self._str != obj:
+ raise thandy.FormatException("Expected %r; got %r"%(self._str, obj))
+
+class AnyStr(Schema):
+ """
+ >>> s = AnyStr()
+ >>> s.matches("")
+ True
+ >>> s.matches("a string")
+ True
+ >>> s.matches(["a"])
+ False
+ >>> s.matches(3)
+ False
+ >>> s.matches(u"a unicode string")
+ True
+ >>> s.matches({})
+ False
+ """
+ def __init__(self):
+ pass
+ def checkMatch(self, obj):
+ if not isinstance(obj, basestring):
+ raise thandy.FormatException("Expected a string; got %r"%obj)
+
+class ListOf(Schema):
+ """
+ >>> s = ListOf(RE("(?:..)*"))
+ >>> s.matches("hi")
+ False
+ >>> s.matches([])
+ True
+ >>> s.matches({})
+ False
+ >>> s.matches(["Hi", "this", "list", "is", "full", "of", "even", "strs"])
+ True
+ >>> s.matches(["This", "one", "is not"])
+ False
+ """
+ def __init__(self, schema, minCount=0, maxCount=sys.maxint,listName="list"):
+ self._schema = schema
+ self._minCount = minCount
+ self._maxCount = maxCount
+ self._listName = listName
+ def checkMatch(self, obj):
+ if not isinstance(obj, (list, tuple)):
+ raise thandy.FormatException("Expected %s; got %r"
+ %(self._listName,obj))
+ for item in obj:
+ try:
+ self._schema.checkMatch(item)
+ except thandy.FormatException, e:
+ raise thandy.FormatException("%s in %s"%(e, self._listName))
+
+ if not (self._minCount <= len(obj) <= self._maxCount):
+ raise thandy.FormatException("Length of %s out of range"
+ %self._listName)
+
+class Struct(Schema):
+ """
+ >>> s = Struct([ListOf(AnyStr()), AnyStr(), Str("X")])
+ >>> s.matches(False)
+ False
+ >>> s.matches("Foo")
+ False
+ >>> s.matches([[], "Q", "X"])
+ True
+ >>> s.matches([[], "Q", "D"])
+ False
+ >>> s.matches([[3], "Q", "X"])
+ False
+ >>> s.matches([[], "Q", "X", "Y"])
+ False
+ """
+ def __init__(self, subschemas, allowMore=False, structName="list"):
+ self._subschemas = subschemas[:]
+ self._allowMore = allowMore
+ self._structName = structName
+ def checkMatch(self, obj):
+ if not isinstance(obj, (list, tuple)):
+ raise thandy.FormatException("Expected %s; got %r"
+ %(self._structName,obj))
+ elif len(obj) < len(self._subschemas):
+ raise thandy.FormatException(
+ "Too few fields in %s"%self._structName)
+ elif len(obj) > len(self._subschemas) and not self._allowMore:
+ raise thandy.FormatException(
+ "Too many fields in %s"%self._structName)
+ for item, schema in zip(obj, self._subschemas):
+ schema.checkMatch(item)
+
+class DictOf(Schema):
+ """
+ >>> s = DictOf(RE(r'[aeiou]+'), Struct([AnyStr(), AnyStr()]))
+ >>> s.matches("")
+ False
+ >>> s.matches({})
+ True
+ >>> s.matches({"a": ["x", "y"], "e" : ["", ""]})
+ True
+ >>> s.matches({"a": ["x", 3], "e" : ["", ""]})
+ False
+ >>> s.matches({"a": ["x", "y"], "e" : ["", ""], "d" : ["a", "b"]})
+ False
+ """
+ def __init__(self, keySchema, valSchema):
+ self._keySchema = keySchema
+ self._valSchema = valSchema
+ def checkMatch(self, obj):
+ try:
+ iter = obj.iteritems()
+ except AttributeError:
+ raise thandy.FormatException("Expected a dict; got %r"%obj)
+
+ for k,v in iter:
+ self._keySchema.checkMatch(k)
+ self._valSchema.checkMatch(v)
+
+class Opt:
+ """Helper; applied to a value in Obj to mark it optional.
+
+ >>> s = Obj(k1=Str("X"), k2=Opt(Str("Y")))
+ >>> s.matches({'k1': "X", 'k2': "Y"})
+ True
+ >>> s.matches({'k1': "X", 'k2': "Z"})
+ False
+ >>> s.matches({'k1': "X"})
+ True
+ """
+ def __init__(self, schema):
+ self._schema = schema
+ def checkMatch(self, obj):
+ self._schema.checkMatch(obj)
+
+class Obj(Schema):
+ """
+ >>> s = Obj(a=AnyStr(), bc=Struct([Int(), Int()]))
+ >>> s.matches({'a':"ZYYY", 'bc':[5,9]})
+ True
+ >>> s.matches({'a':"ZYYY", 'bc':[5,9], 'xx':5})
+ True
+ >>> s.matches({'a':"ZYYY", 'bc':[5,9,3]})
+ False
+ >>> s.matches({'a':"ZYYY"})
+ False
+
+ """
+ def __init__(self, _objname="object", **d):
+ self._objname = _objname
+ self._required = d.items()
+
+
+ def checkMatch(self, obj):
+ for k,schema in self._required:
+ try:
+ item = obj[k]
+ except KeyError:
+ if not isinstance(schema, Opt):
+ raise thandy.FormatException("Missing key %s in %s"
+ %(k,self._objname))
+
+ else:
+ try:
+ schema.checkMatch(item)
+ except thandy.FormatException, e:
+ raise thandy.FormatException("%s in %s.%s"
+ %(e,self._objname,k))
+
+
+class Int(Schema):
+ """
+ >>> s = Int()
+ >>> s.matches(99)
+ True
+ >>> s.matches(False)
+ False
+ >>> s.matches(0L)
+ True
+ >>> s.matches("a string")
+ False
+ >>> Int(lo=10, hi=30).matches(25)
+ True
+ >>> Int(lo=10, hi=30).matches(5)
+ False
+ """
+ def __init__(self, lo=-sys.maxint, hi=sys.maxint):
+ self._lo = lo
+ self._hi = hi
+ def checkMatch(self, obj):
+ if isinstance(obj, bool) or not isinstance(obj, (int, long)):
+ # We need to check for bool as a special case, since bool
+ # is for historical reasons a subtype of int.
+ raise thandy.FormatException("Got %r instead of an integer"%obj)
+ elif not (self._lo <= obj <= self._hi):
+ raise thandy.FormatException("%r not in range [%r,%r]"
+ %(obj, self._lo, self._hi))
+
+class Bool(Schema):
+ """
+ >>> s = Bool()
+ >>> s.matches(True) and s.matches(False)
+ True
+ >>> s.matches(11)
+ False
+ """
+ def __init__(self):
+ pass
+ def checkMatch(self, obj):
+ if not isinstance(obj, bool):
+ raise thandy.FormatException("Got %r instead of a boolean"%obj)
Copied: updater/trunk/lib/thandy/download.py (from rev 17084, updater/trunk/lib/glider/download.py)
===================================================================
--- updater/trunk/lib/thandy/download.py (rev 0)
+++ updater/trunk/lib/thandy/download.py 2008-10-14 05:10:30 UTC (rev 17085)
@@ -0,0 +1,127 @@
+
+
+import urllib2
+import httplib
+import random
+
+import threading, Queue
+
+import thandy.util
+
+class Downloads:
+ def __init__(self, n_threads=2):
+ self._lock = threading.RLock()
+ self.downloads = {}
+ self.haveDownloaded = {}
+ self.downloadQueue = Queue.Queue()
+ self.threads = [ threading.Thread(target=self._thread) ]
+ for t in self.threads:
+ t.setDaemon(True)
+
+ def start(self):
+ for t in self.threads:
+ t.start()
+
+ def isCurrentlyDownloading(self, relPath):
+ self._lock.acquire()
+ try:
+ return self.downloads.has_key(relPath)
+ finally:
+ self._lock.release()
+
+ def isRedundant(self, relPath):
+ self._lock.acquire()
+ try:
+ return (self.downloads.has_key(relPath) or
+ self.haveDownloaded.has_key(relPath))
+ finally:
+ self._lock.release()
+
+ def addDownloadJob(self, job):
+ rp = job.getRelativePath()
+ self._lock.acquire()
+ self.downloads[rp] = job
+ self._lock.release()
+ self.downloadQueue.put(job)
+
+ def _thread(self):
+ while True:
+ job = self.downloadQueue.get()
+ job.download()
+ rp = job.getRelativePath()
+ self._lock.acquire()
+ try:
+ del self.downloads[rp]
+ self.haveDownloaded[rp] = True
+ finally:
+ self._lock.release()
+
+class DownloadJob:
+ def __init__(self, relPath, destPath, mirrorlist=None,
+ wantHash=None, canStall=False):
+ self._relPath = relPath
+ self._wantHash = wantHash
+ self._mirrorList = mirrorlist
+ self._destPath = destPath
+
+ tmppath = thandy.util.userFilename("tmp")
+ if relPath.startswith("/"):
+ relPath = relPath[1:]
+ self._tmppath = os.path.join(tmppath, relPath)
+
+ d = os.path.dirname(self._tmppath)
+ if not os.path.exists(d):
+ os.makedirs(d, 0700)
+
+ def getRelativePath(self):
+ return self._relPath
+
+ def haveStalledFile(self):
+ return os.path.exists(self._tmppath)
+
+ def getURL(self, mirrorlist=None):
+ if mirrorlist is None:
+ mirrorlist = self._mirrorList
+ weightSoFar = 0
+ usable = []
+
+ for m in mirrorlist['mirrors']:
+ for c in m['contents']:
+ # CHECK FOR URL SUITABILITY XXXXX
+
+ if thandy.formats.rolePathMatches(c, self._relPath):
+ weightSoFar += m['weight']
+ usable.append( (weightSoFar, m) )
+ break
+
+ wTarget = random.randint(0, weightSoFar)
+ mirror = None
+ # Could use bisect here instead
+ for w, m in mirrorlist:
+ if w >= wTarget:
+ mirror = m
+ break
+
+ return m['urlbase'] + self._relPath
+
+ def download(self):
+ # XXXX RESUME
+
+ f_in = urllib2.urlopen(self.getURL())
+ f_out = open(self._tmpPath, 'w')
+ while True:
+ c = f_in.read(1024)
+ if not c:
+ break
+ f_out.write(c)
+ f_in.close()
+ f_out.close()
+ # XXXXX retry on failure
+
+ if self._wantHash:
+ gotHash = thandy.formats.getFileDigest(self._tmpPath)
+ if gotHash != self._wantHash:
+ # XXXX Corrupt file.
+ pass
+
+ thandy.utils.moveFile(self._tmpPath, self._destPath)
Deleted: updater/trunk/lib/thandy/formats.py
===================================================================
--- updater/trunk/lib/glider/formats.py 2008-10-06 10:23:52 UTC (rev 17049)
+++ updater/trunk/lib/thandy/formats.py 2008-10-14 05:10:30 UTC (rev 17085)
@@ -1,276 +0,0 @@
-
-import sexp.access
-import sexp.encode
-import time
-import re
-
-class FormatException(Exception):
- pass
-
-class KeyDB:
- def __init__(self):
- self.keys = {}
- def addKey(self, k):
- self.keys[k.getKeyID()] = k
- def getKey(self, keyid):
- return self.keys[keyid]
-
-_rolePathCache = {}
-def rolePathMatches(rolePath, path):
- """
-
- >>> rolePathMatches("a/b/c/", "a/b/c/")
- True
- >>> rolePathMatches("**/c.*", "a/b/c.txt")
- True
- >>> rolePathMatches("**/c.*", "a/b/c.txt/foo")
- False
- >>> rolePathMatches("a/*/c", "a/b/c")
- True
- >>> rolePathMatches("a/*/c", "a/b/c.txt")
- False
- >>> rolePathMatches("a/*/c", "a/b/c.txt") #Check cache
- False
- """
- try:
- regex = _rolePathCache[rolePath]
- except KeyError:
- rolePath = re.sub(r'/+', '/', rolePath)
- rolePath = re.escape(rolePath).replace(r'\*\*', r'.*')
- rolePath = rolePath.replace(r'\*', r'[^/]*')
- rolePath += "$"
- regex = _rolePathCache[rolePath] = re.compile(rolePath)
- return regex.match(path) != None
-
-def checkSignatures(signed, keyDB, role, path):
- goodSigs = []
- badSigs = []
- unknownSigs = []
- tangentialSigs = []
-
- assert signed[0] == "signed"
- data = signed[1]
-
- d_obj = Crypto.Hash.SHA256.new()
- sexp.encode.hash_canonical(data, d_obj)
- digest = d_obj.digest()
-
- for signature in sexp.access.s_children(signed, "signature"):
- attrs = signature[1]
- sig = attrs[2]
- keyid = s_child(attrs, "keyid")[1]
- try:
- key = keyDB.getKey(keyid)
- except KeyError:
- unknownSigs.append(keyid)
- continue
- method = s_child(attrs, "method")[1]
- try:
- result = key.checkSignature(method, sig, digest=digest)
- except UnknownMethod:
- continue
- if result == True:
- if role is not None:
- for r,p in key.getRoles():
- if r == role and rolePathMatches(p, path):
- break
- else:
- tangentialSigs.append(sig)
- continue
-
- goodSigs.append(keyid)
- else:
- badSigs.append(keyid)
-
- return goodSigs, badSigs, unknownSigs, tangentialSigs
-
-def sign(signed, key):
- assert sexp.access.s_tag(signed) == 'signed'
- s = signed[1]
- keyid = key.keyID()
-
- oldsignatures = [ s for s in signed[2:] if s_child(s[1], "keyid") != keyid ]
- signed[2:] = oldsignatures
-
- for method, sig in key.sign(s):
- signed.append(['signature', [['keyid', keyid], ['method', method]],
- sig])
-
-def formatTime(t):
- """
- >>> formatTime(1221265172)
- '2008-09-13 00:19:32'
- """
- return time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(t))
-
-def parseTime(s):
- return time.timegm(time.strptime(s, "%Y-%m-%d %H:%M:%S"))
-
-def _parseSchema(s, t=None):
- sexpr = sexp.parse.parse(s)
- schema = sexp.access.parseSchema(sexpr, t)
- return schema
-
-SCHEMA_TABLE = { }
-
-PUBKEY_TEMPLATE = r"""
- (=pubkey ((:unordered (=type .) (:anyof (. _)))) _)
-"""
-
-SCHEMA_TABLE['PUBKEY'] = _parseSchema(PUBKEY_TEMPLATE)
-
-TIME_TEMPLATE = r"""/\{d}4-\d{2}-\d{2} \d{2}:\d{2}:\d{2}/"""
-
-SCHEMA_TABLE['TIME'] = sexp.access.parseSchema(TIME_TEMPLATE)
-
-ATTRS_TEMPLATE = r"""(:anyof (_ *))"""
-
-SCHEMA_TABLE['ATTRS'] = _parseSchema(ATTRS_TEMPLATE)
-
-SIGNED_TEMPLATE = r"""
- (=signed
- _
- (:someof
- (=signature ((:unordered
- (=keyid _) (=method _) .ATTRS)) _)
- )
- )"""
-
-SIGNED_SCHEMA = _parseSchema(SIGNED_TEMPLATE, SCHEMA_TABLE)
-
-KEYLIST_TEMPLATE = r"""
- (=keylist
- (=ts .TIME)
- (=keys
- (:anyof
- (=key ((:unordered (=roles (:someof (. .))) .ATTRS)) _)
- ))
- *
- )"""
-
-KEYLIST_SCHEMA = _parseSchema(KEYLIST_TEMPLATE, SCHEMA_TABLE)
-
-MIRRORLIST_TEMPLATE = r"""
- (=mirrorlist
- (=ts .TIME)
- (=mirrors (:anyof
- (=mirror ((:unordered (=name .) (=urlbase .) (=contents (:someof .))
- .ATTRS)))))
- *)
-"""
-
-MIRRORLIST_SCHEMA = _parseSchema(MIRRORLIST_TEMPLATE, SCHEMA_TABLE)
-
-TIMESTAMP_TEMPLATE = r"""
- (=ts
- ((:unordered (=at .TIME) (=m .TIME .) (=k .TIME .)
- (:anyof (=b . . .TIME . .)) .ATTRS))
- )"""
-
-TIMESTAMP_SCHEMA = _parseSchema(TIMESTAMP_TEMPLATE, SCHEMA_TABLE)
-
-BUNDLE_TEMPLATE = r"""
- (=bundle
- (=at .TIME)
- (=os .)
- (:maybe (=arch .))
- (=packages
- (:someof
- (. . . . ((:unordered
- (:maybe (=order . . .))
- (:maybe (=optional))
- (:anyof (=gloss . .))
- (:anyof (=longgloss . .))
- .ATTRS)))
- )
- )
- *
- )"""
-
-BUNDLE_SCHEMA = _parseSchema(BUNDLE_TEMPLATE, SCHEMA_TABLE)
-
-PACKAGE_TEMPLATE = r"""
- (=package
- ((:unordered (=name .)
- (=version .)
- (=format . (.ATTRS))
- (=path .)
- (=ts .TIME)
- (=digest .)
- (:anyof (=shortdesc . .))
- (:anyof (=longdesc . .))
- .ATTRS)))
-"""
-
-PACKAGE_SCHEMA = _parseSchema(PACKAGE_TEMPLATE, SCHEMA_TABLE)
-
-ALL_ROLES = ('timestamp', 'mirrors', 'bundle', 'package', 'master')
-
-class Key:
- def __init__(self, key, roles):
- self.key = key
- self.roles = []
- for r,p in roles:
- self.addRole(r,p)
-
- def addRole(self, role, path):
- assert role in ALL_ROLES
- self.roles.append(role, path)
-
- def getRoles(self):
- return self.rules
-
- @staticmethod
- def fromSExpression(sexpr):
- # must match PUBKEY_SCHEMA
- typeattr = sexp.access.s_attr(sexpr[1], "type")
- if typeattr == 'rsa':
- key = glider.keys.RSAKey.fromSExpression(sexpr)
- if key is not None:
- return Key(key)
- else:
- return None
-
- def format(self):
- return self.key.format()
-
- def getKeyID(self):
- return self.key.getKeyID()
-
- def sign(self, sexpr=None, digest=None):
- return self.key.sign(sexpr, digest=digest)
-
- def checkSignature(self, method, sexpr=None, digest=None):
- if digest == None:
- _, digest = self.key._digest(sexpr, method)
- ok = self.key.checkSignature(method, digest=digest)
- # XXXX CACHE HERE.
- return ok
-
-class Keystore(KeyDB):
- def __init__(self):
- KeyDB.__init__(self)
-
- @staticmethod
- def addFromKeylist(sexpr, allowMasterKeys=False):
- # Don't do this until we have validated the structure.
- for ks in sexpr.access.s_lookup_all("keys.key"):
- attrs = ks[1]
- key_s = ks[2]
- roles = s_attr(attrs, "roles")
- #XXXX Use interface of Key, not RSAKey.
- key = Key.fromSExpression(key_s)
- if not key:
- #LOG skipping key.
- continue
- for r,p in roles:
- if r == 'master' and not allowMasterKeys:
- #LOG
- continue
- if r not in ALL_ROLES:
- continue
- key.addRole(r,p)
-
- self.addKey(key)
-
-
Copied: updater/trunk/lib/thandy/formats.py (from rev 17084, updater/trunk/lib/glider/formats.py)
===================================================================
--- updater/trunk/lib/thandy/formats.py (rev 0)
+++ updater/trunk/lib/thandy/formats.py 2008-10-14 05:10:30 UTC (rev 17085)
@@ -0,0 +1,747 @@
+
+import simplejson
+import time
+import re
+import binascii
+import calendar
+
+import thandy.checkJson
+
+import Crypto.Hash.SHA256
+
+class KeyDB:
+ """A KeyDB holds public keys, indexed by their key IDs."""
+ def __init__(self):
+ self._keys = {}
+ def addKey(self, k):
+ keyid = k.getKeyID()
+ try:
+ oldkey = self._keys[keyid]
+ for r, p in oldkey.getRoles():
+ if (r, p) not in k.getRoles():
+ k.addRole(r,p)
+ except KeyError:
+ pass
+ self._keys[k.getKeyID()] = k
+ def getKey(self, keyid):
+ return self._keys[keyid]
+ def getKeysByRole(self, role, path):
+ results = []
+ for key in self._keys.itervalues():
+ for r,p in key.getRoles():
+ if r == role:
+ if rolePathMatches(p, path):
+ results.append(key)
+ return results
+
+ def getKeysFuzzy(self, keyid):
+ r = []
+ for k,v in self._keys.iteritems():
+ if k.startswith(keyid):
+ r.append(v)
+ return r
+ def iterkeys(self):
+ return self._keys.itervalues()
+
+_rolePathCache = {}
+def rolePathMatches(rolePath, path):
+ """Return true iff the relative path in the filesystem 'path' conforms
+ to the pattern 'rolePath': a path that a given key is
+ authorized to sign. Patterns are allowed to contain * to
+ represent one or more characters in a filename, and ** to
+ represent any level of directory structure.
+
+ >>> rolePathMatches("a/b/c/", "a/b/c/")
+ True
+ >>> rolePathMatches("**/c.*", "a/b/c.txt")
+ True
+ >>> rolePathMatches("**/c.*", "a/b/ctxt")
+ False
+ >>> rolePathMatches("**/c.*", "a/b/c.txt/foo")
+ False
+ >>> rolePathMatches("a/*/c", "a/b/c")
+ True
+ >>> rolePathMatches("a/*/c", "a/b/c.txt")
+ False
+ >>> rolePathMatches("a/*/c", "a/b/c.txt") #Check cache
+ False
+ """
+ try:
+ regex = _rolePathCache[rolePath]
+ except KeyError:
+ orig = rolePath
+ # remove duplicate slashes.
+ rolePath = re.sub(r'/+', '/', rolePath)
+ # escape, then ** becomes .*
+ rolePath = re.escape(rolePath).replace(r'\*\*', r'.*')
+ # * becomes [^/]*
+ rolePath = rolePath.replace(r'\*', r'[^/]*')
+ # and no extra text is allowed.
+ rolePath += "$"
+ regex = _rolePathCache[orig] = re.compile(rolePath)
+ return regex.match(path) != None
+
+class SignatureStatus:
+ """Represents the outcome of checking signature(s) on an object."""
+ def __init__(self, good, bad, unrecognized, unauthorized):
+ # keyids for all the valid signatures
+ self._good = good[:]
+ # keyids for the invalid signatures (we had the key, and it failed).
+ self._bad = bad[:]
+ # keyids for signatures where we didn't recognize the key
+ self._unrecognized = unrecognized[:]
+ # keyids for signatures where we recognized the key, but it doesn't
+ # seem to be allowed to sign this kind of document.
+ self._unauthorized = unauthorized[:]
+
+ def isValid(self, threshold=1):
+ """Return true iff we got at least 'threshold' good signatures."""
+ return len(self._good) >= threshold
+
+ def mayNeedNewKeys(self):
+ """Return true iff downloading a new set of keys might tip this
+ signature status over to 'valid.'"""
+ return len(self._unrecognized) or len(self._unauthorized)
+
+def checkSignatures(signed, keyDB, role=None, path=None):
+ """Given an object conformant to SIGNED_SCHEMA and a set of public keys
+ in keyDB, verify the signed object in 'signed'."""
+
+ SIGNED_SCHEMA.checkMatch(signed)
+
+ goodSigs = []
+ badSigs = []
+ unknownSigs = []
+ tangentialSigs = []
+
+ signable = signed['signed']
+ signatures = signed['signatures']
+
+ d_obj = Crypto.Hash.SHA256.new()
+ getDigest(signable, d_obj)
+ digest = d_obj.digest()
+
+ for signature in signatures:
+ sig = signature['sig']
+ keyid = signature['keyid']
+ method = signature['method']
+
+ try:
+ key = keyDB.getKey(keyid)
+ except KeyError:
+ unknownSigs.append(keyid)
+ continue
+
+ try:
+ result = key.checkSignature(method, sig, digest=digest)
+ except thandy.UnknownMethod:
+ continue
+
+ if result == True:
+ if role is not None:
+ for r,p in key.getRoles():
+ if r == role and rolePathMatches(p, path):
+ break
+ else:
+ tangentialSigs.append(sig)
+ continue
+
+ goodSigs.append(keyid)
+ else:
+ badSigs.append(keyid)
+
+ return SignatureStatus(goodSigs, badSigs, unknownSigs, tangentialSigs)
+
+def encodeCanonical(obj, outf=None):
+ """Encode the object obj in canoncial JSon form, as specified at
+ http://wiki.laptop.org/go/Canonical_JSON . It's a restricted
+ dialect of json in which keys are always lexically sorted,
+ there is no whitespace, floats aren't allowed, and only quote
+ and backslash get escaped. The result is encoded in UTF-8,
+ and the resulting bits are passed to outf (if provided), or joined
+ into a string and returned.
+
+ >>> encodeCanonical("")
+ '""'
+ >>> encodeCanonical([1, 2, 3])
+ '[1,2,3]'
+ >>> encodeCanonical({"x" : 3, "y" : 2})
+ '{"x":3,"y":2}'
+ """
+ def default(o):
+ raise TypeError("Can't encode %r", o)
+ def floatstr(o):
+ raise TypeError("Floats not allowed.")
+ def canonical_str_encoder(s):
+ return '"%s"' % re.sub(r'(["\\])', r'\\\1', s)
+
+ # XXX This is, alas, a hack. I'll submit a canonical JSon patch to
+ # the simplejson folks.
+
+ iterator = simplejson.encoder._make_iterencode(
+ None, default, canonical_str_encoder, None, floatstr,
+ ":", ",", True, False, True)(obj, 0)
+
+ result = None
+ if outf == None:
+ result = [ ]
+ outf = result.append
+
+ for u in iterator:
+ outf(u.encode("utf-8"))
+ if result is not None:
+ return "".join(result)
+
+def getDigest(obj, digestObj=None):
+ """Update 'digestObj' (typically a SHA256 object) with the digest of
+ the canonical json encoding of obj. If digestObj is none,
+ compute the SHA256 hash and return it.
+
+ DOCDOC string equivalence.
+ """
+ useTempDigestObj = (digestObj == None)
+ if useTempDigestObj:
+ digestObj = Crypto.Hash.SHA256.new()
+
+ if isinstance(obj, str):
+ digestObj.update(obj)
+ elif isinstance(obj, unicode):
+ digestObj.update(obj.encode("utf-8"))
+ else:
+ encodeCanonical(obj, digestObj.update)
+
+ if useTempDigestObj:
+ return digestObj.digest()
+
+def getFileDigest(f, digestObj=None):
+ """Update 'digestObj' (typically a SHA256 object) with the digest of
+ the file object in f. If digestObj is none, compute the SHA256
+ hash and return it.
+
+ >>> s = "here is a long string"*1000
+ >>> import cStringIO, Crypto.Hash.SHA256
+ >>> h1 = Crypto.Hash.SHA256.new()
+ >>> h2 = Crypto.Hash.SHA256.new()
+ >>> getFileDigest(cStringIO.StringIO(s), h1)
+ >>> h2.update(s)
+ >>> h1.digest() == h2.digest()
+ True
+ """
+ useTempDigestObj = (digestObj == None)
+ if useTempDigestObj:
+ digestObj = Crypto.Hash.SHA256.new()
+
+ while 1:
+ s = f.read(4096)
+ if not s:
+ break
+ digestObj.update(s)
+
+ if useTempDigestObj:
+ return digestObj.digest()
+
+def makeSignable(obj):
+ return { 'signed' : obj, 'signatures' : [] }
+
+def sign(signed, key):
+ """Add an element to the signatures of 'signed', containing a new signature
+ of the "signed" part.
+ """
+
+ SIGNED_SCHEMA.checkMatch(signed)
+
+ signable = signed["signed"]
+ signatures = signed['signatures']
+
+ keyid = key.getKeyID()
+
+ signatures = [ s for s in signatures if s['keyid'] != keyid ]
+
+ method, sig = key.sign(signable)
+ signatures.append({ 'keyid' : keyid,
+ 'method' : method,
+ 'sig' : sig })
+ signed['signatures'] = signatures
+
+def formatTime(t):
+ """Encode the time 't' in YYYY-MM-DD HH:MM:SS format.
+
+ >>> formatTime(1221265172)
+ '2008-09-13 00:19:32'
+ """
+ return time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(t))
+
+def parseTime(s):
+ """Parse a time 's' in YYYY-MM-DD HH:MM:SS format."""
+ try:
+ return calendar.timegm(time.strptime(s, "%Y-%m-%d %H:%M:%S"))
+ except ValueError:
+ raise thandy.FormatError("Malformed time %r", s)
+
+def formatBase64(h):
+ """Return the base64 encoding of h with whitespace and = signs omitted."""
+ return binascii.b2a_base64(h).rstrip("=\n ")
+
+formatHash = formatBase64
+
+def parseBase64(s):
+ """Parse a base64 encoding with whitespace and = signs omitted. """
+ extra = len(s) % 4
+ if extra:
+ padding = "=" * (4 - extra)
+ s += padding
+ try:
+ return binascii.a2b_base64(s)
+ except binascii.Error:
+ raise thandy.FormatError("Invalid base64 encoding")
+
+def parseHash(s):
+ h = parseBase64(s)
+ if len(h) != Crypto.Hash.SHA256.digest_size:
+ raise thandy.FormatError("Bad hash length")
+ return h
+
+S = thandy.checkJson
+
+# A date, in YYYY-MM-DD HH:MM:SS format.
+TIME_SCHEMA = S.RE(r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}')
+# A hash, base64-encoded
+HASH_SCHEMA = S.RE(r'[a-zA-Z0-9\+\/]{43}')
+
+# A hexadecimal value.
+HEX_SCHEMA = S.RE(r'[a-fA-F0-9]+')
+# A base-64 encoded value
+BASE64_SCHEMA = S.RE(r'[a-zA-Z0-9\+\/]+')
+# An RSA key; subtype of PUBKEY_SCHEMA.
+RSAKEY_SCHEMA = S.Obj(
+ _keytype=S.Str("rsa"),
+ e=BASE64_SCHEMA,
+ n=BASE64_SCHEMA)
+# Any public key.
+PUBKEY_SCHEMA = S.Obj(
+ _keytype=S.AnyStr())
+
+KEYID_SCHEMA = HASH_SCHEMA
+SIG_METHOD_SCHEMA = S.AnyStr()
+RELPATH_SCHEMA = PATH_PATTERN_SCHEMA = S.AnyStr()
+URL_SCHEMA = S.AnyStr()
+VERSION_SCHEMA = S.ListOf(S.Any()) #XXXX WRONG
+
+# A single signature of an object. Indicates the signature, the id of the
+# signing key, and the signing method.
+SIGNATURE_SCHEMA = S.Obj(
+ keyid=KEYID_SCHEMA,
+ method=SIG_METHOD_SCHEMA,
+ sig=BASE64_SCHEMA)
+
+# A signed object.
+SIGNED_SCHEMA = S.Obj(
+ signed=S.Any(),
+ signatures=S.ListOf(SIGNATURE_SCHEMA))
+
+ROLENAME_SCHEMA = S.AnyStr()
+
+# A role: indicates that a key is allowed to certify a kind of
+# document at a certain place in the repo.
+ROLE_SCHEMA = S.Struct([ROLENAME_SCHEMA, PATH_PATTERN_SCHEMA])
+
+# A Keylist: indicates a list of live keys and their roles.
+KEYLIST_SCHEMA = S.Obj(
+ _type=S.Str("Keylist"),
+ ts=TIME_SCHEMA,
+ keys=S.ListOf(S.Obj(key=PUBKEY_SCHEMA, roles=S.ListOf(ROLE_SCHEMA))))
+
+# A Mirrorlist: indicates all the live mirrors, and what documents they
+# serve.
+MIRRORLIST_SCHEMA = S.Obj(
+ _type=S.Str("Mirrorlist"),
+ ts=TIME_SCHEMA,
+ mirrors=S.ListOf(S.Obj(name=S.AnyStr(),
+ urlbase=URL_SCHEMA,
+ contents=S.ListOf(PATH_PATTERN_SCHEMA),
+ weight=S.Int(lo=0),
+ )))
+
+# A timestamp: indicates the lastest versions of all top-level signed objects.
+TIMESTAMP_SCHEMA = S.Obj(
+ _type = S.Str("Timestamp"),
+ at = TIME_SCHEMA,
+ m = S.Struct([TIME_SCHEMA, HASH_SCHEMA]),
+ k = S.Struct([TIME_SCHEMA, HASH_SCHEMA]),
+ b = S.DictOf(keySchema=S.AnyStr(),
+ valSchema=
+ S.Struct([ VERSION_SCHEMA, RELPATH_SCHEMA, TIME_SCHEMA, HASH_SCHEMA ]))
+ )
+
+# A Bundle: lists a bunch of packages that should be updated in tandem
+BUNDLE_SCHEMA = S.Obj(
+ _type=S.Str("Bundle"),
+ at=TIME_SCHEMA,
+ name=S.AnyStr(),
+ os=S.AnyStr(),
+ arch=S.Opt(S.AnyStr()),
+ version=VERSION_SCHEMA,
+ location=RELPATH_SCHEMA,
+ packages=S.ListOf(S.Obj(
+ name=S.AnyStr(),
+ version=VERSION_SCHEMA,
+ path=RELPATH_SCHEMA,
+ hash=HASH_SCHEMA,
+ order=S.Struct([S.Int(), S.Int(), S.Int()]),
+ optional=S.Opt(S.Bool()),
+ gloss=S.DictOf(S.AnyStr(), S.AnyStr()),
+ longgloss=S.DictOf(S.AnyStr(), S.AnyStr()))))
+
+PACKAGE_SCHEMA = S.Obj(
+ _type=S.Str("Package"),
+ name=S.AnyStr(),
+ location=RELPATH_SCHEMA,
+ version=VERSION_SCHEMA,
+ format=S.Obj(),
+ ts=TIME_SCHEMA,
+ files=S.ListOf(S.Struct([RELPATH_SCHEMA, HASH_SCHEMA])),
+ shortdesc=S.DictOf(S.AnyStr(), S.AnyStr()),
+ longdesc=S.DictOf(S.AnyStr(), S.AnyStr()))
+
+ALL_ROLES = ('timestamp', 'mirrors', 'bundle', 'package', 'master')
+
+class Key:
+ #XXXX UNUSED.
+ def __init__(self, key, roles=()):
+ self.key = key
+ self.roles = []
+ for r,p in roles:
+ self.addRole(r,p)
+
+ def addRole(self, role, path):
+ assert role in ALL_ROLES
+ self.roles.append((role, path))
+
+ def getRoles(self):
+ return self.roles
+
+ @staticmethod
+ def fromJSon(obj):
+ # must match PUBKEY_SCHEMA
+ keytype = obj['_keytype']
+ if keytype == 'rsa':
+ return Key(thandy.keys.RSAKey.fromJSon(obj))
+
+ if typeattr == 'rsa':
+ key = thandy.keys.RSAKey.fromSExpression(sexpr)
+ if key is not None:
+ return Key(key)
+ else:
+ return None
+
+ def format(self):
+ return self.key.format()
+
+ def getKeyID(self):
+ return self.key.getKeyID()
+
+ def sign(self, sexpr=None, digest=None):
+ return self.key.sign(sexpr, digest=digest)
+
+ def checkSignature(self, method, data, signatute):
+ ok = self.key.checkSignature(method, data, signature)
+ # XXXX CACHE HERE.
+ return ok
+
+class Keylist(KeyDB):
+ def __init__(self):
+ KeyDB.__init__(self)
+
+ def addFromKeylist(self, obj, allowMasterKeys=False):
+ for keyitem in obj['keys']:
+ key = keyitem['key']
+ roles = keyitem['roles']
+
+ try:
+ key = thandy.keys.RSAKey.fromJSon(key)
+ except thandy.FormatException, e:
+ print e
+ #LOG skipping key.
+ continue
+
+ for r,p in roles:
+ if r == 'master' and not allowMasterKeys:
+ #LOG
+ continue
+ if r not in ALL_ROLES:
+ continue
+ key.addRole(r,p)
+
+ self.addKey(key)
+
+class StampedInfo:
+ def __init__(self, ts, hash, version=None, relpath=None):
+ self._ts = ts
+ self._hash = hash
+ self._version = version
+ self._relpath = relpath
+
+ @staticmethod
+ def fromJSonFields(timeStr, hashStr):
+ t = parseTime(timeStr)
+ h = parseHash(hashStr)
+ return StampedInfo(t, h)
+
+ def getHash(self):
+ return self._hash
+
+ def getRelativePath(self):
+ return self._relpath
+
+class TimestampFile:
+ def __init__(self, at, mirrorlistinfo, keylistinfo, bundleinfo):
+ self._time = at
+ self._mirrorListInfo = mirrorlistinfo
+ self._keyListInfo = keylistinfo
+ self._bundleInfo = bundleinfo
+
+ @staticmethod
+ def fromJSon(obj):
+ # must be validated.
+ at = parseTime(obj['at'])
+ m = StampedInfo.fromJSonFields(*obj['m'][:2])
+ k = StampedInfo.fromJSonFields(*obj['k'][:2])
+ b = {}
+ for name, bundle in obj['b'].iteritems():
+ v = bundle[0]
+ rp = bundle[1]
+ t = parseTime(bundle[2])
+ h = parseHash(bundle[3])
+ b[name] = StampedInfo(t, h, v, rp)
+
+ return TimestampFile(at, m, k, b)
+
+ def getTime(self):
+ return self._time
+
+ def getMirrorlistInfo(self):
+ return self._mirrorListInfo
+
+ def getKeylistInfo(self):
+ return self._keyListInfo
+
+ def getBundleInfo(self, name):
+ return self._bundleInfo[name]
+
+def readConfigFile(fname, needKeys=(), optKeys=(), preload={}):
+ parsed = preload.copy()
+ result = {}
+ execfile(fname, parsed)
+
+ for k in needKeys:
+ try:
+ result[k] = parsed[k]
+ except KeyError:
+ raise thandy.FormatError("Missing value for %s in %s"%k,fname)
+
+ for k in optKeys:
+ try:
+ result[k] = parsed[k]
+ except KeyError:
+ pass
+
+ return result
+
+def makePackageObj(config_fname, package_fname):
+ preload = {}
+ shortDescs = {}
+ longDescs = {}
+ def ShortDesc(lang, val): shortDescs[lang] = val
+ def LongDesc(lang, val): longDescs[lang] = val
+ preload = { 'ShortDesc' : ShortDesc, 'LongDesc' : LongDesc }
+ r = readConfigFile(config_fname,
+ ['name',
+ 'version',
+ 'format',
+ 'location',
+ 'relpath',
+ ], (), preload)
+
+ f = open(package_fname, 'rb')
+ digest = getFileDigest(f)
+
+ # Check fields!
+ result = { '_type' : "Package",
+ 'ts' : formatTime(time.time()),
+ 'name' : r['name'],
+ 'location' : r['location'], #DOCDOC
+ 'version' : r['version'],
+ 'format' : r['format'],
+ 'files' : [ [ r['relpath'], formatHash(digest) ] ],
+ 'shortdesc' : shortDescs,
+ 'longdesc' : longDescs
+ }
+
+ PACKAGE_SCHEMA.checkMatch(result)
+
+ return result
+
+def makeBundleObj(config_fname, getPackageHash):
+ packages = []
+ def ShortGloss(lang, val): packages[-1]['gloss'][lang] = val
+ def LongGloss(lang, val): packages[-1]['longgloss'][lang] = val
+ def Package(name, version, path, order, optional=False):
+ packages.append({'name' : name,
+ 'version' : version,
+ 'path' : path,
+ 'order' : order,
+ 'optional' : optional,
+ 'gloss' : {},
+ 'longgloss' : {} })
+ preload = { 'ShortGloss' : ShortGloss, 'LongGloss' : LongGloss,
+ 'Package' : Package }
+ r = readConfigFile(config_fname,
+ ['name',
+ 'os',
+ 'version',
+ 'location',
+ ], ['arch'], preload)
+
+ result = { '_type' : "Bundle",
+ 'at' : formatTime(time.time()),
+ 'name' : r['name'],
+ 'os' : r['os'],
+ 'version' : r['version'],
+ 'location' : r['location'],
+ 'packages' : packages }
+ if r.has_key('arch'):
+ result['arch'] = r['arch']
+
+ for p in packages:
+ try:
+ p['hash'] = formatHash(getPackageHash(p['path']))
+ except KeyError:
+ raise thandy.FormatException("No such package as %s"%p['path'])
+
+ BUNDLE_SCHEMA.checkMatch(result)
+ return result
+
+def versionIsNewer(v1, v2):
+ return v1 > v2
+
+def makeTimestampObj(mirrorlist_obj, keylist_obj,
+ bundle_objs):
+ result = { '_type' : 'Timestamp',
+ 'at' : formatTime(time.time()) }
+ result['m'] = [ mirrorlist_obj['ts'],
+ formatHash(getDigest(mirrorlist_obj)) ]
+ result['k'] = [ keylist_obj['ts'],
+ formatHash(getDigest(keylist_obj)) ]
+ result['b'] = bundles = {}
+ for bundle in bundle_objs:
+ name = bundle['name']
+ v = bundle['version']
+ entry = [ v, bundle['location'], bundle['at'], formatHash(getDigest(bundle)) ]
+ if not bundles.has_key(name) or versionIsNewer(v, bundles[name][0]):
+ bundles[name] = entry
+
+ TIMESTAMP_SCHEMA.checkMatch(result)
+
+ return result
+
+class MirrorInfo:
+ def __init__(self, name, urlbase, contents, weight):
+ self._name = name
+ self._urlbase = urlbase
+ self._contents = contents
+ self._weight = weight
+
+ def canServeFile(self, fname):
+ for c in self._contents:
+ if rolePathMatches(c, fname):
+ return True
+ return False
+
+ def getFileURL(self, fname):
+ if self._urlbase[-1] == '/':
+ return self._urlbase+fname
+ else:
+ return "%s/%s" % (self._urlbase, fname)
+
+ def format(self):
+ return { 'name' : self._name,
+ 'urlbase' : self._urlbase,
+ 'contents' : self._contents,
+ 'weight' : self._weight }
+
+def makeMirrorListObj(mirror_fname):
+ mirrors = []
+ def Mirror(*a, **kw): mirrors.append(MirrorInfo(*a, **kw))
+ preload = {'Mirror' : Mirror}
+ r = readConfigFile(mirror_fname, (), (), preload)
+ result = { '_type' : "Mirrorlist",
+ 'ts' : formatTime(time.time()),
+ 'mirrors' : [ m.format() for m in mirrors ] }
+
+ MIRRORLIST_SCHEMA.checkMatch(result)
+ return result
+
+def makeKeylistObj(keylist_fname, includePrivate=False):
+ keys = []
+ def Key(obj): keys.append(obj)
+ preload = {'Key': Key}
+ r = readConfigFile(keylist_fname, (), (), preload)
+
+ klist = []
+ for k in keys:
+ k = thandy.keys.RSAKey.fromJSon(k)
+ klist.append({'key': k.format(private=includePrivate), 'roles' : k.getRoles() })
+
+ result = { '_type' : "Keylist",
+ 'ts' : formatTime(time.time()),
+ 'keys' : klist }
+
+ KEYLIST_SCHEMA.checkMatch(result)
+ return result
+
+SCHEMAS_BY_TYPE = {
+ 'Keylist' : KEYLIST_SCHEMA,
+ 'Mirrorlist' : MIRRORLIST_SCHEMA,
+ 'Timestamp' : TIMESTAMP_SCHEMA,
+ 'Bundle' : BUNDLE_SCHEMA,
+ 'Package' : PACKAGE_SCHEMA,
+ }
+
+def checkSignedObj(obj, keydb=None):
+ # Returns signaturestatus, role, path on sucess.
+
+ SIGNED_SCHEMA.checkMatch(obj)
+ try:
+ tp = obj['signed']['_type']
+ except KeyError:
+ raise thandy.FormatException("Untyped object")
+ try:
+ schema = SCHEMAS_BY_TYPE[tp]
+ except KeyError:
+ raise thandy.FormatException("Unrecognized type %r" % tp)
+ schema.checkMatch(obj['signed'])
+
+ if tp == 'Keylist':
+ role = "master"
+ path = "/meta/keys.txt"
+ elif tp == 'Mirrorlist':
+ role = "mirrors"
+ path = "/meta/mirrors.txt"
+ elif tp == "Timestamp":
+ role = 'timestamp'
+ path = "/meta/timestamp.txt"
+ elif tp == 'Bundle':
+ role = 'bundle'
+ path = obj['signed']['location']
+ elif tp == 'Package':
+ role = 'package'
+ path = obj['signed']['location']
+ else:
+ print tp
+ raise "Foo"
+
+ ss = None
+ if keydb is not None:
+ ss = checkSignatures(obj, keydb, role, path)
+
+ return ss, role, path
Deleted: updater/trunk/lib/thandy/keys.py
===================================================================
--- updater/trunk/lib/glider/keys.py 2008-10-06 10:23:52 UTC (rev 17049)
+++ updater/trunk/lib/thandy/keys.py 2008-10-14 05:10:30 UTC (rev 17085)
@@ -1,283 +0,0 @@
-
-# These require PyCrypto.
-import Crypto.PublicKey.RSA
-import Crypto.Hash.SHA256
-import Crypto.Cipher.AES
-
-import sexp.access
-import sexp.encode
-import sexp.parse
-
-import cPickle as pickle
-import binascii
-import os
-import struct
-
-class CryptoError(Exception):
- pass
-
-class PubkeyFormatException(Exception):
- pass
-
-class UnknownMethod(Exception):
- pass
-
-class PublicKey:
- def format(self):
- raise NotImplemented()
- def sign(self, data):
- # returns a list of method,signature tuples.
- raise NotImplemented()
- def checkSignature(self, method, data, signature):
- # returns True, False, or raises UnknownMethod.
- raise NotImplemented()
- def getKeyID(self):
- raise NotImplemented()
- def getRoles(self):
- raise NotImplemented()
-
-if hex(1L).upper() == "0X1L":
- def intToBinary(number):
- """Convert an int or long into a big-endian series of bytes.
- """
- # This "convert-to-hex, then use binascii" approach may look silly,
- # but it's over 10x faster than the Crypto.Util.number approach.
- h = hex(long(number))
- h = h[2:-1]
- if len(h)%2:
- h = "0"+h
- return binascii.a2b_hex(h)
-elif hex(1L).upper() == "0X1":
- def intToBinary(number):
- h = hex(long(number))
- h = h[2:]
- if len(h)%2:
- h = "0"+h
- return binascii.a2b_hex(h)
-else:
- import Crypto.Util.number
- intToBinary = Crypto.Util.number.long_to_bytes
- assert None
-
-def binaryToInt(binary):
- """Convert a big-endian series of bytes into a long.
- """
- return long(binascii.b2a_hex(binary), 16)
-
-def _pkcs1_padding(m, size):
-
- # I'd rather use OAEP+, but apparently PyCrypto barely supports
- # signature verification, and doesn't seem to support signature
- # verification with nondeterministic padding. "argh."
-
- s = [ "\x00\x01", "\xff"* (size-3-len(m)), "\x00", m ]
- r = "".join(s)
- return r
-
-def _xor(a,b):
- if a:
- return not b
- else:
- return b
-
-class RSAKey(PublicKey):
- """
- >>> k = RSAKey.generate(bits=512)
- >>> sexpr = k.format()
- >>> sexpr[:2]
- ('pubkey', [('type', 'rsa')])
- >>> k1 = RSAKey.fromSExpression(sexpr)
- >>> k1.key.e == k.key.e
- True
- >>> k1.key.n == k.key.n
- True
- >>> k.getKeyID() == k1.getKeyID()
- True
- >>> s = ['tag1', ['foobar'], [['foop', 'bar']], 'baz']
- >>> method, sig = k.sign(sexpr=s)
- >>> k.checkSignature(method, sig, sexpr=s)
- True
- >>> s2 = [ s ]
- >>> k.checkSignature(method, sig, sexpr=s2)
- False
- """
- def __init__(self, key):
- self.key = key
- self.keyid = None
-
- @staticmethod
- def generate(bits=2048):
- key = Crypto.PublicKey.RSA.generate(bits=bits, randfunc=os.urandom)
- return RSAKey(key)
-
- @staticmethod
- def fromSExpression(sexpr):
- # sexpr must match PUBKEY_SCHEMA
- typeattr = sexp.access.s_attr(sexpr[1], "type")
- if typeattr != "rsa":
- return None
- if len(sexpr[2]) != 2:
- raise PubkeyFormatException("RSA keys must have an e,n pair")
- e,n = sexpr[2]
- key = Crypto.PublicKey.RSA.construct((binaryToInt(n), binaryToInt(e)))
- return RSAKey(key)
-
- def format(self):
- n = intToBinary(self.key.n)
- e = intToBinary(self.key.e)
- return ("pubkey", [("type", "rsa")], (e, n))
-
- def getKeyID(self):
- if self.keyid == None:
- n = intToBinary(self.key.n)
- e = intToBinary(self.key.e)
- keyval = (e,n)
- d_obj = Crypto.Hash.SHA256.new()
- sexp.encode.hash_canonical(keyval, d_obj)
- self.keyid = ("rsa", d_obj.digest())
- return self.keyid
-
- def _digest(self, sexpr, method=None):
- if method in (None, "sha256-pkcs1"):
- d_obj = Crypto.Hash.SHA256.new()
- sexp.encode.hash_canonical(sexpr, d_obj)
- digest = d_obj.digest()
- return ("sha256-pkcs1", digest)
-
- raise UnknownMethod(method)
-
- def sign(self, sexpr=None, digest=None):
- assert _xor(sexpr == None, digest == None)
- if digest == None:
- method, digest = self._digest(sexpr)
- m = _pkcs1_padding(digest, (self.key.size()+1) // 8)
- sig = intToBinary(self.key.sign(m, "")[0])
- return (method, sig)
-
- def checkSignature(self, method, sig, sexpr=None, digest=None):
- assert _xor(sexpr == None, digest == None)
- if method != "sha256-pkcs1":
- raise UnknownMethod("method")
- if digest == None:
- method, digest = self._digest(sexpr, method)
- sig = binaryToInt(sig)
- m = _pkcs1_padding(digest, (self.key.size()+1) // 8)
- return self.key.verify(m, (sig,))
-
-SALTLEN=16
-
-def secretToKey(salt, secret):
- """Convert 'secret' to a 32-byte key, using a version of the algorithm
- from RFC2440. The salt must be SALTLEN+1 bytes long, and should
- be random, except for the last byte, which encodes how time-
- consuming the computation should be.
-
- (The goal is to make offline password-guessing attacks harder by
- increasing the time required to convert a password to a key, and to
- make precomputed password tables impossible to generate by )
- """
- assert len(salt) == SALTLEN+1
-
- # The algorithm is basically, 'call the last byte of the salt the
- # "difficulty", and all other bytes of the salt S. Now make
- # an infinite stream of S|secret|S|secret|..., and hash the
- # first N bytes of that, where N is determined by the difficulty.
- #
- # Obviously, this wants a hash algorithm that's tricky to
- # parallelize.
- #
- # Unlike RFC2440, we use a 16-byte salt. Because CPU times
- # have improved, we start at 16 times the previous minimum.
-
- difficulty = ord(salt[-1])
- count = (16L+(difficulty & 15)) << ((difficulty >> 4) + 10)
-
- # Make 'data' nice and long, so that we don't need to call update()
- # a zillion times.
- data = salt[:-1]+secret
- if len(data)<1024:
- data *= (1024 // len(data))+1
-
- d = Crypto.Hash.SHA256.new()
- iters, leftover = divmod(count, len(data))
- for _ in xrange(iters):
- d.update(data)
- #count -= len(data)
- if leftover:
- d.update(data[:leftover])
- #count -= leftover
- #assert count == 0
-
- return d.digest()
-
-def encryptSecret(secret, password, difficulty=0x80):
- """Encrypt the secret 'secret' using the password 'password',
- and return the encrypted result."""
- # The encrypted format is:
- # "GKEY1" -- 5 octets, fixed, denotes data format.
- # SALT -- 17 bytes, used to hash password
- # IV -- 16 bytes; salt for encryption
- # ENCRYPTED IN AES256-OFB, using a key=s2k(password, salt) and IV=IV:
- # SLEN -- 4 bytes; length of secret, big-endian.
- # SECRET -- len(secret) bytes
- # D -- 32 bytes; SHA256 hash of (salt|secret|salt).
- #
- # This format leaks the secret length, obviously.
- assert 0 <= difficulty < 256
- salt = os.urandom(SALTLEN)+chr(difficulty)
- key = secretToKey(salt, password)
-
- d_obj = Crypto.Hash.SHA256.new()
- d_obj.update(salt)
- d_obj.update(secret)
- d_obj.update(salt)
- d = d_obj.digest()
-
- iv = os.urandom(16)
- e = Crypto.Cipher.AES.new(key, Crypto.Cipher.AES.MODE_OFB, iv)
-
- # Stupidly, pycrypto doesn't accept that stream ciphers don't need to
- # take their input in blocks. So pad it, then ignore the padded output.
-
- padlen = 16-((len(secret)+len(d)+4) % 16)
- if padlen == 16: padlen = 0
- pad = '\x00' * padlen
-
- slen = struct.pack("!L",len(secret))
- encrypted = e.encrypt("%s%s%s%s" % (slen, secret, d, pad))[:-padlen]
- return "GKEY1%s%s%s"%(salt, iv, encrypted)
-
-def decryptSecret(encrypted, password):
- if encrypted[:5] != "GKEY1":
- raise UnknownFormat()
- encrypted = encrypted[5:]
- if len(encrypted) < SALTLEN+1+16:
- raise FormatError()
-
- salt = encrypted[:SALTLEN+1]
- iv = encrypted[SALTLEN+1:SALTLEN+1+16]
- encrypted = encrypted[SALTLEN+1+16:]
-
- key = secretToKey(salt, password)
-
- e = Crypto.Cipher.AES.new(key, Crypto.Cipher.AES.MODE_OFB, iv)
- padlen = 16-(len(encrypted) % 16)
- if padlen == 16: padlen = 0
- pad = '\x00' * padlen
-
- decrypted = e.decrypt("%s%s"%(encrypted,pad))
- slen = struct.unpack("!L", decrypted[:4])[0]
- secret = decrypted[4:4+slen]
- hash = decrypted[4+slen:4+slen+Crypto.Hash.SHA256.digest_size]
-
- d = Crypto.Hash.SHA256.new()
- d.update(salt)
- d.update(secret)
- d.update(salt)
-
- if d.digest() != hash:
- print repr(decrypted)
- raise BadPassword()
-
- return secret
-
Copied: updater/trunk/lib/thandy/keys.py (from rev 17084, updater/trunk/lib/glider/keys.py)
===================================================================
--- updater/trunk/lib/thandy/keys.py (rev 0)
+++ updater/trunk/lib/thandy/keys.py 2008-10-14 05:10:30 UTC (rev 17085)
@@ -0,0 +1,399 @@
+
+# These require PyCrypto.
+import Crypto.PublicKey.RSA
+import Crypto.Hash.SHA256
+import Crypto.Cipher.AES
+
+import cPickle as pickle
+import binascii
+import logging
+import os
+import struct
+import sys
+import simplejson
+import getpass
+
+import thandy.formats
+import thandy.util
+
+class PublicKey:
+ def __init__(self):
+ # Confusingly, these roles are the ones used for a private key to
+ # remember what we're willing to do with it.
+ self._roles = []
+ def format(self):
+ raise NotImplemented()
+ def sign(self, data):
+ # returns a list of method,signature tuples.
+ raise NotImplemented()
+ def checkSignature(self, method, data, signature):
+ # returns True, False, or raises UnknownMethod.
+ raise NotImplemented()
+ def getKeyID(self):
+ raise NotImplemented()
+ def getRoles(self):
+ return self._roles
+ def addRole(self, role, path):
+ assert role in thandy.formats.ALL_ROLES
+ self._roles.append((role, path))
+ def clearRoles(self):
+ del self._roles[:]
+ def hasRole(self, role, path):
+ for r, p in self._roles:
+ if r == role and thandy.formats.rolePathMatches(p, path):
+ return True
+ return False
+
+if hex(1L).upper() == "0X1L":
+ def intToBinary(number):
+ """Convert an int or long into a big-endian series of bytes.
+ """
+ # This "convert-to-hex, then use binascii" approach may look silly,
+ # but it's over 10x faster than the Crypto.Util.number approach.
+ h = hex(long(number))
+ h = h[2:-1]
+ if len(h)%2:
+ h = "0"+h
+ return binascii.a2b_hex(h)
+elif hex(1L).upper() == "0X1":
+ def intToBinary(number):
+ h = hex(long(number))
+ h = h[2:]
+ if len(h)%2:
+ h = "0"+h
+ return binascii.a2b_hex(h)
+else:
+ import Crypto.Util.number
+ intToBinary = Crypto.Util.number.long_to_bytes
+ assert None
+
+def binaryToInt(binary):
+ """Convert a big-endian series of bytes into a long.
+ """
+ return long(binascii.b2a_hex(binary), 16)
+
+def intToBase64(number):
+ return thandy.formats.formatBase64(intToBinary(number))
+
+def base64ToInt(number):
+ return binaryToInt(thandy.formats.parseBase64(number))
+
+def _pkcs1_padding(m, size):
+ # I'd rather use OAEP+, but apparently PyCrypto barely supports
+ # signature verification, and doesn't seem to support signature
+ # verification with nondeterministic padding. "argh."
+
+ s = [ "\x00\x01", "\xff"* (size-3-len(m)), "\x00", m ]
+ r = "".join(s)
+ return r
+
+def _xor(a,b):
+ if a:
+ return not b
+ else:
+ return b
+
+class RSAKey(PublicKey):
+ """
+ >>> k = RSAKey.generate(bits=512)
+ >>> obj = k.format()
+ >>> obj['_keytype']
+ 'rsa'
+ >>> base64ToInt(obj['e'])
+ 65537L
+ >>> k1 = RSAKey.fromJSon(obj)
+ >>> k1.key.e == k.key.e
+ True
+ >>> k1.key.n == k.key.n
+ True
+ >>> k.getKeyID() == k1.getKeyID()
+ True
+ >>> s = { 'A B C' : "D", "E" : [ "F", "g", 99] }
+ >>> method, sig = k.sign(obj=s)
+ >>> k.checkSignature(method, sig, obj=s)
+ True
+ >>> s2 = [ s ]
+ >>> k.checkSignature(method, sig, obj=s2)
+ False
+ """
+ def __init__(self, key):
+ PublicKey.__init__(self)
+ self.key = key
+ self.keyid = None
+
+ @staticmethod
+ def generate(bits=2048):
+ key = Crypto.PublicKey.RSA.generate(bits=bits, randfunc=os.urandom)
+ return RSAKey(key)
+
+ @staticmethod
+ def fromJSon(obj):
+ # obj must match RSAKEY_SCHEMA
+
+ thandy.formats.RSAKEY_SCHEMA.checkMatch(obj)
+ n = base64ToInt(obj['n'])
+ e = base64ToInt(obj['e'])
+ if obj.has_key('d'):
+ d = base64ToInt(obj['d'])
+ p = base64ToInt(obj['p'])
+ q = base64ToInt(obj['q'])
+ u = base64ToInt(obj['u'])
+ key = Crypto.PublicKey.RSA.construct((n, e, d, p, q, u))
+ else:
+ key = Crypto.PublicKey.RSA.construct((n, e))
+
+ result = RSAKey(key)
+ if obj.has_key('roles'):
+ for r, p in obj['roles']:
+ result.addRole(r,p)
+
+ return result
+
+ def isPrivateKey(self):
+ return hasattr(self.key, 'd')
+
+ def format(self, private=False, includeRoles=False):
+ n = intToBase64(self.key.n)
+ e = intToBase64(self.key.e)
+ result = { '_keytype' : 'rsa',
+ 'e' : e,
+ 'n' : n }
+ if private:
+ result['d'] = intToBase64(self.key.d)
+ result['p'] = intToBase64(self.key.p)
+ result['q'] = intToBase64(self.key.q)
+ result['u'] = intToBase64(self.key.u)
+ if includeRoles:
+ result['roles'] = self.getRoles()
+ return result
+
+ def getKeyID(self):
+ if self.keyid == None:
+ d_obj = Crypto.Hash.SHA256.new()
+ thandy.formats.getDigest(self.format(), d_obj)
+ self.keyid = thandy.formats.formatHash(d_obj.digest())
+ return self.keyid
+
+ def _digest(self, obj, method=None):
+ if method in (None, "sha256-pkcs1"):
+ d_obj = Crypto.Hash.SHA256.new()
+ thandy.formats.getDigest(obj, d_obj)
+ digest = d_obj.digest()
+ return ("sha256-pkcs1", digest)
+
+ raise UnknownMethod(method)
+
+ def sign(self, obj=None, digest=None):
+ assert _xor(obj == None, digest == None)
+ if digest == None:
+ method, digest = self._digest(obj)
+ m = _pkcs1_padding(digest, (self.key.size()+1) // 8)
+ sig = intToBase64(self.key.sign(m, "")[0])
+ return (method, sig)
+
+ def checkSignature(self, method, sig, obj=None, digest=None):
+ assert _xor(obj == None, digest == None)
+ if method != "sha256-pkcs1":
+ raise UnknownMethod("method")
+ if digest == None:
+ method, digest = self._digest(obj, method)
+ sig = base64ToInt(sig)
+ m = _pkcs1_padding(digest, (self.key.size()+1) // 8)
+ return bool(self.key.verify(m, (sig,)))
+
+SALTLEN=16
+
+def secretToKey(salt, secret):
+ """Convert 'secret' to a 32-byte key, using a version of the algorithm
+ from RFC2440. The salt must be SALTLEN+1 bytes long, and should
+ be random, except for the last byte, which encodes how time-
+ consuming the computation should be.
+
+ (The goal is to make offline password-guessing attacks harder by
+ increasing the time required to convert a password to a key, and to
+ make precomputed password tables impossible to generate by )
+ """
+ assert len(salt) == SALTLEN+1
+
+ # The algorithm is basically, 'call the last byte of the salt the
+ # "difficulty", and all other bytes of the salt S. Now make
+ # an infinite stream of S|secret|S|secret|..., and hash the
+ # first N bytes of that, where N is determined by the difficulty.
+ #
+ # Obviously, this wants a hash algorithm that's tricky to
+ # parallelize.
+ #
+ # Unlike RFC2440, we use a 16-byte salt. Because CPU times
+ # have improved, we start at 16 times the previous minimum.
+
+ difficulty = ord(salt[-1])
+ count = (16L+(difficulty & 15)) << ((difficulty >> 4) + 10)
+
+ # Make 'data' nice and long, so that we don't need to call update()
+ # a zillion times.
+ data = salt[:-1]+secret
+ if len(data)<1024:
+ data *= (1024 // len(data))+1
+
+ d = Crypto.Hash.SHA256.new()
+ iters, leftover = divmod(count, len(data))
+ for _ in xrange(iters):
+ d.update(data)
+ #count -= len(data)
+ if leftover:
+ d.update(data[:leftover])
+ #count -= leftover
+ #assert count == 0
+
+ return d.digest()
+
+def encryptSecret(secret, password, difficulty=0x80):
+ """Encrypt the secret 'secret' using the password 'password',
+ and return the encrypted result."""
+ # The encrypted format is:
+ # "GKEY1" -- 5 octets, fixed, denotes data format.
+ # SALT -- 17 bytes, used to hash password
+ # IV -- 16 bytes; salt for encryption
+ # ENCRYPTED IN AES256-OFB, using a key=s2k(password, salt) and IV=IV:
+ # SLEN -- 4 bytes; length of secret, big-endian.
+ # SECRET -- len(secret) bytes
+ # D -- 32 bytes; SHA256 hash of (salt|secret|salt).
+ #
+ # This format leaks the secret length, obviously.
+ assert 0 <= difficulty < 256
+ salt = os.urandom(SALTLEN)+chr(difficulty)
+ key = secretToKey(salt, password)
+
+ d_obj = Crypto.Hash.SHA256.new()
+ d_obj.update(salt)
+ d_obj.update(secret)
+ d_obj.update(salt)
+ d = d_obj.digest()
+
+ iv = os.urandom(16)
+ e = Crypto.Cipher.AES.new(key, Crypto.Cipher.AES.MODE_OFB, iv)
+
+ # Stupidly, pycrypto doesn't accept that stream ciphers don't need to
+ # take their input in blocks. So pad it, then ignore the padded output.
+
+ padlen = 16-((len(secret)+len(d)+4) % 16)
+ if padlen == 16: padlen = 0
+ pad = '\x00' * padlen
+
+ slen = struct.pack("!L",len(secret))
+ encrypted = e.encrypt("%s%s%s%s" % (slen, secret, d, pad))
+ if padlen:
+ encrypted = encrypted[:-padlen]
+ return "GKEY1%s%s%s"%(salt, iv, encrypted)
+
+def decryptSecret(encrypted, password):
+ """Decrypt a value encrypted with encryptSecret. Raises UnknownFormat
+ or FormatError if 'encrypted' was not generated with encryptSecret.
+ Raises BadPassword if the password was not correct.
+ """
+ if encrypted[:5] != "GKEY1":
+ raise thandy.UnknownFormat()
+ encrypted = encrypted[5:]
+ if len(encrypted) < SALTLEN+1+16:
+ raise thandy.FormatException()
+
+ salt = encrypted[:SALTLEN+1]
+ iv = encrypted[SALTLEN+1:SALTLEN+1+16]
+ encrypted = encrypted[SALTLEN+1+16:]
+
+ key = secretToKey(salt, password)
+
+ e = Crypto.Cipher.AES.new(key, Crypto.Cipher.AES.MODE_OFB, iv)
+ padlen = 16-(len(encrypted) % 16)
+ if padlen == 16: padlen = 0
+ pad = '\x00' * padlen
+
+ decrypted = e.decrypt("%s%s"%(encrypted,pad))
+ slen = struct.unpack("!L", decrypted[:4])[0]
+ secret = decrypted[4:4+slen]
+ hash = decrypted[4+slen:4+slen+Crypto.Hash.SHA256.digest_size]
+
+ d = Crypto.Hash.SHA256.new()
+ d.update(salt)
+ d.update(secret)
+ d.update(salt)
+
+ if d.digest() != hash:
+ raise thandy.BadPassword()
+
+ return secret
+
+class KeyStore(thandy.formats.KeyDB):
+ def __init__(self, fname, encrypted=True):
+ thandy.formats.KeyDB.__init__(self)
+
+ self._loaded = None
+ self._fname = fname
+ self._passwd = None
+ self._encrypted = encrypted
+
+ def getpass(self, reprompt=False):
+ if self._passwd != None:
+ return self._passwd
+ while 1:
+ pwd = getpass.getpass("Password: ", sys.stderr)
+ if not reprompt:
+ return pwd
+
+ pwd2 = getpass.getpass("Confirm: ", sys.stderr)
+ if pwd == pwd2:
+ return pwd
+ else:
+ print "Mismatch; try again."
+
+ def load(self, password=None):
+ logging.info("Loading private keys from %r...", self._fname)
+ if not os.path.exists(self._fname):
+ logging.info("...no such file.")
+ self._loaded = True
+ return
+
+ if password is None and self._encrypted:
+ password = self.getpass()
+
+ contents = open(self._fname, 'rb').read()
+ if self._encrypted:
+ contents = decryptSecret(contents, password)
+
+ listOfKeys = simplejson.loads(contents)
+ self._passwd = password # It worked.
+ if not listOfKeys.has_key('keys'):
+ listOfKeys['keys'] = []
+ for obj in listOfKeys['keys']:
+ key = RSAKey.fromJSon(obj)
+ self.addKey(key)
+ logging.info("Loaded key %s", key.getKeyID())
+
+ self._loaded = True
+
+ def setPassword(self, passwd):
+ self._passwd = passwd
+
+ def clearPassword(self):
+ self._passwd = None
+
+ def save(self, password=None):
+ if not self._loaded and self._encrypted:
+ self.load(password)
+
+ if password is None:
+ password = self.getpass(True)
+
+ logging.info("Saving private keys into %r...", self._fname)
+ listOfKeys = { 'keys' :
+ [ key.format(private=True, includeRoles=True) for key in
+ self._keys.values() ]
+ }
+ contents = simplejson.dumps(listOfKeys)
+ if self._encrypted:
+ contents = encryptSecret(contents, password)
+ thandy.util.replaceFile(self._fname, contents)
+ self._passwd = password # It worked.
+ logging.info("Done.")
+
+
Copied: updater/trunk/lib/thandy/master_keys.py (from rev 17084, updater/trunk/lib/glider/master_keys.py)
===================================================================
--- updater/trunk/lib/thandy/master_keys.py (rev 0)
+++ updater/trunk/lib/thandy/master_keys.py 2008-10-14 05:10:30 UTC (rev 17085)
@@ -0,0 +1,5 @@
+
+
+MASTER_KEYS = [
+
+]
Deleted: updater/trunk/lib/thandy/repository.py
===================================================================
--- updater/trunk/lib/glider/repository.py 2008-10-06 10:23:52 UTC (rev 17049)
+++ updater/trunk/lib/thandy/repository.py 2008-10-14 05:10:30 UTC (rev 17085)
@@ -1,121 +0,0 @@
-
-import sexp.parse
-import sexp.access
-import glider.formats
-
-import os
-import threading
-
-class RepositoryFile:
- def __init__(self, repository, relativePath, schema,
- needRole=None, signedFormat=True, needSigs=1):
- self._repository = repository
- self._relativePath = relativePath
- self._schema = schema
- self._needRole = needRole
- self._signedFormat = signedFormat
- self._needSigs = needSigs
-
- self._signed_sexpr = None
- self._main_sexpr = None
- self._mtime = None
-
- def getPath(self):
- return os.path.join(self._repository._root, self._relativePath)
-
- def _load(self):
- fname = self.getPath()
-
- # Propagate OSError
- f = None
- fd = os.open(fname, os.O_RDONLY)
- try:
- f = os.fdopen(fd, 'r')
- except:
- os.close(fd)
- raise
- try:
- mtime = os.fstat(fd).st_mtime
- content = f.read()
- finally:
- f.close()
-
- signed_sexpr,main_sexpr = self._checkContent(content)
-
- self._signed_sexpr = signed_sexpr
- self._main_sexpr = main_sexpr
- self._mtime = mtime
-
- def _save(self, content=None):
- if content == None:
- content = sexpr.encode
-
- signed_sexpr,main_sexpr = self._checkContent(content)
-
- fname = self.getPath()
- fname_tmp = fname+"_tmp"
-
- fd = os.open(fname_tmp, os.WRONLY|os.O_CREAT|os.O_TRUNC, 0644)
- try:
- os.write(fd, contents)
- finally:
- os.close(fd)
- if sys.platform in ('cygwin', 'win32'):
- # Win32 doesn't let rename replace an existing file.
- try:
- os.unlink(fname)
- except OSError:
- pass
- os.rename(fname_tmp, fname)
-
- self._signed_sexpr = signed_sexpr
- self._main_sexpr = main_sexpr
- self._mtime = mtime
-
- def _checkContent(self, content):
- sexpr = sexp.parse.parse(content)
- if not sexpr:
- raise ParseError()
-
- if self._signedFormat:
- if not glider.formats.SIGNED_SCHEMA.matches(sexpr):
- raise FormatError()
-
- sigs = checkSignatures(sexpr, self._repository._keyDB,
- self._needRole, self._relativePath)
- good = sigs[0]
- # XXXX If good is too low but unknown is high, we may need
- # a new key file.
- if len(good) < 1:
- raise SignatureError()
-
- main_sexpr = sexpr[1]
- signed_sexpr = sexpr
- else:
- signed_sexpr = None
- main_sexpr = sexpr
-
- if self._schema != None and not self._schema.matches(main_sexpr):
- raise FormatError()
-
- return signed_sexpr, main_sexpr
-
- def load(self):
- if self._main_sexpr == None:
- self._load()
-
-class LocalRepository:
- def __init__(self, root):
- self._root = root
- self._keyDB = None
-
- self._keylistFile = RepositoryFile(
- self, "meta/keys.txt", glider.formats.KEYLIST_SCHEMA,
- needRole="master")
- self._timestampFile = RepositoryFile(
- self, "meta/timestamp.txt", glider.formats.TIMESTAMP_SCHEMA,
- needRole="timestamp")
- self._mirrorlistFile = RepositoryFile(
- self, "meta/mirrors.txt", glider.formats.MIRRORLIST_SCHEMA,
- needRole="mirrors")
-
Copied: updater/trunk/lib/thandy/repository.py (from rev 17084, updater/trunk/lib/glider/repository.py)
===================================================================
--- updater/trunk/lib/thandy/repository.py (rev 0)
+++ updater/trunk/lib/thandy/repository.py 2008-10-14 05:10:30 UTC (rev 17085)
@@ -0,0 +1,313 @@
+
+import thandy.formats
+import thandy.util
+
+import simplejson
+import logging
+import os
+import threading
+import time
+
+MAX_TIMESTAMP_AGE = 24*60*60
+
+class RepositoryFile:
+ def __init__(self, repository, relativePath, schema,
+ needRole=None, signedFormat=True, needSigs=1):
+ self._repository = repository
+ self._relativePath = relativePath
+ self._schema = schema
+ self._needRole = needRole
+ self._signedFormat = signedFormat
+ self._needSigs = needSigs
+
+ self._signed_obj = self._main_obj = None
+ self._sigStatus = None
+ self._mtime = None
+
+ def getRelativePath(self):
+ return self._relativePath
+
+ def getPath(self):
+ return self._repository.getFilename(self._relativePath)
+
+ def _load(self):
+ fname = self.getPath()
+
+ # Propagate OSError
+ f = None
+ fd = os.open(fname, os.O_RDONLY)
+ try:
+ f = os.fdopen(fd, 'r')
+ except:
+ os.close(fd)
+ raise
+ try:
+ mtime = os.fstat(fd).st_mtime
+ content = f.read()
+ finally:
+ f.close()
+
+ signed_obj,main_obj = self._checkContent(content)
+
+ self._signed_obj = signed_obj
+ self._main_obj = main_obj
+ self._mtime = mtime
+
+ def _save(self, content=None):
+ if content == None:
+ content = sexpr.encode
+
+ signed_obj,main_obj = self._checkContent(content)
+
+ fname = self.getPath()
+ thandy.util.replaceFile(fname, contents)
+
+ self._signed_obj = signed_obj
+ self._main_obj = main_obj
+ self._mtime = mtime
+
+ def _checkContent(self, content):
+
+ try:
+ obj = simplejson.loads(content)
+ except ValueError, e:
+ raise thandy.FormatException("Couldn't decode content: %s"%e)
+
+ if self._signedFormat:
+ # This is supposed to be signed.
+ thandy.formats.SIGNED_SCHEMA.checkMatch(obj)
+
+ main_obj = obj['signed']
+ signed_obj = obj
+ else:
+ signed_obj = None
+ main_obj = obj
+
+ if self._schema != None:
+ self._schema.checkMatch(main_obj)
+
+ return signed_obj, main_obj
+
+ def load(self):
+ if self._main_obj == None:
+ self._load()
+
+ def get(self):
+ return self._main_obj
+
+ def isLoaded(self):
+ return self._main_obj != None
+
+ def getContent(self):
+ self.load()
+ return self._main_obj
+
+ def _checkSignatures(self):
+ self.load()
+ sigStatus = thandy.formats.checkSignatures(self._signed_obj,
+ self._repository._keyDB,
+ self._needRole, self._relativePath)
+ self._sigStatus = sigStatus
+
+ def checkSignatures(self):
+ if self._sigStatus is None:
+ self._checkSignatures()
+ return self._sigStatus
+
+class LocalRepository:
+ def __init__(self, root):
+ self._root = root
+ self._keyDB = thandy.util.getKeylist(None)
+
+ self._keylistFile = RepositoryFile(
+ self, "/meta/keys.txt", thandy.formats.KEYLIST_SCHEMA,
+ needRole="master")
+ self._timestampFile = RepositoryFile(
+ self, "/meta/timestamp.txt", thandy.formats.TIMESTAMP_SCHEMA,
+ needRole="timestamp")
+ self._mirrorlistFile = RepositoryFile(
+ self, "/meta/mirrors.txt", thandy.formats.MIRRORLIST_SCHEMA,
+ needRole="mirrors")
+ self._metaFiles = [ self._keylistFile,
+ self._timestampFile,
+ self._mirrorlistFile ]
+
+ self._packageFiles = {}
+ self._bundleFiles = {}
+
+ def getFilename(self, relativePath):
+ if relativePath.startswith("/"):
+ relativePath = relativePath[1:]
+ return os.path.join(self._root, relativePath)
+
+ def getKeylistFile(self):
+ return self._keylistFile
+
+ def getTimestampFile(self):
+ return self._timestampFile
+
+ def getMirrorlistFile(self):
+ return self._mirrorlistFile
+
+ def getPackageFile(self, relPath):
+ try:
+ return self._packageFiles[relPath]
+ except KeyError:
+ self._packageFiles[relPath] = pkg = RepositoryFile(
+ self, relPath, thandy.formats.PACKAGE_SCHEMA,
+ needRole='package')
+ return pkg
+
+ def getBundleFile(self, relPath):
+ try:
+ return self._bundleFiles[relPath]
+ except KeyError:
+ self._bundleFiles[relPath] = pkg = RepositoryFile(
+ self, relPath, thandy.formats.BUNDLE_SCHEMA,
+ needRole='bundle')
+ return pkg
+
+ def getFilesToUpdate(self, now=None, trackingBundles=()):
+ if now == None:
+ now = time.time()
+
+ need = set()
+
+ # Fetch missing metafiles.
+ for f in self._metaFiles:
+ try:
+ f.load()
+ except OSError, e:
+ print "need", f.getPath()
+ logging.info("Couldn't load %s: %s. Must fetch it.",
+ f.getPath(), e)
+ need.add(f.getRelativePath())
+
+ # If the timestamp file is out of date, we need to fetch it no
+ # matter what. (Even if it is isn't signed, it can't possibly
+ # be good.)
+ ts = self._timestampFile.get()
+ if ts:
+ age = now - thandy.formats.parseTime(ts['at'])
+ ts = thandy.formats.TimestampFile.fromJSon(ts)
+ if age > MAX_TIMESTAMP_AGE:
+ need.add(self._timestampFile.getRelativePath())
+
+ # If the keylist isn't signed right, we can't check the
+ # signatures on anything else.
+ if self._keylistFile.get():
+ s = self._keylistFile.checkSignatures()
+ if not s.isValid(): # For now only require one master key.
+ need.add(self._keylistFile.getRelativePath())
+
+ if need:
+ return need
+
+ # Import the keys from the keylist.
+ self._keyDB.addFromKeylist(self._keylistFile.get())
+
+ # If the timestamp isn't signed right, get a new timestamp and a
+ # new keylist.
+ s = self._timestampFile.checkSignatures()
+ if not s.isValid():
+ need.add(self._keylistFile.getRelativePath())
+ need.add(self._timestampFile.getRelativePath())
+ return need
+
+ # FINALLY, we know we have an up-to-date, signed timestamp
+ # file. Check whether the keys and mirrors file are as
+ # authenticated.
+ h_kf = thandy.formats.getDigest(self._keylistFile.get())
+ h_expected = ts.getKeylistInfo().getHash()
+ if h_kf != h_expected:
+ need.add(self._keylistFile.getRelativePath())
+
+ if need:
+ return need
+
+ s = self._mirrorlistFile.checkSignatures()
+ if not s.isValid():
+ need.add(self._mirrorlistFile.getRelativePath())
+
+ h_mf = thandy.formats.getDigest(self._mirrorlistFile.get())
+ h_expected = ts.getMirrorlistInfo().getHash()
+ if h_mf != h_expected:
+ need.add(self._mirrorlistFile.getRelativePath())
+
+ if need:
+ return need
+
+ # Okay; that's it for the metadata. Do we have the right
+ # bundles?
+ bundles = {}
+ for b in trackingBundles:
+ try:
+ binfo = ts.getBundleInfo(b)
+ except KeyError:
+ logging.warn("Unrecognized bundle %s"%b)
+ continue
+
+ rp = binfo.getRelativePath()
+ bfile = self.getBundleFile(rp)
+ try:
+ bfile.load()
+ except OSError:
+ need.add(rp)
+ continue
+
+ h_b = thandy.formats.getDigest(bfile.get())
+ h_expected = binfo.getHash()
+ if h_b != h_expected:
+ need.add(rp)
+ continue
+
+ s = bfile.checkSignatures()
+ if not s.isValid():
+ # Can't actually use it.
+ continue
+
+ bundles[rp] = bfile
+
+ # Okay. So we have some bundles. See if we have their packages.
+ packages = {}
+ for bfile in bundles.values():
+ bundle = bfile.get()
+ for pkginfo in bundle['packages']:
+ rp = pkginfo['path']
+ pfile = self.getPackageFile(rp)
+ try:
+ pfile.load()
+ except OSError:
+ need.add(rp)
+ continue
+
+ h_p = thandy.formats.getDigest(pfile.get())
+ h_expected = thandy.formats.parseHash(pkginfo['hash'])
+ if h_p != h_expected:
+ need.add(rp)
+ continue
+
+ s = pfile.checkSignatures()
+ if not s.isValid():
+ # Can't use it.
+ continue
+ packages[rp] = pfile
+
+ # Finally, we have some packages. Do we have their underlying
+ # files?
+ for pfile in packages.values():
+ package = pfile.get()
+ for f in package['files']:
+ rp, h = f[:2]
+ h_expected = thandy.formats.parseHash(h)
+ fn = self.getFilename(rp)
+ try:
+ h_got = thandy.formats.getFileDigest(fn)
+ except OSError:
+ need.add(rp)
+ continue
+ if h_got != h_expected:
+ need.add(rp)
+
+ # Okay; these are the files we need.
+ return need
Deleted: updater/trunk/lib/thandy/tests.py
===================================================================
--- updater/trunk/lib/glider/tests.py 2008-10-06 10:23:52 UTC (rev 17049)
+++ updater/trunk/lib/thandy/tests.py 2008-10-14 05:10:30 UTC (rev 17085)
@@ -1,28 +0,0 @@
-
-import unittest
-import doctest
-
-import glider.keys
-import glider.formats
-import glider.repository
-
-import glider.tests
-
-class EncryptionTest(unittest.TestCase):
- pass
-
-def suite():
- suite = unittest.TestSuite()
-
- suite.addTest(doctest.DocTestSuite(glider.formats))
- suite.addTest(doctest.DocTestSuite(glider.keys))
-
- loader = unittest.TestLoader()
- suite.addTest(loader.loadTestsFromModule(glider.tests))
-
- return suite
-
-
-if __name__ == '__main__':
-
- unittest.TextTestRunner(verbosity=1).run(suite())
Copied: updater/trunk/lib/thandy/tests.py (from rev 17084, updater/trunk/lib/glider/tests.py)
===================================================================
--- updater/trunk/lib/thandy/tests.py (rev 0)
+++ updater/trunk/lib/thandy/tests.py 2008-10-14 05:10:30 UTC (rev 17085)
@@ -0,0 +1,64 @@
+
+import unittest
+import doctest
+import os
+import tempfile
+
+import thandy.keys
+import thandy.formats
+import thandy.repository
+import thandy.checkJson
+
+import thandy.tests
+
+class CanonicalEncodingTest(unittest.TestCase):
+ def test_encode(self):
+ enc = thandy.formats.encodeCanonical
+ self.assertEquals(enc(''), '""')
+ self.assertEquals(enc('"'), '"\\""')
+ self.assertEquals(enc('\t\\\n"\r'),
+ '"\t\\\\\n\\"\r"')
+
+class CryptoTests(unittest.TestCase):
+ def test_encrypt(self):
+ s = "The Secret words are marzipan habidashery zeugma."
+ password = "the password is swordfish."
+ encrypted = thandy.keys.encryptSecret(s, password)
+ self.assertNotEquals(encrypted, s)
+ self.assert_(encrypted.startswith("GKEY1"))
+ self.assertEquals(s, thandy.keys.decryptSecret(encrypted, password))
+ self.assertRaises(thandy.BadPassword, thandy.keys.decryptSecret,
+ encrypted, "password")
+ self.assertRaises(thandy.UnknownFormat, thandy.keys.decryptSecret,
+ "foobar", password)
+
+ def test_keystore(self):
+ passwd = "umfitty noonah"
+ fname = tempfile.mktemp()
+ ks = thandy.keys.KeyStore(fname)
+ key1 = thandy.keys.RSAKey.generate(512)
+ key2 = thandy.keys.RSAKey.generate(512)
+ ks.addKey(key1)
+ ks.addKey(key2)
+ ks.save(passwd)
+
+ ks2 = thandy.keys.KeyStore(fname)
+ ks2.load(passwd)
+ self.assertEquals(key1.key.n, ks2.getKey(key1.getKeyID()).key.n)
+
+def suite():
+ suite = unittest.TestSuite()
+
+ suite.addTest(doctest.DocTestSuite(thandy.formats))
+ suite.addTest(doctest.DocTestSuite(thandy.keys))
+ suite.addTest(doctest.DocTestSuite(thandy.checkJson))
+
+ loader = unittest.TestLoader()
+ suite.addTest(loader.loadTestsFromModule(thandy.tests))
+
+ return suite
+
+
+if __name__ == '__main__':
+
+ unittest.TextTestRunner(verbosity=1).run(suite())
Copied: updater/trunk/lib/thandy/util.py (from rev 17084, updater/trunk/lib/glider/util.py)
===================================================================
--- updater/trunk/lib/thandy/util.py (rev 0)
+++ updater/trunk/lib/thandy/util.py 2008-10-14 05:10:30 UTC (rev 17085)
@@ -0,0 +1,73 @@
+
+import os
+import sys
+import tempfile
+
+import simplejson
+
+import thandy.formats
+import thandy.keys
+import thandy.master_keys
+
+def moveFile(fromLocation, toLocation):
+ if sys.platform in ('cygwin', 'win32'):
+ # Win32 doesn't let rename replace an existing file.
+ try:
+ os.unlink(toLocation)
+ except OSError:
+ pass
+ os.rename(fromLocation, toLocation)
+
+
+def replaceFile(fname, contents, textMode=False):
+ """overwrite the file in 'fname' atomically with the content of 'contents'
+ """
+ dir, prefix = os.path.split(fname)
+ fd, fname_tmp = tempfile.mkstemp(prefix=prefix, dir=dir, text=textMode)
+
+ try:
+ os.write(fd, contents)
+ finally:
+ os.close(fd)
+
+ moveFile(fname_tmp, fname)
+
+def userFilename(name):
+ try:
+ base = os.environ["THANDY_HOME"]
+ except KeyError:
+ base = "~/.thandy"
+ base = os.path.expanduser(base)
+ if not os.path.exists(base):
+ os.makedirs(base, 0700)
+ return os.path.join(base, name)
+
+def getKeylist(keys_fname, checkKeys=True):
+ import thandy.master_keys
+
+ keydb = thandy.formats.Keylist()
+
+ for key in thandy.master_keys.MASTER_KEYS:
+ keydb.addKey(key)
+
+ user_keys = userFilename("preload_keys")
+ if os.path.exists(user_keys):
+ #XXXX somewhat roundabout.
+ keylist = thandy.formats.makeKeylistObj(user_keys)
+ keydb.addFromKeylist(keylist, allowMasterKeys=True)
+
+ if keys_fname and os.path.exists(keys_fname):
+ f = open(keys_fname, 'r')
+ try:
+ obj = simplejson.load(f)
+ finally:
+ f.close()
+ ss, role, path = thandy.formats.checkSignedObj(obj, keydb)
+ if role != 'master':
+ raise thandy.FormatException("%s wasn't a keylist."%keys_fname)
+ if checkKeys and not ss.isValid():
+ raise thandy.FormatException("%s not signed by enough master keys"%
+ keys_fname)
+ keydb.addFromKeylist(obj['signed'], allowMasterKeys=False)
+
+ return keydb
Deleted: updater/trunk/specs/glider-spec.txt
===================================================================
--- updater/trunk/specs/glider-spec.txt 2008-10-14 05:04:40 UTC (rev 17084)
+++ updater/trunk/specs/glider-spec.txt 2008-10-14 05:10:30 UTC (rev 17085)
@@ -1,710 +0,0 @@
-
- Glider: Automatic updates for Tor bundles
-
-0. Preliminaries
-
-0.0. Scope
-
- This document describes a system for distributing Tor binary bundle
- updates.
-
-0.1. Proposed code name
-
- Since "auto-update" is so generic, I've been thinking about going with
- "glider", based on the sugar glider you get when you search for "handy
- pocket creature". I haven't yet done a search to find out whether
- somebody else is using the name, so we shouldn't get too attached to it
- before we see if it's taken.
-
-0.2. Non-goals
-
- This is not meant to replace any existing download mechanism for
- users who prefer that mechanism. For example, just downloading
- source will still work fine.
-
- Similarly, we're not trying to force users who do not want to use
- downloaded binaries to use them, or to force users who do not want
- automatic updates to get them. {This should be obvious, but enough
- people have asked that I'm putting it in the document.}
-
- This is not a general-purpose package manager like yum or apt: it
- assumes that users will want to have one or more of a set of
- "bundles", not an arbitrary selection of packages dependant on one
- another. (Rationale: these systems do what they do pretty well.)
-
- This is also not a general-purpose package format. It assumes the
- existence of an external package format that can handle install,
- update, remove, and version query.
-
-0.3. Goals
-
- Once Tor was a single executable that you could just run. Then it
- required Privoxy. Now, thanks to the Tor Browser Bundle and related
- projects, a full installation can contain Tor, Privoxy, Torbutton,
- Firefox, and more.
-
- We need to keep this software updated. When we make security fixes,
- quick uptake helps narrow the window in which attackers can exploit
- them.
-
- We need updates to be easy. Each additional step a user must take to
- get updated means that more users will stay with older insecure
- versions.
-
- We need updates to be secure. We're supposed to be good at crypto;
- let's act like it. There is no good reason in this day and age to
- subject users to rollback attacks or unsigned packages or whatever.
-
- We need administration to be simple. Tor doesn't have a release
- engineering team, so we can't add too many hard steps to putting out
- a new release.
-
- The system should be easy to implement; we may need to do multiple
- implementations on the client side at least.
-
-0.3.1. Goals for package formats and PKIs
-
- It should be possible to mirror a repository using only rsync and
- cron.
-
- Separate keys should be used for different people and different
- roles.
-
- Only a minimal set of keys should have to be kept online to keep
- the system running.
-
- The system should handle any single computer or system or person
- being unavailable.
-
- The formats and protocols should be pretty future-proof.
-
-1. System overview
-
- The basic unit of updatability is a "bundle". A bundle is a set of
- software components, or "packages", plus some rules about installing
- them. Example bundles could be "Tor Browser, stable series" or
- "Basic Tor, development series".
-
- When Glider has responsibility for keeping a bundle up to date, we
- say that a user has "subscribed" to that bundle.
-
- Conceptually, there are four parts to keeping a bundle up to date:
-
- Polling:
- - Periodically, Glider asks a mirror whether there is a newer
- version of some bundle that a user has subscribed to. If so,
- Glider determines what's in the bundle.
-
- Fetching:
- - If the bundle contains packages that Glider hasn't installed
- or hasn't cached, it needs to download them from a mirror.
- This can happen over any protocol; v1 should support at least
- http and https-over-Tor. V1 should also support resuming
- partial downloads, since many users have unreliable
- connections.
-
- Later versions could support Bittorrent, or whatever.
-
- Validation:
- - Throughout the process, Glider must ensure that all the
- bundles are signed correctly, all the packages are signed
- correctly, and everything is up-to-date.
-
- We want to specify this so that users can't be tricked about
- the contents of a bundle, can't install a malicious package,
- and can't be fooled into believing that an old bundle is
- actually the latest.
-
- Installation:
- - Now Glider has a set of packages to install. The format of
- these packages will be platform-dependent: they could be pkg
- files on OSX, MSI files on Win32, RPMs or DEBs on Linux, and
- so on. Glider should query the user for permission to start
- installing packages, then install the packages. All other
- steps should generally happen automatically, in the
- background, without needing user intervention. This part
- needs user intervention because (A) it isn't nice to install
- updates without permission, and (B) in some configurations,
- it needs administrator privileges.
-
- (NO OTHER PART of this design needs administrator privileges.)
-
-1.1. The repository
-
- Each Glider instance knows about one or more "repositories". A
- repository is a filesystem somewhere that contains the packages in a
- set of bundles, and some associated metadata. A repository must
- exist at one or more canonical hosts, and may have a number of full
- or partial mirrors.
-
- In v1, each Glider instance will know about only one repository.
-
-1.2. The PKI
-
- The trust root for the whole system is, necessarily, whatever users
- download when they first download a copy of Glider. We need to make
- sure that the first download happens from a site we trust, using
- HTTPS.
-
- Glider ships with root keys, which in turn are used to verify the
- keys for all the other roles. There are a few root keys, operated by
- trusted admins for the system. If root keys ever need to be changed,
- we can just ship an update of Glider: it's supposed to be
- self-updating anyway.
-
- The root keys are only used to sign a 'key list' of all the other
- keys and their roles. A key list is valid if it has been signed by a
- threshold of root keys.
-
- Each package is signed with the key of its authorized builder. For
- example, one volunteer may be authorized to build the mac versions of
- several packages, and another may be authorized to build the windows
- version of just one.
-
- Each bundle is signed with the key of its maintainer. It's assumed
- that the bundle maintainer might be the package maintainer for some
- but not all of the packages.
-
- The list of mirrors is also signed. If the mirror list is
- automatically updated, this key must be kept online; otherwise, it
- can be offline.
-
- To prevent an adversary from replaying an out-of-date signed
- document, an automated process periodically signs a timestamped
- statement containing the hashes of the mirror list, the latest
- bundles, and the key list, using yet another special-purpose key.
- This key must be kept online.
-
-1.3. Threat Model And Analysis
-
- We assume an adversary who can operate compromised mirrors, and who
- can possibly compromise the main repository. At worst, such an
- adversary can DOS users in a way that they can detect.
-
- We're assuming for the moment an OSX/Win32-like execution model,
- where all packages will run equal privilege, but occasionally
- installation will require higher privilege. This means that once a
- hostile package is installed, it can basically do whatever it
- wants. As rootkit writers demonstrate, compromise is really
- tenuous: any attacker who can induce a user to install a hostile
- piece of code has, in effect, permanently compromised that user
- until they reinstall.
-
- Thus, if an adversary compromises enough keys to sign a compromised
- package, or tricks a packager into signing a compromised package,
- and manages to get that package into a signed bundle, the best we
- can do is to limit the number of users who are affected. We do
- this by compartmentalizing signing keys so that only the package
- and bundle in question are at risk.
-
- (If we had replicated build processes and a bit-by-bit reliable
- build process, we could have multiple packagers test that a binary
- was built properly, and multiply sign it. This would be effective
- against an adversary compromising a single packaging key, but not
- against one compromising a source repository.)
-
-2. The repository layout
-
- The filesystem layout in the repository is used for two purposes:
- - To give mirrors an easy way to mirror only some of the repository.
- - To specify which parts of the repository a given key has the
- authority to sign.
-
- The following files exist in all repositories and mirrors:
-
- /meta/keys.txt
-
- Signed by the root keys; indicates keys and roles.
- [???? I'm using the txt extension here. Is that smart?]
-
- /meta/mirrors.txt
-
- Signed by the mirror key; indicates which parts of the
- repository are mirrored at what mirrors.
-
- /meta/timestamp.txt
-
- Signed by the timestamp key; indicates hashes and timestamps
- for the latest versions of keys.txt and mirrors.txt. Also
- indicates the latest version of each bundle for each os/arch.
-
- This is the only file that needs to be downloaded for polling.
-
- /bundleinfo/bundlename/os-arch/bundlename-os-arch-bundleversion.txt
-
- Signed by the appropriate bundle key. Describes what
- packages make up a bundle, and what order to install,
- uninstall, and upgrade them in.
-
- /pkginfo/packagename/os-arch/version/packagename-os-arch-packageversion.txt
-
- Signed by the appropriate package key. Tells the name of the
- file that makes up a package, its hash, and what procedure
- is used to install it.
-
- /packages/packagename/os-arch/version/(some filename)
-
- The actual package file. Its naming convention will depend
- on the underlying packaging system.
-
-3. Document formats
-
-3.1. Metaformat
-
- All documents use Rivest's SEXP meta-format as documented at
- http://people.csail.mit.edu/rivest/sexp.html
- with the restriction that no "display hint" fields are to be used,
- and the base64 transit encoding isn't used either.
-
- (We use SEXP because it's really easy to parse, really portable,
- and unlike most other tagged data formats, has a
- trivially-specified canonical format suitable for hashing.)
-
- In descriptions of syntax below, we use regex-style qualifiers, so
- that in
- (sofa slipcover? occupant* leg+)
- the sofa will have an optional slipcover, zero or more occupants,
- and one or more legs. This pattern matches (sofa leg) and (sofa
- slipcover occupant occupant leg leg leg leg) but not (sofa leg
- slipcover).
-
- We also use a braces notation to indicate elements that can occur
- in any order. For example,
- (bread {flour+ eggs? yeast})
- matches a list starting with "bread", and then containing one or
- more of flours, zero or one occurrences of eggs, and one
- occurrence of yeast, in any order. This pattern matches (bread eggs
- yeast flour) but not (bread yeast) or (bread flour eggs yeast
- macadamias).
-
-3.2. File formats: general principles
-
- We use tagged lists (lists whose first element is a string) to
- indicate typed objects. Tags are generally lower-case, with
- hyphens used for separation. Think Lispy.
-
- We use attrlists [lists of (key value) lists] to indicate a
- multimap from keys to values. Clients MUST accept unrecognized
- keys in these attrlists. The syntax for an attrlist with two
- recognized and required keys is typically given as ({(key1 val1)
- (key2 val2) (ATTR VAL)*}), indicating that the keys can occur in
- any order, intermixed with other attributes.
-
- Timestamp files will be downloaded very frequently; all other files
- will be much smaller in size than package files. Thus,
- size-optimization for timestamp files makes sense and most other
- other space optimizations don't.
-
- Versions are represented as lists of the form (v I1 I2 I3 I4 ...)
- where each item is a number or alphanumeric version component. For
- example, the version "0.2.1.5-alpha" is represented as (v 0 2 1 5
- alpha).
-
- All signed files are of the format:
-
- (signed
- X
- (signature ({(keyid K) (method M) (ATTR VAL)*}) SIG)+
- )
-
- { "_type" : "Signed",
- "signed" : X,
- "sigatures" : [
- { "keyid" : K,
- "method" : M,
- ...
- "sig" : S } ]
-
- where: X is a list whose first element describes the signed object.
- K is the identifier of a key signing the document
- M is the method to be used to make the signature
- (ATTR VAL) is an arbitrary list whose first element is a
- string.
- SIG is a signature of the canonical encoding of X using the
- identified key.
-
- We define two signing methods at present:
- sha256-oaep : A signature of the SHA256 hash of the canonical
- encoding of X, using OAEP+ padding. [XXXX say more about mgf]
-
- All times are given as strings of the format "YYYY-MM-DD HH:MM:SS",
- in UTC.
-
- All keys are of the format:
- (pubkey ({(type TYPE) (ATTR VAL)*}) KEYVAL)
-
- { "_keytype" : TYPE,
- ...
- "keyval" : KEYVAL }
-
- where TYPE is a string describing the type of the key and how it's
- used to sign documents. The type determines the interpretation of
- KEYVAL.
-
- The ID of a key is a two-element list of the type and the SHA-256
- hash of the canonical encoding of the KEYVAL field.
-
- We define one keytype at present: 'rsa'. The KEYVAL in this case
- is a 2-element list of (e n), with both values given in big-endian
- binary format. [This makes keys 45-60% more compact than using
- decimal integers.]
-
- {Values given as integers.}
-
- {'e' : e, 'n' : n, big-endian hex. }
-
- All RSA keys must be at least 2048 bits long.
-
-
- Every role in the system is associated with a key. Replacing
- anything but a root key is supposed to be relatively easy.
-
- Root-keys sign other keys, and certify them as belonging to roles.
- Clients are configured to know the root keys.
-
- Bundle keys certify the contents of a bundle.
-
- Package keys certify packages for a given program or set of
- programs.
-
- Mirror keys certify a list of mirrors. We expect this to be an
- automated process.
-
- Timestamp keys certify that given versions of other metadata
- documents are up-to-date. They are the only keys that absolutely
- need to be kept online. (If they are not, timestamps won't be
- generated.)
-
-3.3. File formats: key list
-
- The key list file is signed by multiple root keys. It indicates
- which keys are authorized to sign which parts of the repository.
-
- (keylist
- (ts TIME)
- (keys
- ((key ({(roles (ROLE PATH)+) (ATTR VAL)*}) KEY)*)
- ...
- )
-
- { "_type" : "Keylist",
- "ts" : TIME,
- "keys" : [
- { "roles" : [ [ ROLE, PATH ], ... ],
- ...
- "key" : KEY }, ... ] }
-
- The "ts" line describes when the keys file was updated. Clients
- MUST NOT replace a file with an older one, and SHOULD NOT accept a
- file too far in the future.
-
- A ROLE is one of "timestamp" "mirrors" "bundle" or "package".
-
- PATH is a path relative to the top of the directory hierarchy. It
- may contain "*" elements to indicate "any file", and may end with a
- "/**" element to indicate all files under a given point.
-
-3.4. File formats: mirror list
-
- The mirror list is signed by a mirror key. It indicates which
- mirrors are active and believed to be mirroring which parts of the
- repository.
-
- (mirrorlist
- (ts TIME)
- (mirrors
- ( (mirror ({(name N) (urlbase U) (contents PATH+) (weight W)
- (official)? (ATTR VAL)})) * )
- ...
- )
-
- { "_type" : "Mirrorlist",
- "mirrors" : [
- { "name" : N,
- "urlbase" : U,
- "contents" : [PATH ... ] ,
- "weight" : W,
- "official" : BOOL,
- ...
- }, ... ]
- }
-
- Every mirror is a copy of some or all of the directory hierarchy
- containing at least the /meta, /bundles/, and /pkginfo directories.
-
- N is a descriptive name for the mirror; U is the URL of the mirror's
- base (i.e., the parent of the "meta" directory); and the PATH
- elements are the components describing how much of the packages
- directory is mirrored. Their format is as in the keylist file.
-
- W is an integer used to weight mirrors when picking at random;
- mirrors with more bandwidth should have higher weigths. The
- "official" element should only be present if the mirror is (one of
- the) official repositories operated by the Tor Project.
-
-3.5. File formats: timestamp files
-
- The timestamp file is signed by a timestamp key. It indicates the
- latest versions of other files, and contains a regularly updated
- timestamp to prevent rollback attacks.
-
- (ts
- ({(at TIME)
- (m TIME MIRRORLISTHASH)
- (k TIME KEYLISTHASH)
- (b NAME VERSION PATH TIME HASH)*})
- )
-
- { "_type" : Timestamp,
- "at" : TIME,
- "m" : [ TIME, HASH ],
- "k" : [ TIME, HASH ],
- "b" : { NAME :
- [ [ Version, Path, Time, Hash ] ] }
- }
-
- TIME is when the timestamp was signed. MIRRORLISTHASH is the digest
- of the mirror-list file; KEYLISTHASH is the digest of the key list
- file; and the 'b' entries are a list of the latest version of all
- bundles and their locations and hashes.
-
-3.6. File formats: bundle files
-
- (bundle
- (at TIME)
- (os OS)
- [(arch ARCH)]
- (version V)
- (packages
- (NAME VERSION PATH HASH ({(order INST UPDATE REMOVE)
- (optional)?
- (gloss LANG TEXT)*
- (longloss LANG TEXT)*
- (ATTR VAL)*})? )* )
- )
-
- { "_type" : "Bundle",
- "name" : NAME,
- "at" : TIME,
- "os" : OS,
- [ "arch" : ARCH, ]
- "version" : V
- "packages" :
- [ { "name" : NAME,
- "version" : VERSION,
- "path" : PATH,
- "hash" : HASH,
- "order" : [ INST, UPDATE, REMOVE ],
- [ "optional : BOOL, ]
- "gloss" : { LANG : TEXT },
- "longgloss" : { LANG : TEXT },
- } ] }
-
- Most elements are self-explanatory; the INST, UPDATE, and REMOVE
- elements of the order element are numbers defining the order in
- which the packages are installed, updated, and removed respectively.
- The "optional" element is present if the package is optional.
- "Gloss" is a short utf-8 human-readable string explaining what the
- package provides for the bundle; "longloss" is a longer such
- utf-8 string.
-
- (Note that the gloss strings are meant, not to describe the package,
- but to describe what the package provides for the bundle. For
- example, "The Anonymous Email Bundle needs the Python Runtime to run
- Mixminion.")
-
- Multiple gloss strings are allowed; each should have a different
- language. The UI should display the must appropriate language to the
- user.
-
-3.7. File formats: package files
-
- (package
- ({(name NAME)
- (version VERSION)
- (format FMT ((ATTR VAL)*)? )
- (path PATH)
- (ts TIME)
- (digest HASH)
- (shortdesc LANG TEXT)*
- (longdesc LANG TEXT)*
- (ATTR VAL)* })
- )
-
- Most elements are self-explanatory. The "FMT" element describes the
- file format of the package, which should give enough information
- about how to install it.
-
- No two package files in the same repository should have the same
- name and version. If a package needs to be changed, the version
- MUST be incremented.
-
- Descriptions are tagged with languages in the same way as glosses.
-
-4. Detailed Workflows
-
-4.1. The client application
-
- Periodically, the client updater fetches a timestamp file from a
- mirror. If the timestamp in the file is up-to-date, the client
- first checks to see whether the keys file listed is one that the
- client has. If not, the client fetches it, makes sure the hash of
- the keys file matches the hash in the timestamp file, makes sure its
- date is more recent than any keys file they have but not too far in
- the future, and that it is signed by enough root keys that the
- client recognizes.
-
- [If the timestamp file is not up-to-date, the client tries a
- few mirrors until it finds one with a good timestamp.]
-
- [If the keys file from a mirror does not match the timestamp
- file, the client tries a new mirror for both.]
-
- [If the keys file is not signed by enough root keys, the client
- warns the user and tries another mirror for both the timestamp
- file and the keys file.]
-
- Once the client has an up-to-date keys file, the client checks the
- signature on the timestamp file. Assuming it checks out, the client
- refreshes the mirror list as needed, and refreshes any bundle files
- to which the user is subscribed if the client does not have
- the latest version of those files. The client checks signatures on
- these files, and fetches package metadata for any packages listed in
- the bundle file that the client does not have, checks signatures on
- these, and fetches binaries for packages that might need to be
- installed or updated. As the packages arrive, clients check their
- hashes.
-
- Once the client has gotten enough packages, it informs the user that
- new packages have arrived, and asks them if they want to update.
-
- Clients SHOULD cache at least the latest versions they have received
- of all files.
-
-4.1.1. Download preferences
-
- Users should be able to specify that packages must be only
- downloaded over Tor, or must only be downloaded over encrypted
- protocols, or both. Users should also be able to express preference
- for Tor vs non-Tor and encrypted vs non-encrypted, even if they
- allow both.
-
-4.2. Mirrors
-
- Periodically, mirrors do an rsync or equivalent to fetch the latest
- version of whatever parts of the repository have changed since the
- version they currently hold. Mirrors SHOULD replace older versions
- of the repository idempotently, so that clients are less likely to
- see inconsistent state. Mirrors SHOULD validate the information
- they receive, and not serve partial or inconsistent files.
-
-4.3. Workflow: Packagers
-
- When a new binary package is done, the person making the package
- runs a tool to generate and sign a package file, and sends both the
- package and the package file to a repository admin. Typically, the
- base package file will be generated by inserting a version into a
- template.
-
- Packages MAY have as part of their build process a script to
- generate the appropriately versioned package file. This script
- should at a minimum demand a build version, or use a timestamp in
- place of a build version, to prevent two packages with the same
- version from being created.
-
-4.4. Workflow: bundlers
-
- When the packages in a bundle are done, the bundler runs a tool on
- the package files to generate and sign a bundle file. Typically,
- this tool uses a template bundle file.
-
-4.5. Workflow: repository administrators
-
- Repository administrators use a tool to validate signed files into the
- repository. The repository should not be altered manually.
-
- This tool acts as follows:
- - Package files may be added, but never replaced.
- - Bundle files may be added, but never replaced.
- - No file may be added unless it is syntactically valid and
- signed by a key in the keys file authorized to sign files of
- this type in this file's location location.
-
- - A package file may not be added unless all of its binary
- packages match their hashes.
-
- - A bundle file may not be added unless all of its package files
- are present and match their hashes.
-
- - When adding a new keylist, bundle, or mirrors list, the
- timestamp file must be regenerated immediately.
-
-5. Parameter setting and corner cases.
-
-5.1. Timing
-
- The timestamp file SHOULD be regenerated every 15 minutes. Mirrors
- SHOULD attempt to update every hour. Clients SHOULD accept a
- timestamp file up to 6 hours old.
-
-5.2. Format versioning and forward-compatibility:
-
- All of the above formats include the ability to add more
- attribute-value fields for backwards-compatible format changes. If
- we need to make a backwards incompatible format change, we create a
- new filename for the new format.
-
-5.3. Key management and migration:
-
- Root keys should be kept offline. All keys except timestamp and
- mirror keys should be stored encrypted.
-
- All the formats above allow for multiple keys to sign a single
- document. To replace a compromised root key, it suffices to sign
- keylist documents with both the compromised key and its replacement
- until all clients have updated to a new version of the autoupdater.
-
- To replace another key, it suffices to authorize the new key in the
- keylist. Note that a new package or bundle key must re-sign and
- issue new versions of all packages or bundles it has generated.
-
-
-
-F. Future directions and open questions
-
-F.1. Package decomposition
-
- It would be neat to decouple existing packages. Right now, we'd
- never want a windows user to have to fetch an openssl dll and Tor
- separately. But if they're using an auto-update tool, it'd be
- pretty keen to have them not need to fetch a new openssl every time
- Tor has a bugfix.
-
-F.2. Caching at Tor servers.
-
- See Tor Proposal number 127.
-
-F.3. Support for more download methods
-
- Ozymandns, chunked downloads, and bittorrent would all be neat
- ideas.
-
-F.4. Support for bogus clocks.
-
- Glider should have a user configurable "no, my clock is _supposed_
- to be wrong" mode, since lots of users seem to _like_ having their
- clocks in 1970 forever.
-
-R. Ideas I'm rejecting for the moment
-
-R.1. Considering recommended versions from Tor consensus directory documents
-
- This requires a working Tor to update Tor; that's not necessarily a
- great idea.
-
-R.2. Integration with existing GPG signatures
-
- The OpenPGP signature and key format is so complicated that you'd
- have to be mad to touch it.
-
-
Copied: updater/trunk/specs/thandy-spec.txt (from rev 17084, updater/trunk/specs/glider-spec.txt)
===================================================================
--- updater/trunk/specs/thandy-spec.txt (rev 0)
+++ updater/trunk/specs/thandy-spec.txt 2008-10-14 05:10:30 UTC (rev 17085)
@@ -0,0 +1,713 @@
+
+ Thandy: Automatic updates for Tor bundles
+
+0. Preliminaries
+
+0.0. Scope
+
+ This document describes a system for distributing Tor binary bundle
+ updates.
+
+0.1. Proposed code name
+
+ Since "auto-update" is so generic, I had been thinking about going with
+ "glider", based on the sugar glider you get when you search for "handy
+ pocket creature". Based on conversations, it seems that "glider"
+ is taken by a well-known WoW bot, so I'm rechristening this thing
+ as "Thandy" (which could stand for Tor's Handy pocket creature if
+ you want it to, or which could also be a person's first name).
+
+ Some of this document still refers to "Glider", and needs to be updated.
+
+0.2. Non-goals
+
+ This is not meant to replace any existing download mechanism for
+ users who prefer that mechanism. For example, just downloading
+ source will still work fine.
+
+ Similarly, we're not trying to force users who do not want to use
+ downloaded binaries to use them, or to force users who do not want
+ automatic updates to get them. {This should be obvious, but enough
+ people have asked that I'm putting it in the document.}
+
+ This is not a general-purpose package manager like yum or apt: it
+ assumes that users will want to have one or more of a set of
+ "bundles", not an arbitrary selection of packages dependant on one
+ another. (Rationale: these systems do what they do pretty well.)
+
+ This is also not a general-purpose package format. It assumes the
+ existence of an external package format that can handle install,
+ update, remove, and version query.
+
+0.3. Goals
+
+ Once Tor was a single executable that you could just run. Then it
+ required Privoxy. Now, thanks to the Tor Browser Bundle and related
+ projects, a full installation can contain Tor, Privoxy, Torbutton,
+ Firefox, and more.
+
+ We need to keep this software updated. When we make security fixes,
+ quick uptake helps narrow the window in which attackers can exploit
+ them.
+
+ We need updates to be easy. Each additional step a user must take to
+ get updated means that more users will stay with older insecure
+ versions.
+
+ We need updates to be secure. We're supposed to be good at crypto;
+ let's act like it. There is no good reason in this day and age to
+ subject users to rollback attacks or unsigned packages or whatever.
+
+ We need administration to be simple. Tor doesn't have a release
+ engineering team, so we can't add too many hard steps to putting out
+ a new release.
+
+ The system should be easy to implement; we may need to do multiple
+ implementations on the client side at least.
+
+0.3.1. Goals for package formats and PKIs
+
+ It should be possible to mirror a repository using only rsync and
+ cron.
+
+ Separate keys should be used for different people and different
+ roles.
+
+ Only a minimal set of keys should have to be kept online to keep
+ the system running.
+
+ The system should handle any single computer or system or person
+ being unavailable.
+
+ The formats and protocols should be pretty future-proof.
+
+1. System overview
+
+ The basic unit of updatability is a "bundle". A bundle is a set of
+ software components, or "packages", plus some rules about installing
+ them. Example bundles could be "Tor Browser, stable series" or
+ "Basic Tor, development series".
+
+ When Glider has responsibility for keeping a bundle up to date, we
+ say that a user has "subscribed" to that bundle.
+
+ Conceptually, there are four parts to keeping a bundle up to date:
+
+ Polling:
+ - Periodically, Glider asks a mirror whether there is a newer
+ version of some bundle that a user has subscribed to. If so,
+ Glider determines what's in the bundle.
+
+ Fetching:
+ - If the bundle contains packages that Glider hasn't installed
+ or hasn't cached, it needs to download them from a mirror.
+ This can happen over any protocol; v1 should support at least
+ http and https-over-Tor. V1 should also support resuming
+ partial downloads, since many users have unreliable
+ connections.
+
+ Later versions could support Bittorrent, or whatever.
+
+ Validation:
+ - Throughout the process, Glider must ensure that all the
+ bundles are signed correctly, all the packages are signed
+ correctly, and everything is up-to-date.
+
+ We want to specify this so that users can't be tricked about
+ the contents of a bundle, can't install a malicious package,
+ and can't be fooled into believing that an old bundle is
+ actually the latest.
+
+ Installation:
+ - Now Glider has a set of packages to install. The format of
+ these packages will be platform-dependent: they could be pkg
+ files on OSX, MSI files on Win32, RPMs or DEBs on Linux, and
+ so on. Glider should query the user for permission to start
+ installing packages, then install the packages. All other
+ steps should generally happen automatically, in the
+ background, without needing user intervention. This part
+ needs user intervention because (A) it isn't nice to install
+ updates without permission, and (B) in some configurations,
+ it needs administrator privileges.
+
+ (NO OTHER PART of this design needs administrator privileges.)
+
+1.1. The repository
+
+ Each Glider instance knows about one or more "repositories". A
+ repository is a filesystem somewhere that contains the packages in a
+ set of bundles, and some associated metadata. A repository must
+ exist at one or more canonical hosts, and may have a number of full
+ or partial mirrors.
+
+ In v1, each Glider instance will know about only one repository.
+
+1.2. The PKI
+
+ The trust root for the whole system is, necessarily, whatever users
+ download when they first download a copy of Glider. We need to make
+ sure that the first download happens from a site we trust, using
+ HTTPS.
+
+ Glider ships with root keys, which in turn are used to verify the
+ keys for all the other roles. There are a few root keys, operated by
+ trusted admins for the system. If root keys ever need to be changed,
+ we can just ship an update of Glider: it's supposed to be
+ self-updating anyway.
+
+ The root keys are only used to sign a 'key list' of all the other
+ keys and their roles. A key list is valid if it has been signed by a
+ threshold of root keys.
+
+ Each package is signed with the key of its authorized builder. For
+ example, one volunteer may be authorized to build the mac versions of
+ several packages, and another may be authorized to build the windows
+ version of just one.
+
+ Each bundle is signed with the key of its maintainer. It's assumed
+ that the bundle maintainer might be the package maintainer for some
+ but not all of the packages.
+
+ The list of mirrors is also signed. If the mirror list is
+ automatically updated, this key must be kept online; otherwise, it
+ can be offline.
+
+ To prevent an adversary from replaying an out-of-date signed
+ document, an automated process periodically signs a timestamped
+ statement containing the hashes of the mirror list, the latest
+ bundles, and the key list, using yet another special-purpose key.
+ This key must be kept online.
+
+1.3. Threat Model And Analysis
+
+ We assume an adversary who can operate compromised mirrors, and who
+ can possibly compromise the main repository. At worst, such an
+ adversary can DOS users in a way that they can detect.
+
+ We're assuming for the moment an OSX/Win32-like execution model,
+ where all packages will run equal privilege, but occasionally
+ installation will require higher privilege. This means that once a
+ hostile package is installed, it can basically do whatever it
+ wants. As rootkit writers demonstrate, compromise is really
+ tenuous: any attacker who can induce a user to install a hostile
+ piece of code has, in effect, permanently compromised that user
+ until they reinstall.
+
+ Thus, if an adversary compromises enough keys to sign a compromised
+ package, or tricks a packager into signing a compromised package,
+ and manages to get that package into a signed bundle, the best we
+ can do is to limit the number of users who are affected. We do
+ this by compartmentalizing signing keys so that only the package
+ and bundle in question are at risk.
+
+ (If we had replicated build processes and a bit-by-bit reliable
+ build process, we could have multiple packagers test that a binary
+ was built properly, and multiply sign it. This would be effective
+ against an adversary compromising a single packaging key, but not
+ against one compromising a source repository.)
+
+2. The repository layout
+
+ The filesystem layout in the repository is used for two purposes:
+ - To give mirrors an easy way to mirror only some of the repository.
+ - To specify which parts of the repository a given key has the
+ authority to sign.
+
+ The following files exist in all repositories and mirrors:
+
+ /meta/keys.txt
+
+ Signed by the root keys; indicates keys and roles.
+ [???? I'm using the txt extension here. Is that smart?]
+
+ /meta/mirrors.txt
+
+ Signed by the mirror key; indicates which parts of the
+ repository are mirrored at what mirrors.
+
+ /meta/timestamp.txt
+
+ Signed by the timestamp key; indicates hashes and timestamps
+ for the latest versions of keys.txt and mirrors.txt. Also
+ indicates the latest version of each bundle for each os/arch.
+
+ This is the only file that needs to be downloaded for polling.
+
+ /bundleinfo/bundlename/os-arch/bundlename-os-arch-bundleversion.txt
+
+ Signed by the appropriate bundle key. Describes what
+ packages make up a bundle, and what order to install,
+ uninstall, and upgrade them in.
+
+ /pkginfo/packagename/os-arch/version/packagename-os-arch-packageversion.txt
+
+ Signed by the appropriate package key. Tells the name of the
+ file that makes up a package, its hash, and what procedure
+ is used to install it.
+
+ /packages/packagename/os-arch/version/(some filename)
+
+ The actual package file. Its naming convention will depend
+ on the underlying packaging system.
+
+3. Document formats
+
+3.1. Metaformat
+
+ All documents use Rivest's SEXP meta-format as documented at
+ http://people.csail.mit.edu/rivest/sexp.html
+ with the restriction that no "display hint" fields are to be used,
+ and the base64 transit encoding isn't used either.
+
+ (We use SEXP because it's really easy to parse, really portable,
+ and unlike most other tagged data formats, has a
+ trivially-specified canonical format suitable for hashing.)
+
+ In descriptions of syntax below, we use regex-style qualifiers, so
+ that in
+ (sofa slipcover? occupant* leg+)
+ the sofa will have an optional slipcover, zero or more occupants,
+ and one or more legs. This pattern matches (sofa leg) and (sofa
+ slipcover occupant occupant leg leg leg leg) but not (sofa leg
+ slipcover).
+
+ We also use a braces notation to indicate elements that can occur
+ in any order. For example,
+ (bread {flour+ eggs? yeast})
+ matches a list starting with "bread", and then containing one or
+ more of flours, zero or one occurrences of eggs, and one
+ occurrence of yeast, in any order. This pattern matches (bread eggs
+ yeast flour) but not (bread yeast) or (bread flour eggs yeast
+ macadamias).
+
+3.2. File formats: general principles
+
+ We use tagged lists (lists whose first element is a string) to
+ indicate typed objects. Tags are generally lower-case, with
+ hyphens used for separation. Think Lispy.
+
+ We use attrlists [lists of (key value) lists] to indicate a
+ multimap from keys to values. Clients MUST accept unrecognized
+ keys in these attrlists. The syntax for an attrlist with two
+ recognized and required keys is typically given as ({(key1 val1)
+ (key2 val2) (ATTR VAL)*}), indicating that the keys can occur in
+ any order, intermixed with other attributes.
+
+ Timestamp files will be downloaded very frequently; all other files
+ will be much smaller in size than package files. Thus,
+ size-optimization for timestamp files makes sense and most other
+ other space optimizations don't.
+
+ Versions are represented as lists of the form (v I1 I2 I3 I4 ...)
+ where each item is a number or alphanumeric version component. For
+ example, the version "0.2.1.5-alpha" is represented as (v 0 2 1 5
+ alpha).
+
+ All signed files are of the format:
+
+ (signed
+ X
+ (signature ({(keyid K) (method M) (ATTR VAL)*}) SIG)+
+ )
+
+ { "_type" : "Signed",
+ "signed" : X,
+ "sigatures" : [
+ { "keyid" : K,
+ "method" : M,
+ ...
+ "sig" : S } ]
+
+ where: X is a list whose first element describes the signed object.
+ K is the identifier of a key signing the document
+ M is the method to be used to make the signature
+ (ATTR VAL) is an arbitrary list whose first element is a
+ string.
+ SIG is a signature of the canonical encoding of X using the
+ identified key.
+
+ We define two signing methods at present:
+ sha256-oaep : A signature of the SHA256 hash of the canonical
+ encoding of X, using OAEP+ padding. [XXXX say more about mgf]
+
+ All times are given as strings of the format "YYYY-MM-DD HH:MM:SS",
+ in UTC.
+
+ All keys are of the format:
+ (pubkey ({(type TYPE) (ATTR VAL)*}) KEYVAL)
+
+ { "_keytype" : TYPE,
+ ...
+ "keyval" : KEYVAL }
+
+ where TYPE is a string describing the type of the key and how it's
+ used to sign documents. The type determines the interpretation of
+ KEYVAL.
+
+ The ID of a key is a two-element list of the type and the SHA-256
+ hash of the canonical encoding of the KEYVAL field.
+
+ We define one keytype at present: 'rsa'. The KEYVAL in this case
+ is a 2-element list of (e n), with both values given in big-endian
+ binary format. [This makes keys 45-60% more compact than using
+ decimal integers.]
+
+ {Values given as integers.}
+
+ {'e' : e, 'n' : n, big-endian hex. }
+
+ All RSA keys must be at least 2048 bits long.
+
+
+ Every role in the system is associated with a key. Replacing
+ anything but a root key is supposed to be relatively easy.
+
+ Root-keys sign other keys, and certify them as belonging to roles.
+ Clients are configured to know the root keys.
+
+ Bundle keys certify the contents of a bundle.
+
+ Package keys certify packages for a given program or set of
+ programs.
+
+ Mirror keys certify a list of mirrors. We expect this to be an
+ automated process.
+
+ Timestamp keys certify that given versions of other metadata
+ documents are up-to-date. They are the only keys that absolutely
+ need to be kept online. (If they are not, timestamps won't be
+ generated.)
+
+3.3. File formats: key list
+
+ The key list file is signed by multiple root keys. It indicates
+ which keys are authorized to sign which parts of the repository.
+
+ (keylist
+ (ts TIME)
+ (keys
+ ((key ({(roles (ROLE PATH)+) (ATTR VAL)*}) KEY)*)
+ ...
+ )
+
+ { "_type" : "Keylist",
+ "ts" : TIME,
+ "keys" : [
+ { "roles" : [ [ ROLE, PATH ], ... ],
+ ...
+ "key" : KEY }, ... ] }
+
+ The "ts" line describes when the keys file was updated. Clients
+ MUST NOT replace a file with an older one, and SHOULD NOT accept a
+ file too far in the future.
+
+ A ROLE is one of "timestamp" "mirrors" "bundle" or "package".
+
+ PATH is a path relative to the top of the directory hierarchy. It
+ may contain "*" elements to indicate "any file", and may end with a
+ "/**" element to indicate all files under a given point.
+
+3.4. File formats: mirror list
+
+ The mirror list is signed by a mirror key. It indicates which
+ mirrors are active and believed to be mirroring which parts of the
+ repository.
+
+ (mirrorlist
+ (ts TIME)
+ (mirrors
+ ( (mirror ({(name N) (urlbase U) (contents PATH+) (weight W)
+ (official)? (ATTR VAL)})) * )
+ ...
+ )
+
+ { "_type" : "Mirrorlist",
+ "mirrors" : [
+ { "name" : N,
+ "urlbase" : U,
+ "contents" : [PATH ... ] ,
+ "weight" : W,
+ "official" : BOOL,
+ ...
+ }, ... ]
+ }
+
+ Every mirror is a copy of some or all of the directory hierarchy
+ containing at least the /meta, /bundles/, and /pkginfo directories.
+
+ N is a descriptive name for the mirror; U is the URL of the mirror's
+ base (i.e., the parent of the "meta" directory); and the PATH
+ elements are the components describing how much of the packages
+ directory is mirrored. Their format is as in the keylist file.
+
+ W is an integer used to weight mirrors when picking at random;
+ mirrors with more bandwidth should have higher weigths. The
+ "official" element should only be present if the mirror is (one of
+ the) official repositories operated by the Tor Project.
+
+3.5. File formats: timestamp files
+
+ The timestamp file is signed by a timestamp key. It indicates the
+ latest versions of other files, and contains a regularly updated
+ timestamp to prevent rollback attacks.
+
+ (ts
+ ({(at TIME)
+ (m TIME MIRRORLISTHASH)
+ (k TIME KEYLISTHASH)
+ (b NAME VERSION PATH TIME HASH)*})
+ )
+
+ { "_type" : Timestamp,
+ "at" : TIME,
+ "m" : [ TIME, HASH ],
+ "k" : [ TIME, HASH ],
+ "b" : { NAME :
+ [ [ Version, Path, Time, Hash ] ] }
+ }
+
+ TIME is when the timestamp was signed. MIRRORLISTHASH is the digest
+ of the mirror-list file; KEYLISTHASH is the digest of the key list
+ file; and the 'b' entries are a list of the latest version of all
+ bundles and their locations and hashes.
+
+3.6. File formats: bundle files
+
+ (bundle
+ (at TIME)
+ (os OS)
+ [(arch ARCH)]
+ (version V)
+ (packages
+ (NAME VERSION PATH HASH ({(order INST UPDATE REMOVE)
+ (optional)?
+ (gloss LANG TEXT)*
+ (longloss LANG TEXT)*
+ (ATTR VAL)*})? )* )
+ )
+
+ { "_type" : "Bundle",
+ "name" : NAME,
+ "at" : TIME,
+ "os" : OS,
+ [ "arch" : ARCH, ]
+ "version" : V
+ "packages" :
+ [ { "name" : NAME,
+ "version" : VERSION,
+ "path" : PATH,
+ "hash" : HASH,
+ "order" : [ INST, UPDATE, REMOVE ],
+ [ "optional : BOOL, ]
+ "gloss" : { LANG : TEXT },
+ "longgloss" : { LANG : TEXT },
+ } ] }
+
+ Most elements are self-explanatory; the INST, UPDATE, and REMOVE
+ elements of the order element are numbers defining the order in
+ which the packages are installed, updated, and removed respectively.
+ The "optional" element is present if the package is optional.
+ "Gloss" is a short utf-8 human-readable string explaining what the
+ package provides for the bundle; "longloss" is a longer such
+ utf-8 string.
+
+ (Note that the gloss strings are meant, not to describe the package,
+ but to describe what the package provides for the bundle. For
+ example, "The Anonymous Email Bundle needs the Python Runtime to run
+ Mixminion.")
+
+ Multiple gloss strings are allowed; each should have a different
+ language. The UI should display the must appropriate language to the
+ user.
+
+3.7. File formats: package files
+
+ (package
+ ({(name NAME)
+ (version VERSION)
+ (format FMT ((ATTR VAL)*)? )
+ (path PATH)
+ (ts TIME)
+ (digest HASH)
+ (shortdesc LANG TEXT)*
+ (longdesc LANG TEXT)*
+ (ATTR VAL)* })
+ )
+
+ Most elements are self-explanatory. The "FMT" element describes the
+ file format of the package, which should give enough information
+ about how to install it.
+
+ No two package files in the same repository should have the same
+ name and version. If a package needs to be changed, the version
+ MUST be incremented.
+
+ Descriptions are tagged with languages in the same way as glosses.
+
+4. Detailed Workflows
+
+4.1. The client application
+
+ Periodically, the client updater fetches a timestamp file from a
+ mirror. If the timestamp in the file is up-to-date, the client
+ first checks to see whether the keys file listed is one that the
+ client has. If not, the client fetches it, makes sure the hash of
+ the keys file matches the hash in the timestamp file, makes sure its
+ date is more recent than any keys file they have but not too far in
+ the future, and that it is signed by enough root keys that the
+ client recognizes.
+
+ [If the timestamp file is not up-to-date, the client tries a
+ few mirrors until it finds one with a good timestamp.]
+
+ [If the keys file from a mirror does not match the timestamp
+ file, the client tries a new mirror for both.]
+
+ [If the keys file is not signed by enough root keys, the client
+ warns the user and tries another mirror for both the timestamp
+ file and the keys file.]
+
+ Once the client has an up-to-date keys file, the client checks the
+ signature on the timestamp file. Assuming it checks out, the client
+ refreshes the mirror list as needed, and refreshes any bundle files
+ to which the user is subscribed if the client does not have
+ the latest version of those files. The client checks signatures on
+ these files, and fetches package metadata for any packages listed in
+ the bundle file that the client does not have, checks signatures on
+ these, and fetches binaries for packages that might need to be
+ installed or updated. As the packages arrive, clients check their
+ hashes.
+
+ Once the client has gotten enough packages, it informs the user that
+ new packages have arrived, and asks them if they want to update.
+
+ Clients SHOULD cache at least the latest versions they have received
+ of all files.
+
+4.1.1. Download preferences
+
+ Users should be able to specify that packages must be only
+ downloaded over Tor, or must only be downloaded over encrypted
+ protocols, or both. Users should also be able to express preference
+ for Tor vs non-Tor and encrypted vs non-encrypted, even if they
+ allow both.
+
+4.2. Mirrors
+
+ Periodically, mirrors do an rsync or equivalent to fetch the latest
+ version of whatever parts of the repository have changed since the
+ version they currently hold. Mirrors SHOULD replace older versions
+ of the repository idempotently, so that clients are less likely to
+ see inconsistent state. Mirrors SHOULD validate the information
+ they receive, and not serve partial or inconsistent files.
+
+4.3. Workflow: Packagers
+
+ When a new binary package is done, the person making the package
+ runs a tool to generate and sign a package file, and sends both the
+ package and the package file to a repository admin. Typically, the
+ base package file will be generated by inserting a version into a
+ template.
+
+ Packages MAY have as part of their build process a script to
+ generate the appropriately versioned package file. This script
+ should at a minimum demand a build version, or use a timestamp in
+ place of a build version, to prevent two packages with the same
+ version from being created.
+
+4.4. Workflow: bundlers
+
+ When the packages in a bundle are done, the bundler runs a tool on
+ the package files to generate and sign a bundle file. Typically,
+ this tool uses a template bundle file.
+
+4.5. Workflow: repository administrators
+
+ Repository administrators use a tool to validate signed files into the
+ repository. The repository should not be altered manually.
+
+ This tool acts as follows:
+ - Package files may be added, but never replaced.
+ - Bundle files may be added, but never replaced.
+ - No file may be added unless it is syntactically valid and
+ signed by a key in the keys file authorized to sign files of
+ this type in this file's location location.
+
+ - A package file may not be added unless all of its binary
+ packages match their hashes.
+
+ - A bundle file may not be added unless all of its package files
+ are present and match their hashes.
+
+ - When adding a new keylist, bundle, or mirrors list, the
+ timestamp file must be regenerated immediately.
+
+5. Parameter setting and corner cases.
+
+5.1. Timing
+
+ The timestamp file SHOULD be regenerated every 15 minutes. Mirrors
+ SHOULD attempt to update every hour. Clients SHOULD accept a
+ timestamp file up to 6 hours old.
+
+5.2. Format versioning and forward-compatibility:
+
+ All of the above formats include the ability to add more
+ attribute-value fields for backwards-compatible format changes. If
+ we need to make a backwards incompatible format change, we create a
+ new filename for the new format.
+
+5.3. Key management and migration:
+
+ Root keys should be kept offline. All keys except timestamp and
+ mirror keys should be stored encrypted.
+
+ All the formats above allow for multiple keys to sign a single
+ document. To replace a compromised root key, it suffices to sign
+ keylist documents with both the compromised key and its replacement
+ until all clients have updated to a new version of the autoupdater.
+
+ To replace another key, it suffices to authorize the new key in the
+ keylist. Note that a new package or bundle key must re-sign and
+ issue new versions of all packages or bundles it has generated.
+
+
+
+F. Future directions and open questions
+
+F.1. Package decomposition
+
+ It would be neat to decouple existing packages. Right now, we'd
+ never want a windows user to have to fetch an openssl dll and Tor
+ separately. But if they're using an auto-update tool, it'd be
+ pretty keen to have them not need to fetch a new openssl every time
+ Tor has a bugfix.
+
+F.2. Caching at Tor servers.
+
+ See Tor Proposal number 127.
+
+F.3. Support for more download methods
+
+ Ozymandns, chunked downloads, and bittorrent would all be neat
+ ideas.
+
+F.4. Support for bogus clocks.
+
+ Glider should have a user configurable "no, my clock is _supposed_
+ to be wrong" mode, since lots of users seem to _like_ having their
+ clocks in 1970 forever.
+
+R. Ideas I'm rejecting for the moment
+
+R.1. Considering recommended versions from Tor consensus directory documents
+
+ This requires a working Tor to update Tor; that's not necessarily a
+ great idea.
+
+R.2. Integration with existing GPG signatures
+
+ The OpenPGP signature and key format is so complicated that you'd
+ have to be mad to touch it.
+
+
More information about the tor-commits
mailing list