[or-cvs] r17294: {updater} attach per-file type-specific metainfo to packages. use meta (in updater/trunk/lib/thandy: . packagesys)
nickm at seul.org
nickm at seul.org
Sun Nov 16 22:27:17 UTC 2008
Author: nickm
Date: 2008-11-16 17:27:17 -0500 (Sun, 16 Nov 2008)
New Revision: 17294
Modified:
updater/trunk/lib/thandy/ClientCLI.py
updater/trunk/lib/thandy/formats.py
updater/trunk/lib/thandy/packagesys/ExePackages.py
updater/trunk/lib/thandy/packagesys/PackageDB.py
updater/trunk/lib/thandy/packagesys/PackageSystem.py
updater/trunk/lib/thandy/packagesys/RPMPackages.py
updater/trunk/lib/thandy/repository.py
Log:
attach per-file type-specific metainfo to packages. use metainfo to tell what is already installed. use database to track things we have installed with not attacked version. notice when we have installable files and optionally install them.
Modified: updater/trunk/lib/thandy/ClientCLI.py
===================================================================
--- updater/trunk/lib/thandy/ClientCLI.py 2008-11-16 21:10:45 UTC (rev 17293)
+++ updater/trunk/lib/thandy/ClientCLI.py 2008-11-16 22:27:17 UTC (rev 17294)
@@ -4,32 +4,78 @@
import logging
import os
import sys
+import time
+import thandy.formats
import thandy.util
import thandy.repository
import thandy.download
import thandy.master_keys
+import thandy.packagesys.PackageSystem
def update(args):
repoRoot = thandy.util.userFilename("cache")
- options, args = getopt.getopt(args, "", [ "repo=", "no-download" ])
+ options, args = getopt.getopt(args, "", [ "repo=", "no-download",
+ "loop", "no-packagesys",
+ "install"])
download = True
+ keep_looping = False
+ use_packagesys = True
+ install = False
for o, v in options:
if o == '--repo':
repoRoot = v
elif o == "--no-download":
download = False
+ elif o == '--loop':
+ keep_looping = True
+ elif o == '--no-packagesys':
+ use_packagesys = False
+ elif o == '--install':
+ install = True
repo = thandy.repository.LocalRepository(repoRoot)
+ packagesys = None
+ if use_packagesys:
+ packagesys = thandy.packagesys.PackageSystem.PackageMetasystem.create(repo)
while True:
hashes = {}
+ installable = {}
logging.info("Checking for files to update.")
- files = repo.getFilesToUpdate(trackingBundles=args, hashDict=hashes)
+ files = repo.getFilesToUpdate(trackingBundles=args, hashDict=hashes,
+ pkgSystems=packagesys,
+ installableDict=installable)
+
+ if installable and not files:
+ logging.notice("Ready to install files: %s",
+ ", ".join(sorted(installable.keys())))
+ if install:
+ # XXXX handle ordering
+ for h in installable.values():
+ h.install()
+ return
+
+ elif not files:
+ logging.info("No files to download")
+ if not keep_looping:
+ return
+
+ ts = repo.getTimestampFile().get()
+ age = time.time() - thandy.formats.parseTime(ts['at'])
+ delay = thandy.repository.MAX_TIMESTAMP_AGE - age
+ if delay > 3600:
+ delay = 3600
+ elif delay < 0:
+ delay = 300
+ logging.info("Will check again in %s seconds", delay)
+ time.sleep(delay)
+ continue
+
logging.info("Files to download are: %s", ", ".join(sorted(files)))
- if not download or not files:
+ if not download:
return
mirrorlist = repo.getMirrorlistFile().get()
@@ -59,18 +105,13 @@
logging.info("All downloads finished.")
-# Check my repository
-
-# Tell me what I need to download
-
-# Download stuff
-
# Tell me what to install.
def usage():
print "Known commands:"
- print " update [--repo=repository] [--no-download]"
+ print " update [--repo=repository] [--no-download] [--loop]"
+ print " [--no-packagesys] [--install]"
sys.exit(1)
def main():
@@ -81,7 +122,7 @@
usage()
cmd = sys.argv[1]
args = sys.argv[2:]
- if cmd in [ "update", "geturls" ]:
+ if cmd in [ "update" ]:
globals()[cmd](args)
else:
usage()
Modified: updater/trunk/lib/thandy/formats.py
===================================================================
--- updater/trunk/lib/thandy/formats.py 2008-11-16 21:10:45 UTC (rev 17293)
+++ updater/trunk/lib/thandy/formats.py 2008-11-16 22:27:17 UTC (rev 17294)
@@ -451,7 +451,8 @@
version=VERSION_SCHEMA,
format=S.Obj(),
ts=TIME_SCHEMA,
- files=S.ListOf(S.Struct([RELPATH_SCHEMA, HASH_SCHEMA])),
+ files=S.ListOf(S.Struct([RELPATH_SCHEMA, HASH_SCHEMA],
+ allowMore=True)),
shortdesc=S.DictOf(S.AnyStr(), S.AnyStr()),
longdesc=S.DictOf(S.AnyStr(), S.AnyStr()))
@@ -612,23 +613,29 @@
'format',
'location',
'relpath',
- ], (), preload)
+ ], ['rpm_version', 'exe_args'], preload)
f = open(package_fname, 'rb')
digest = getFileDigest(f)
# Check fields!
+ extra = {}
result = { '_type' : "Package",
'ts' : formatTime(time.time()),
'name' : r['name'],
'location' : r['location'], #DOCDOC
'version' : r['version'],
'format' : r['format'],
- 'files' : [ [ r['relpath'], formatHash(digest) ] ],
+ 'files' : [ [ r['relpath'], formatHash(digest), extra ] ],
'shortdesc' : shortDescs,
'longdesc' : longDescs
}
+ if format == 'rpm' and r.get('rpm_version'):
+ extra['rpm_version'] = r['rpm_version']
+ elif format == 'exe' and r.get('exe_args') != None:
+ extra['exe_args'] = r['exe_args']
+
PACKAGE_SCHEMA.checkMatch(result)
return result
Modified: updater/trunk/lib/thandy/packagesys/ExePackages.py
===================================================================
--- updater/trunk/lib/thandy/packagesys/ExePackages.py 2008-11-16 21:10:45 UTC (rev 17293)
+++ updater/trunk/lib/thandy/packagesys/ExePackages.py 2008-11-16 22:27:17 UTC (rev 17294)
@@ -1,16 +1,39 @@
# Copyright 2008 The Tor Project, Inc. See LICENSE for licensing information.
+import subprocess
+
+import thandy.util
import thandy.packagesys.PackageSystem as ps
import thandy.packagesys.PackageDB as pdb
class ExePackageSystem(pdb.DBBackedPackageSystem):
+ def __init__(self, repo):
+ self._repo = repo
def getName(self):
- return "executable"
+ return "exe"
- def packageHandleFromJSON(self, json):
- raise NotImplemented() #XXXX????
+ def packageHandlesFromJSON(self, pkg):
+ if pkg['format'] != 'exe':
+ raise thandy.FormatException()
+ handles = []
+ for entry in pkg['files']:
+ if len(entry) < 3:
+ continue
+ rp, h, extra = entry[:3]
+ version = package['version']
+
+ handles.append(
+ ExePackageHandle(self.getDB(),
+ package['name'],
+ version,
+ [], # filelist not implemented in this.
+ rp,
+ self._repo.getFilename(rp),
+ extra['exe_args']))
+ return handles
+
def canBeAutomatic(self):
return True
@@ -18,12 +41,16 @@
return True
class ExePackageHandle(pdb.DBBackedPackageHandle):
- def __init__(self, packageDB, name, version, filelist, filename,
+ def __init__(self, packageDB, name, version, filelist, relpath, filename,
arguments):
pdb.DBBackedPackageHandle.__init__(packageDB, name, version, filelist)
+ self._relPath = relpath
self._filename = filename
self._arguments = arguments
+ def getRelativePath(self):
+ return self._relPath
+
def _doInstall(self):
commandline = [ self._filename ] + self._arguments
logging.info("Installing %s. Command line: %s", self._filename,
Modified: updater/trunk/lib/thandy/packagesys/PackageDB.py
===================================================================
--- updater/trunk/lib/thandy/packagesys/PackageDB.py 2008-11-16 21:10:45 UTC (rev 17293)
+++ updater/trunk/lib/thandy/packagesys/PackageDB.py 2008-11-16 22:27:17 UTC (rev 17294)
@@ -1,34 +1,48 @@
# Copyright 2008 The Tor Project, Inc. See LICENSE for licensing information.
import anydbm
+import atexit
import shelve
import thandy.util
import thandy.formats
+import thandy.packagesys.PackageSystem
+
class SimplePackageDB:
-
def __init__(self, filename):
+ thandy.util.ensureParent(filename)
self._db = anydbm.open(filename, 'c')
+ atexit.register(self.close)
+ def close(self):
+ self._db.close()
+
def setVersion(self, package, version, filelist):
- pass
+ self._db['pv_%s'%package] = (version, filelist)
def setInstallParameters(self, package, params):
- pass
+ self._db['ip_%s'%package] = params
def getCurVersion(self, package):
- pass
+ v = self._db.get('pv_%s'%package)
+ if v != None:
+ return v[0]
def getInstallParameters(self, package):
- pass
+ return self._db.get('pi_%s'%package)
+class DBBackedPackageSystem(thandy.packagesys.PackageSystem.PackageSystem):
+ def __init__(self):
+ self._packageDB = None
-class DBBackedPackageSystem(thandy.packagesys.PackageSystem):
- def __init__(self, packageDB):
- self._packageDB = packageDB
+ def getDB(self):
+ if self._packageDB is None:
+ fname = thandy.util.userFilename("db/packages")
+ self._packageDB = pdb.PackageDB(fname)
+ return self._packageDB
-class DBBackedPackageHandle(thandy.packagesys.PackageHandle):
+class DBBackedPackageHandle(thandy.packagesys.PackageSystem.PackageHandle):
def __init__(self, packageDB, name, version, filelist):
thandy.packagesys.PackageSystem.PackageHandle.__init__(self)
self._packageDB = packageDB
@@ -68,14 +82,13 @@
if not os.path.exists(fn):
all_ok = False
else:
- f = open(fn, 'rb')
try:
- try:
- d = thandy.formats.getFileDigest(f)
- except OSError:
+ d = thandy.formats.getFileDigest(fn)
+ if d != hash:
all_ok = False
- break
- finally:
- f.close()
+ except OSError:
+ all_ok = False
+ break
+
return all_ok
Modified: updater/trunk/lib/thandy/packagesys/PackageSystem.py
===================================================================
--- updater/trunk/lib/thandy/packagesys/PackageSystem.py 2008-11-16 21:10:45 UTC (rev 17293)
+++ updater/trunk/lib/thandy/packagesys/PackageSystem.py 2008-11-16 22:27:17 UTC (rev 17294)
@@ -1,10 +1,40 @@
# Copyright 2008 The Tor Project, Inc. See LICENSE for licensing information.
+class PackageMetasystem:
+ def __init__(self, repository):
+ self._repostitory = repository
+ self._systems = {}
+
+ def addPackageSystem(self, system):
+ self._systems[system.getName()] = system
+
+ def getSysForPackage(self, pkg):
+ return self._systems.get(pkg['format'], None)
+
+ @staticmethod
+ def create(repository):
+ r = PackageMetasystem(repository)
+
+ try:
+ import rpm
+ except ImportError:
+ pass
+ else:
+ import thandy.packagesys.RPMPackages
+ r.addPackageSystem(thandy.packagesys.RPMPackages.RPMPackageSystem(
+ repository))
+
+ import thandy.packagesys.ExePackages
+ r.addPackageSystem(thandy.packagesys.ExePackages.ExePackageSystem(
+ repository))
+
+ return r
+
class PackageSystem:
def getName(self):
raise NotImplemented()
- def packageHandleFromJSON(self, json):
+ def packageHandlesFromJSON(self, json):
raise NotImplemented()
def canBeAutomatic(self):
@@ -39,6 +69,9 @@
self._transactions.append(packageHandle.remove)
class PackageHandle:
+ def getRelativePath(self):
+ raise NotImplemented()
+
def isInstalled(self, transaction=None):
raise NotImplemented()
Modified: updater/trunk/lib/thandy/packagesys/RPMPackages.py
===================================================================
--- updater/trunk/lib/thandy/packagesys/RPMPackages.py 2008-11-16 21:10:45 UTC (rev 17293)
+++ updater/trunk/lib/thandy/packagesys/RPMPackages.py 2008-11-16 22:27:17 UTC (rev 17294)
@@ -6,15 +6,40 @@
import rpm
import md5
+import thandy.formats
+
__all__ = [ 'RPMPackageSystem' ]
class RPMPackageSystem(thandy.packagesys.PackageSystem.PackageSystem):
+ def __init__(self, repo):
+ self._repo = repo
+
def getName(self):
- return "RPM"
+ return "rpm"
- def packageHandleFromJSON(self, json):
- raise NotImplemented() # XXXX
+ def packageHandlesFromJSON(self, package):
+ if package['format'] != 'rpm':
+ raise thandy.FormatException()
+ handles = []
+ for entry in package['files']:
+ if len(entry) < 3:
+ continue
+ fn, h, extra = entry[:3]
+ name = os.path.split(fn)[1]
+
+ try:
+ version = extra['rpm_version']
+ except KeyError:
+ raise thandy.FormatException()
+
+ handles.append(RPMPackageHandle(name,
+ version,
+ fn,
+ self._repo.getFilename(fn)))
+
+ return handles
+
def getTransaction(self):
return RPMPackageTransaction()
@@ -115,7 +140,7 @@
logging.info("%s is missing or unreadable from %s %s; "
"that's ok.", fname, name, h['version'])
else:
- logging.warn("%s is missing or unreadable from %s %s."
+ logging.warn("%s is missing or unreadable from %s %s.",
fname, name, h['version'])
all_ok = False
elif haveMD5 == md5sum:
@@ -131,11 +156,15 @@
return found and all_ok
class RPMPackageHandle(thandy.packagesys.PackageSystem.PackageHandle):
- def __init__(self, name, version, filename):
+ def __init__(self, name, version, relativePath, filename):
self._name = name
self._version = version
+ self._relPath = relativePath
self._filename = filename
+ def getRelativePath(self):
+ return self._relPath
+
def anyVersionInstalled(self, transaction=None):
return len(getInstalledRPMVersions(self.name, transaction)) > 1
Modified: updater/trunk/lib/thandy/repository.py
===================================================================
--- updater/trunk/lib/thandy/repository.py 2008-11-16 21:10:45 UTC (rev 17293)
+++ updater/trunk/lib/thandy/repository.py 2008-11-16 22:27:17 UTC (rev 17294)
@@ -260,7 +260,7 @@
needRole='bundle')
return pkg
- def getRequestedFile(self, relPath):
+ def getRequestedFile(self, relPath, pkgSystems=None):
"""DOCDOC"""
for f in self._metafiles:
if f.getRelativePath() == relPath:
@@ -279,12 +279,14 @@
return None
- def getFilesToUpdate(self, now=None, trackingBundles=(), hashDict=None):
+ def getFilesToUpdate(self, now=None, trackingBundles=(), hashDict=None,
+ pkgSystems=None, installableDict=None):
"""Return a set of relative paths for all files that we need
to fetch. Assumes that we care about the bundles
'trackingBundles'. If hashDict is provided, add mappings to it
from the relative paths we want to fecth to the hashes that we
want those items to have, when we know those hashes.
+ DOCDOC pkgSystems, installableDict
"""
if now == None:
@@ -294,6 +296,9 @@
# Use a dummy hashdict.
hashDict = {}
+ if installableDict == None:
+ installableDict = {}
+
need = set()
# Fetch missing metafiles.
@@ -444,8 +449,29 @@
# files?
for pfile in packages.values():
package = pfile.get()
+
+ alreadyInstalled = {}
+ allHandles = {}
+ if pkgSystems is not None:
+ psys = pkgSystems.getSysForPackage(package)
+ if psys is None:
+ logging.info("No way to check whether a %s package is "
+ "up-to-date." % package['format'])
+ else:
+ handles = psys.packageHandlesFromJSon(package)
+
+ for h in handles:
+ allHandles[h.getRelativePath()] = h
+ if h.isInstalled():
+ alreadyInstalled[h.getRelativePath()] = h
+
for f in package['files']:
rp, h = f[:2]
+ if alreadyInstalled.has_key(rp):
+ logging.info("%s is already installed; no need to download",
+ rp)
+ continue
+
h_expected = thandy.formats.parseHash(h)
hashDict[rp] = h_expected
fn = self.getFilename(rp)
@@ -459,6 +485,9 @@
if h_got != h_expected:
logging.info("Hash for %s not as expected; must load.", rp)
need.add(rp)
+ else:
+ if allHandles.has_key(rp):
+ installableDict[rp] = allHandles[rp]
# Okay; these are the files we need.
return need
More information about the tor-commits
mailing list