commit 7d87a55747e82c7b23daa48581135c35b553aeaf Author: Tomás Touceda chiiph@torproject.org Date: Sat Sep 3 21:58:47 2011 -0300

Improve re-install behavior

Also add more controller oriented output --- lib/thandy/ClientCLI.py | 41 +++++++++++++++++++--------------- lib/thandy/download.py | 3 ++ lib/thandy/packagesys/ThpPackages.py | 37 ++++++++++++++++++++++-------- lib/thandy/repository.py | 15 +++++++++--- 4 files changed, 64 insertions(+), 32 deletions(-)

diff --git a/lib/thandy/ClientCLI.py b/lib/thandy/ClientCLI.py index 2987918..f568a2c 100755 --- a/lib/thandy/ClientCLI.py +++ b/lib/thandy/ClientCLI.py @@ -122,6 +122,7 @@ def update(args): installable = {} btMetadata = {} thpTransactions = {} + alreadyInstalled = set() logging.info("Checking for files to update.") files, downloadingFiles = repo.getFilesToUpdate( trackingBundles=args, @@ -130,34 +131,38 @@ def update(args): usePackageSystem=use_packagesys, installableDict=installable, btMetadataDict=btMetadata, - thpTransactionDict=thpTransactions) + thpTransactionDict=thpTransactions, + alreadyInstalledSet=alreadyInstalled)

if forceCheck: files.add("/meta/timestamp.txt") forceCheck = False

if (thpTransactions or installable) and not files: - for p, d in installable.items(): - for n, i in d.items(): - if i.canInstall(): - logCtrl("CAN_INSTALL", PKG=p, ITEM=n) - else: - logCtrl("NO_INSTALL", PKG=p, ITEM=n) - i.setCacheRoot(repoRoot) - - logging.info("Ready to install packages for files: %s", - ", ".join(sorted(installable.keys()))) - if install: - # XXXX handle ordering - for p in installable.values(): - for h in p.values(): - i = h.getInstaller() - if i != None: - i.install() + # for p, d in installable.items(): + # for n, i in d.items(): + # if i.canInstall(): + # logCtrl("CAN_INSTALL", PKG=p, ITEM=n) + # else: + # logCtrl("NO_INSTALL", PKG=p, ITEM=n) + # i.setCacheRoot(repoRoot) + + # logging.info("Ready to install packages for files: %s", + # ", ".join(sorted(installable.keys()))) + # if install: + # # XXXX handle ordering + # for p in installable.values(): + # for h in p.values(): + # i = h.getInstaller() + # if i != None: + # i.install() + + logCtrl("READY", BUNDLE=",".join(thpTransactions.keys()))

for bundle in thpTransactions: if install: thandy.packagesys.ThpPackages.ThpTransaction(thpTransactions[bundle], + alreadyInstalled, repoRoot).install()

return diff --git a/lib/thandy/download.py b/lib/thandy/download.py index 52c6fe4..c2a9e61 100755 --- a/lib/thandy/download.py +++ b/lib/thandy/download.py @@ -13,6 +13,8 @@ import thandy.util import thandy.socksurls import thandy.checkJson

+from thandy.util import logCtrl + class BadCompoundData(thandy.DownloadError): """DOCDOC""" pass @@ -514,6 +516,7 @@ class DownloadJob: total += len(c) logging.debug("Got %s/%s bytes from %s", total, expectLength, url) + logCtrl("DOWNLOAD", TOTAL=str(total), EXPECT=str(expectLength), URL=url) if self._wantLength != None and total > self._wantLength: logging.warn("Read too many bytes from %s; got %s, but " "wanted %s", url, total, self._wantLength) diff --git a/lib/thandy/packagesys/ThpPackages.py b/lib/thandy/packagesys/ThpPackages.py index 493a94a..dd1d399 100755 --- a/lib/thandy/packagesys/ThpPackages.py +++ b/lib/thandy/packagesys/ThpPackages.py @@ -70,7 +70,17 @@ class ThpDB(object): contents = open(fname, "r").read() metadata = json.loads(contents) version = metadata['package_version'] - return fexists, version + + fname = os.path.join(self._thp_db_root, "pkg-status", name+".status") + fexists2 = os.path.exists(fname) + + status = "" + if fexists2: + contents = open(fname, "r").read() + metadata = json.loads(contents) + status = metadata['status'] + + return fexists, version, status

def statusInProgress(self, pkg): thandy.util.replaceFile(os.path.join(self._thp_db_root, "pkg-status", @@ -94,30 +104,35 @@ class ThpChecker(PS.Checker):

def getInstalledVersions(self): versions = [] - (exists, version) = self._db.exists(self._name) + (exists, version, status) = self._db.exists(self._name)

if exists: versions.append(version)

- return versions + return versions, status

def isInstalled(self): - return self._version in self.getInstalledVersions() + versions, status = self.getInstalledVersions() + # if status = IN_PROGRESS a previous installation failed + # we need to reinstall + return (status != "IN_PROGRESS" and self._version in versions)

class ThpTransaction(object): - def __init__(self, packages, repoRoot): + def __init__(self, packages, alreadyInstalled, repoRoot): self._raw_packages = packages self._repo_root = repoRoot self._installers = [] + self._alreadyInstalled = alreadyInstalled self._db = ThpDB()

self._process()

def _process(self): for package in self._raw_packages.keys(): - self._installers.append(ThpInstaller(self._raw_packages[package]['files'][0][0], - self._db, - self._repo_root)) + if not (self._raw_packages[package]['files'][0][0] in self._alreadyInstalled): + self._installers.append(ThpInstaller(self._raw_packages[package]['files'][0][0], + self._db, + self._repo_root))

def _orderByDep(self): """ Orders packages with a topological order by its dependencies """ @@ -148,6 +163,7 @@ class ThpTransaction(object): except LockFailed: print "Can't acquire lock on %s" % lockfile finally: + logging.info("Releasing lock...") lock.release()

def remote(self): @@ -174,10 +190,11 @@ class ThpInstaller(PS.Installer): destPath = os.path.join(self._thp_root, self._pkg.get("package_name")) logging.info("Destination directory: %s" % destPath)

- if self._db.exists(self._pkg.get("package_name")): + (exists, _, _) = self._db.exists(self._pkg.get("package_name")) + if exists: logging.info("%s is already installed, switching to upgrade mode." % self._pkg.get("package_name")) self._db.startUpgrade() - + pkg_metadata = self._pkg.getAll() self._db.insert(pkg_metadata) self._db.statusInProgress(pkg_metadata) diff --git a/lib/thandy/repository.py b/lib/thandy/repository.py index 20a1e9a..0b45c8d 100755 --- a/lib/thandy/repository.py +++ b/lib/thandy/repository.py @@ -5,6 +5,8 @@ import thandy.util import thandy.packagesys.PackageSystem import thandy.bt_compat

+from thandy.util import logCtrl + json = thandy.util.importJSON()

import logging @@ -300,7 +302,7 @@ class LocalRepository: def getFilesToUpdate(self, now=None, trackingBundles=(), hashDict=None, lengthDict=None, usePackageSystem=True, installableDict=None, btMetadataDict=None, - thpTransactionDict=None): + thpTransactionDict=None, alreadyInstalledSet=None): """Return a set of relative paths for all files that we need to fetch, and True if we're fetching actual files to install instead of metadata. Assumes that we care about the bundles @@ -331,6 +333,9 @@ class LocalRepository:

need = set()

+ if alreadyInstalledSet == None: + alreadyInstalledSet = set() + # Fetch missing metafiles. for f in self._metaFiles: try: @@ -526,7 +531,6 @@ class LocalRepository: for pfile in packages.values(): package = pfile.get()

- alreadyInstalled = set() pkgItems = {}

if usePackageSystem: @@ -541,7 +545,9 @@ class LocalRepository: else: try: if item.getChecker().isInstalled(): - alreadyInstalled.add(item.getRelativePath()) + alreadyInstalledSet.add(item.getRelativePath()) + if item.getRelativePath() in thpTransactionDict.keys(): + thpTransactionDict.pop(item.getRelativePath()) except thandy.CheckNotSupported, err: logging.warn("Can't check installed-ness of %s: %s", f[0], err) @@ -550,9 +556,10 @@ class LocalRepository:

for f in package['files']: rp, h = f[:2] - if rp in alreadyInstalled: + if rp in alreadyInstalledSet: logging.info("%s is already installed; no need to download", rp) + logCtrl("ALREADY", PKG=rp) continue

h_expected = thandy.formats.parseHash(h)