November 2011 - tor-commits - lists.torproject.org

[thandy/master] Adds the ThpTransaction class and its basic structure
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit 4e9e40f156aee7b4da7519ae1f71d3b1c32e375a Author: Tomas Touceda <chiiph(a)gentoo.org> Date: Fri Jun 24 10:23:13 2011 -0300 Adds the ThpTransaction class and its basic structure --- lib/thandy/packagesys/ThpPackages.py | 70 ++++++++++++++++++++++++++-------- lib/thandy/repository.py | 2 +- 2 files changed, 55 insertions(+), 17 deletions(-) diff --git a/lib/thandy/packagesys/ThpPackages.py b/lib/thandy/packagesys/ThpPackages.py index 3ca9d52..c976d50 100644 --- a/lib/thandy/packagesys/ThpPackages.py +++ b/lib/thandy/packagesys/ThpPackages.py @@ -21,6 +21,9 @@ class ThpDB(object): if self._thp_db_root is None: raise Exception("There is no THP_DB_ROOT variable set") + def dbPath(self): + return self._thp_db_root + def insert(self, pkg): thandy.util.replaceFile(os.path.join(self._thp_db_root, "pkg-status", pkg['package_name'])+".json", @@ -79,39 +82,70 @@ class ThpChecker(PS.Checker): def isInstalled(self): return self._version in self.getInstalledVersions() -class ThpInstaller(PS.Installer): - def __init__(self, relPath): - PS.Installer.__init__(self, relPath) +class ThpTransaction(object): + def __init__(self, packages): + self._raw_packages = packages + self._installers = [] self._db = ThpDB() - def __repr__(self): - return "ThpInstaller(%r)" %(self._relPath) + self._process() - def install(self): - print "Running thp installer", self._cacheRoot, self._relPath - self._thp_root = os.environ.get("THP_INSTALL_ROOT") - if self._thp_root is None: - raise Exception("There is no THP_INSTALL_ROOT variable set") + def _process(self): + for package in self._raw_packages: + self._installers.append(ThpInstaller(package['path'])) + + def _orderByDep(self): + """ Orders packages with a topological order by its dependencies """ + pass - lockfile = os.path.join(self._thp_db_root, ".lock") + def install(self): + lockfile = os.path.join(self._db.getPath(), ".lock") lock = LockFile(lockfile) try: lock.acquire() - pkg = ThpPackage(os.path.join(self._cacheRoot, self._relPath[1:])) - shutil.copytree() + order = self._orderByDep(self) + for pkg in order: + pkg.install() except AlreadyLocked: print "You can't run more than one instance of Thandy" except LockFailed: print "Can't acquire lock on %s" % lockfile + + lock.release() + + def remote(self): + raise NotImplemented() + +class ThpInstaller(PS.Installer): + def __init__(self, relPath, db = None): + PS.Installer.__init__(self, relPath) + self._db = db + if db is None: + self._db = ThpDB() + self._pkg = ThpPackage(os.path.join(self._cacheRoot, self._relPath[1:])) + + def __repr__(self): + return "ThpInstaller(%r)" %(self._relPath) + + def install(self): + print "Running thp installer", self._cacheRoot, self._relPath + self._thp_root = os.environ.get("THP_INSTALL_ROOT") + if self._thp_root is None: + raise Exception("There is no THP_INSTALL_ROOT variable set") + +# shutil.copytree() + # self._db.insert(pkg.getAll()) # self._db.statusInstalled(pkg.getAll()) # self._db.delete(pkg.getAll()) -# print self._db.exists(pkg.get("package_name")) def remove(self): print "Running thp remover" + def getDeps(self): + return self._pkg.getDeps() + class ThpPackage(object): def __init__(self, thp_path): self._thp_path = thp_path @@ -125,7 +159,7 @@ class ThpPackage(object): def _process(self): tmpPath = tempfile.mkdtemp(suffix=str(time.time()), - prefix="thp") + prefix="thp") thpFile = zipfile.ZipFile(self._thp_path) thpFile.extractall(tmpPath) @@ -137,11 +171,15 @@ class ThpPackage(object): def get(self, key): if self._metadata: - return self._metadata.get(key) + return self._metadata.get(key) def getAll(self): return self._metadata + def getDeps(self): + if 'require_packages' in self._metadata.keys(): + return self._metadata['require_packages'] + def isValid(self): return self._valid diff --git a/lib/thandy/repository.py b/lib/thandy/repository.py index ab1c356..d082afb 100644 --- a/lib/thandy/repository.py +++ b/lib/thandy/repository.py @@ -496,7 +496,7 @@ class LocalRepository: packages[rp] = pfile if thpBundle: - thpTransactionDict[bundle['name']] = [bundle['packages']] + thpTransactionDict[bundle['name']] = bundle['packages'] thpBundle = False # We have the packages. If we're downloading via bittorrent, we need

1 0

[thandy/master] Execute postinst script and serialize script execution
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit 93cc09c9d8f44f4f9e27e74cf9937d4db001b9c7 Author: Tomas Touceda <chiiph(a)gentoo.org> Date: Mon Jun 27 10:44:30 2011 -0300 Execute postinst script and serialize script execution --- lib/thandy/packagesys/ThpPackages.py | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diff --git a/lib/thandy/packagesys/ThpPackages.py b/lib/thandy/packagesys/ThpPackages.py index 4c3d8a8..0579b5c 100644 --- a/lib/thandy/packagesys/ThpPackages.py +++ b/lib/thandy/packagesys/ThpPackages.py @@ -138,6 +138,8 @@ class ThpTransaction(object): print "Preinst script for %s failed" % pkg sys.exit(1) pkg.install() + if pkg.run('postinst') != 0: + print "WARN: postinst script failed" except AlreadyLocked: print "You can't run more than one instance of Thandy" except LockFailed: @@ -309,4 +311,5 @@ class ScriptWrapper(object): def run(self): self._process = subprocess.Popen(["python", self._path], env=self._env) + self._process.wait() return self._process.returncode

1 0

[thandy/master] Update lockfile to work on OSX
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit c288511871e2f90016ae7782d55451cbcffe5d5e Author: Tomás Touceda <chiiph(a)torproject.org> Date: Tue Jul 26 12:47:45 2011 -0300 Update lockfile to work on OSX This changes the file modes, it seems OSX needs it this way. --- lib/thandy/lockfile/__init__.py | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 diff --git a/LICENSE b/LICENSE old mode 100644 new mode 100755 diff --git a/Makefile b/Makefile old mode 100644 new mode 100755 diff --git a/TODO b/TODO old mode 100644 new mode 100755 diff --git a/config/license-win32-bundle.cfg b/config/license-win32-bundle.cfg old mode 100644 new mode 100755 diff --git a/config/license-win32-package.cfg b/config/license-win32-package.cfg old mode 100644 new mode 100755 diff --git a/config/polipo-win32-bundle.cfg b/config/polipo-win32-bundle.cfg old mode 100644 new mode 100755 diff --git a/config/polipo-win32-package.cfg b/config/polipo-win32-package.cfg old mode 100644 new mode 100755 diff --git a/config/tor-src-bundle.cfg b/config/tor-src-bundle.cfg old mode 100644 new mode 100755 diff --git a/config/tor-src-package.cfg b/config/tor-src-package.cfg old mode 100644 new mode 100755 diff --git a/config/tor-win32-bundle.cfg b/config/tor-win32-bundle.cfg old mode 100644 new mode 100755 diff --git a/config/tor-win32-package.cfg b/config/tor-win32-package.cfg old mode 100644 new mode 100755 diff --git a/config/torbutton-win32-bundle.cfg b/config/torbutton-win32-bundle.cfg old mode 100644 new mode 100755 diff --git a/config/torbutton-win32-package.cfg b/config/torbutton-win32-package.cfg old mode 100644 new mode 100755 diff --git a/config/torvm-win32-bundle.cfg b/config/torvm-win32-bundle.cfg old mode 100644 new mode 100755 diff --git a/config/torvm-win32-package.cfg b/config/torvm-win32-package.cfg old mode 100644 new mode 100755 diff --git a/config/vidalia-marble-win32-bundle.cfg b/config/vidalia-marble-win32-bundle.cfg old mode 100644 new mode 100755 diff --git a/config/vidalia-marble-win32-package.cfg b/config/vidalia-marble-win32-package.cfg old mode 100644 new mode 100755 diff --git a/config/vidalia-win32-bundle.cfg b/config/vidalia-win32-bundle.cfg old mode 100644 new mode 100755 diff --git a/config/vidalia-win32-package.cfg b/config/vidalia-win32-package.cfg old mode 100644 new mode 100755 diff --git a/doc/HOWTO b/doc/HOWTO old mode 100644 new mode 100755 diff --git a/doc/interface.txt b/doc/interface.txt old mode 100644 new mode 100755 diff --git a/lib/thandy/ClientCLI.py b/lib/thandy/ClientCLI.py old mode 100644 new mode 100755 diff --git a/lib/thandy/ServerCLI.py b/lib/thandy/ServerCLI.py old mode 100644 new mode 100755 diff --git a/lib/thandy/SignerCLI.py b/lib/thandy/SignerCLI.py old mode 100644 new mode 100755 diff --git a/lib/thandy/__init__.py b/lib/thandy/__init__.py old mode 100644 new mode 100755 diff --git a/lib/thandy/bt_compat.py b/lib/thandy/bt_compat.py old mode 100644 new mode 100755 diff --git a/lib/thandy/checkJson.py b/lib/thandy/checkJson.py old mode 100644 new mode 100755 diff --git a/lib/thandy/download.py b/lib/thandy/download.py old mode 100644 new mode 100755 diff --git a/lib/thandy/encodeToXML.py b/lib/thandy/encodeToXML.py old mode 100644 new mode 100755 diff --git a/lib/thandy/formats.py b/lib/thandy/formats.py old mode 100644 new mode 100755 diff --git a/lib/thandy/keys.py b/lib/thandy/keys.py old mode 100644 new mode 100755 diff --git a/lib/thandy/lockfile/__init__.py b/lib/thandy/lockfile/__init__.py old mode 100644 new mode 100755 index a167cd8..6ed31d5 --- a/lib/thandy/lockfile/__init__.py +++ b/lib/thandy/lockfile/__init__.py @@ -168,6 +168,8 @@ class LockBase: # Thread objects in Python 2.4 and earlier do not have ident # attrs. Worm around that. ident = getattr(t, "ident", hash(t)) + if ident is None: + ident = hash(t) self.tname = "-%x" % (ident & 0xffffffff) else: self.tname = "" diff --git a/lib/thandy/lockfile/linklockfile.py b/lib/thandy/lockfile/linklockfile.py old mode 100644 new mode 100755 diff --git a/lib/thandy/lockfile/mkdirlockfile.py b/lib/thandy/lockfile/mkdirlockfile.py old mode 100644 new mode 100755 diff --git a/lib/thandy/lockfile/pidlockfile.py b/lib/thandy/lockfile/pidlockfile.py old mode 100644 new mode 100755 diff --git a/lib/thandy/lockfile/sqlitelockfile.py b/lib/thandy/lockfile/sqlitelockfile.py old mode 100644 new mode 100755 diff --git a/lib/thandy/master_keys.py b/lib/thandy/master_keys.py old mode 100644 new mode 100755 diff --git a/lib/thandy/packagesys/ExePackages.py b/lib/thandy/packagesys/ExePackages.py old mode 100644 new mode 100755 diff --git a/lib/thandy/packagesys/PackageDB.py b/lib/thandy/packagesys/PackageDB.py old mode 100644 new mode 100755 diff --git a/lib/thandy/packagesys/PackageSystem.py b/lib/thandy/packagesys/PackageSystem.py old mode 100644 new mode 100755 diff --git a/lib/thandy/packagesys/RPMPackages.py b/lib/thandy/packagesys/RPMPackages.py old mode 100644 new mode 100755 diff --git a/lib/thandy/packagesys/ThpPackages.py b/lib/thandy/packagesys/ThpPackages.py old mode 100644 new mode 100755 diff --git a/lib/thandy/packagesys/__init__.py b/lib/thandy/packagesys/__init__.py old mode 100644 new mode 100755 diff --git a/lib/thandy/repository.py b/lib/thandy/repository.py old mode 100644 new mode 100755 diff --git a/lib/thandy/socksurls.py b/lib/thandy/socksurls.py old mode 100644 new mode 100755 diff --git a/lib/thandy/tests.py b/lib/thandy/tests.py old mode 100644 new mode 100755 diff --git a/lib/thandy/util.py b/lib/thandy/util.py old mode 100644 new mode 100755 diff --git a/samples/example-bundle.cfg b/samples/example-bundle.cfg old mode 100644 new mode 100755 diff --git a/samples/example-mirrors.txt b/samples/example-mirrors.txt old mode 100644 new mode 100755 diff --git a/samples/example-package.cfg b/samples/example-package.cfg old mode 100644 new mode 100755 diff --git a/samples/example-thp-package.cfg b/samples/example-thp-package.cfg old mode 100644 new mode 100755 diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 diff --git a/specs/thandy-spec.txt b/specs/thandy-spec.txt old mode 100644 new mode 100755

1 0

[thandy/master] Run transactions only if --install is given, and create pkg-status if it doesn't exist
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit 40b3a86a82054c88f5865cef9b8d7fff8486f768 Author: Tomás Touceda <chiiph(a)torproject.org> Date: Fri Sep 2 19:26:42 2011 -0300 Run transactions only if --install is given, and create pkg-status if it doesn't exist --- lib/thandy/ClientCLI.py | 5 +++-- lib/thandy/packagesys/ThpPackages.py | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/thandy/ClientCLI.py b/lib/thandy/ClientCLI.py index 75aa347..2987918 100755 --- a/lib/thandy/ClientCLI.py +++ b/lib/thandy/ClientCLI.py @@ -156,8 +156,9 @@ def update(args): i.install() for bundle in thpTransactions: - thandy.packagesys.ThpPackages.ThpTransaction(thpTransactions[bundle], - repoRoot).install() + if install: + thandy.packagesys.ThpPackages.ThpTransaction(thpTransactions[bundle], + repoRoot).install() return diff --git a/lib/thandy/packagesys/ThpPackages.py b/lib/thandy/packagesys/ThpPackages.py index 090fe55..f359957 100755 --- a/lib/thandy/packagesys/ThpPackages.py +++ b/lib/thandy/packagesys/ThpPackages.py @@ -24,6 +24,9 @@ class ThpDB(object): self._thp_db_root = os.environ.get("THP_DB_ROOT") if self._thp_db_root is None: raise Exception("There is no THP_DB_ROOT variable set") + dbpath = os.path.join(self._thp_db_root, "pkg-status") + if not os.path.exists(dbpath): + os.mkdir(dbpat) def getPath(self): return self._thp_db_root

1 0

[thandy/master] Fix behavior for bundles with more than one thp and add thp
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit f04c3429dc33c9bcadfe8a3707f8646690838db8 Author: Tomás Touceda <chiiph(a)torproject.org> Date: Fri Sep 2 20:44:53 2011 -0300 Fix behavior for bundles with more than one thp and add thp --- lib/thandy/formats.py | 9 +++++++-- lib/thandy/packagesys/ThpPackages.py | 2 +- lib/thandy/repository.py | 4 ++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/thandy/formats.py b/lib/thandy/formats.py index f57ab46..0a73ff4 100755 --- a/lib/thandy/formats.py +++ b/lib/thandy/formats.py @@ -746,6 +746,7 @@ def makePackageObj(config_fname, package_fname): 'exe_registry_ent', 'db_key', 'db_val', 'command_install', 'command_remove', + 'thp_name', 'thp_version', ], preload) f = open(package_fname, 'rb') @@ -783,8 +784,12 @@ def makePackageObj(config_fname, package_fname): elif format == 'thp': extra['check_type'] = 'thp' extra['install_type'] = 'thp' - extra['thp_name'] = r['name'] - extra['thp_version'] = r['version'] + if not r.get('thp_name'): + raise thandy.FormatException("missing thp_name value") + extra['thp_name'] = r['thp_name'] + if not r.get('thp_version'): + raise thandy.FormatException("missing thp_version value") + extra['thp_version'] = r['thp_version'] if r.get('command_install'): extra['install_type'] = 'command' diff --git a/lib/thandy/packagesys/ThpPackages.py b/lib/thandy/packagesys/ThpPackages.py index f359957..493a94a 100755 --- a/lib/thandy/packagesys/ThpPackages.py +++ b/lib/thandy/packagesys/ThpPackages.py @@ -26,7 +26,7 @@ class ThpDB(object): raise Exception("There is no THP_DB_ROOT variable set") dbpath = os.path.join(self._thp_db_root, "pkg-status") if not os.path.exists(dbpath): - os.mkdir(dbpat) + os.mkdir(dbpath) def getPath(self): return self._thp_db_root diff --git a/lib/thandy/repository.py b/lib/thandy/repository.py index bb7d2f6..20a1e9a 100755 --- a/lib/thandy/repository.py +++ b/lib/thandy/repository.py @@ -493,9 +493,9 @@ class LocalRepository: packages[rp] = pfile if pfile_data["format"] == "thp": - thpTransactionDict[bundle['name']] = {} + if not bundle['name'] in thpTransactionDict.keys(): + thpTransactionDict[bundle['name']] = {} thpTransactionDict[bundle['name']][pfile_data['name']] = pfile_data - continue # We have the packages. If we're downloading via bittorrent, we need # the .torrent metafiles, as well.

1 0

[thandy/master] Log with loggin instead of print, and add thp files to download when needed
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit f3602b42ad6db5918323c3b3be1c0d4c335e9c61 Author: Tomás Touceda <chiiph(a)torproject.org> Date: Thu Sep 1 23:50:54 2011 -0300 Log with loggin instead of print, and add thp files to download when needed --- lib/thandy/packagesys/PackageSystem.py | 9 +++++---- lib/thandy/packagesys/ThpPackages.py | 30 +++++++++++++++--------------- lib/thandy/repository.py | 7 ++++--- 3 files changed, 24 insertions(+), 22 deletions(-) diff --git a/lib/thandy/packagesys/PackageSystem.py b/lib/thandy/packagesys/PackageSystem.py index d1060d0..9fb878f 100755 --- a/lib/thandy/packagesys/PackageSystem.py +++ b/lib/thandy/packagesys/PackageSystem.py @@ -18,8 +18,11 @@ def getItemsFromPackage(pkg): extra = {} checker = getChecker(relPath, extra, defaultFormat=format, package=pkg) - installer = getInstaller(relPath, extra, - defaultFormat=format, package=pkg) + # we don't need this + # installer = getInstaller(relPath, extra, + # defaultFormat=format, package=pkg) + installer = None + result[relPath] = PackageItem(relPath, checker, installer) return result @@ -70,7 +73,6 @@ def getChecker(relPath, extra, defaultFormat, package): k,v=extra['registry_ent'] return thandy.packagesys.ExePackages.RegistryChecker(k,v) elif checkType == 'thp': - print "Getting thp checker" import thandy.packagesys.ThpPackages return thandy.packagesys.ThpPackages.ThpChecker( extra['thp_name'], extra['thp_version']) @@ -109,7 +111,6 @@ def getInstaller(relPath, extra, defaultFormat, package): installer = thandy.packagesys.ExePackages.CommandInstaller( relPath, extra['cmd_install'], extra.get('cmd_remove')) elif installType == 'thp': - print "Getting thp installer" import thandy.packagesys.ThpPackages installer = thandy.packagesys.ThpPackages.ThpInstaller( relPath) diff --git a/lib/thandy/packagesys/ThpPackages.py b/lib/thandy/packagesys/ThpPackages.py index 0579b5c..090fe55 100755 --- a/lib/thandy/packagesys/ThpPackages.py +++ b/lib/thandy/packagesys/ThpPackages.py @@ -124,22 +124,22 @@ class ThpTransaction(object): lockfile = os.path.join(self._db.getPath(), "db") lock = LockFile(lockfile) try: - print "Acquiring lock..." + logging.info("Acquiring lock...") lock.acquire() - print "Lock acquired" + logging.info("Lock acquired") order = self._orderByDep() for pkg in order: if pkg.run('checkinst') != 0: - print "Check inst failed for %s" % pkg + logging.info("Check inst failed for %s" % pkg) sys.exit(1) for pkg in order: - print "Starting installation using", pkg + logging.info("Starting installation using %s" % pkg) if pkg.run('preinst') != 0: - print "Preinst script for %s failed" % pkg + logging.info("Preinst script for %s failed" % pkg) sys.exit(1) pkg.install() if pkg.run('postinst') != 0: - print "WARN: postinst script failed" + logging.info("postinst script failed") except AlreadyLocked: print "You can't run more than one instance of Thandy" except LockFailed: @@ -163,16 +163,16 @@ class ThpInstaller(PS.Installer): return "ThpInstaller(%r)" %(self._relPath) def install(self): - print "Running thp installer", self._cacheRoot, self._relPath + logging.info("Running thp installer %s %s" % (self._cacheRoot, self._relPath)) self._thp_root = os.environ.get("THP_INSTALL_ROOT") if self._thp_root is None: raise Exception("There is no THP_INSTALL_ROOT variable set") destPath = os.path.join(self._thp_root, self._pkg.get("package_name")) - print "Destination directory:", destPath + logging.info("Destination directory: %s" % destPath) if self._db.exists(self._pkg.get("package_name")): - print "%s is already installed, switching to upgrade mode." % self._pkg.get("package_name") + logging.info("%s is already installed, switching to upgrade mode." % self._pkg.get("package_name")) self._db.startUpgrade() pkg_metadata = self._pkg.getAll() @@ -183,13 +183,13 @@ class ThpInstaller(PS.Installer): try: os.mkdir(dir) except: - print "%s: Already exists, using it." % dir + logging.info("%s: Already exists, using it." % dir) for file in self._pkg.get('manifest'): if file['is_config']: - print "Ignoring file:", file + logging.info("Ignoring file: %s" % file) else: - print "Processing file:", file + logging.info("Processing file: %s" % file) try: # Create all the needed dirs os.makedirs(os.sep.join((os.path.join(destPath, file['name']) @@ -201,7 +201,7 @@ class ThpInstaller(PS.Installer): os.path.join(destPath, file['name'])); if self._db.isUpgrading(): - print "Finishing upgrade." + logging.info("Finishing upgrade.") self._db.finishUpgrade(self._pkg.get('package_name')) self._db.statusInstalled(pkg_metadata) @@ -243,8 +243,8 @@ class ThpPackage(object): (allOk, where) = self._validateFiles(self._tmp_path) if not allOk: - print "These files have different digests:" - print where + logging.info("These files have different digests:") + logging.info(where) sys.exit(1) if "scripts" in self._metadata: diff --git a/lib/thandy/repository.py b/lib/thandy/repository.py index f0b51b8..bb7d2f6 100755 --- a/lib/thandy/repository.py +++ b/lib/thandy/repository.py @@ -489,13 +489,14 @@ class LocalRepository: # are thp too. But we continue with the loop to check every # package digest and signature pfile_data = pfile.get() + + packages[rp] = pfile + if pfile_data["format"] == "thp": thpTransactionDict[bundle['name']] = {} thpTransactionDict[bundle['name']][pfile_data['name']] = pfile_data continue - packages[rp] = pfile - # We have the packages. If we're downloading via bittorrent, we need # the .torrent metafiles, as well. if thandy.bt_compat.BtCompat.shouldUseBt(): @@ -524,7 +525,7 @@ class LocalRepository: # files? for pfile in packages.values(): package = pfile.get() - + alreadyInstalled = set() pkgItems = {}

1 0

[thandy/master] Improve re-install behavior
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit 7d87a55747e82c7b23daa48581135c35b553aeaf Author: Tomás Touceda <chiiph(a)torproject.org> Date: Sat Sep 3 21:58:47 2011 -0300 Improve re-install behavior Also add more controller oriented output --- lib/thandy/ClientCLI.py | 41 +++++++++++++++++++--------------- lib/thandy/download.py | 3 ++ lib/thandy/packagesys/ThpPackages.py | 37 ++++++++++++++++++++++-------- lib/thandy/repository.py | 15 +++++++++--- 4 files changed, 64 insertions(+), 32 deletions(-) diff --git a/lib/thandy/ClientCLI.py b/lib/thandy/ClientCLI.py index 2987918..f568a2c 100755 --- a/lib/thandy/ClientCLI.py +++ b/lib/thandy/ClientCLI.py @@ -122,6 +122,7 @@ def update(args): installable = {} btMetadata = {} thpTransactions = {} + alreadyInstalled = set() logging.info("Checking for files to update.") files, downloadingFiles = repo.getFilesToUpdate( trackingBundles=args, @@ -130,34 +131,38 @@ def update(args): usePackageSystem=use_packagesys, installableDict=installable, btMetadataDict=btMetadata, - thpTransactionDict=thpTransactions) + thpTransactionDict=thpTransactions, + alreadyInstalledSet=alreadyInstalled) if forceCheck: files.add("/meta/timestamp.txt") forceCheck = False if (thpTransactions or installable) and not files: - for p, d in installable.items(): - for n, i in d.items(): - if i.canInstall(): - logCtrl("CAN_INSTALL", PKG=p, ITEM=n) - else: - logCtrl("NO_INSTALL", PKG=p, ITEM=n) - i.setCacheRoot(repoRoot) - - logging.info("Ready to install packages for files: %s", - ", ".join(sorted(installable.keys()))) - if install: - # XXXX handle ordering - for p in installable.values(): - for h in p.values(): - i = h.getInstaller() - if i != None: - i.install() + # for p, d in installable.items(): + # for n, i in d.items(): + # if i.canInstall(): + # logCtrl("CAN_INSTALL", PKG=p, ITEM=n) + # else: + # logCtrl("NO_INSTALL", PKG=p, ITEM=n) + # i.setCacheRoot(repoRoot) + + # logging.info("Ready to install packages for files: %s", + # ", ".join(sorted(installable.keys()))) + # if install: + # # XXXX handle ordering + # for p in installable.values(): + # for h in p.values(): + # i = h.getInstaller() + # if i != None: + # i.install() + + logCtrl("READY", BUNDLE=",".join(thpTransactions.keys())) for bundle in thpTransactions: if install: thandy.packagesys.ThpPackages.ThpTransaction(thpTransactions[bundle], + alreadyInstalled, repoRoot).install() return diff --git a/lib/thandy/download.py b/lib/thandy/download.py index 52c6fe4..c2a9e61 100755 --- a/lib/thandy/download.py +++ b/lib/thandy/download.py @@ -13,6 +13,8 @@ import thandy.util import thandy.socksurls import thandy.checkJson +from thandy.util import logCtrl + class BadCompoundData(thandy.DownloadError): """DOCDOC""" pass @@ -514,6 +516,7 @@ class DownloadJob: total += len(c) logging.debug("Got %s/%s bytes from %s", total, expectLength, url) + logCtrl("DOWNLOAD", TOTAL=str(total), EXPECT=str(expectLength), URL=url) if self._wantLength != None and total > self._wantLength: logging.warn("Read too many bytes from %s; got %s, but " "wanted %s", url, total, self._wantLength) diff --git a/lib/thandy/packagesys/ThpPackages.py b/lib/thandy/packagesys/ThpPackages.py index 493a94a..dd1d399 100755 --- a/lib/thandy/packagesys/ThpPackages.py +++ b/lib/thandy/packagesys/ThpPackages.py @@ -70,7 +70,17 @@ class ThpDB(object): contents = open(fname, "r").read() metadata = json.loads(contents) version = metadata['package_version'] - return fexists, version + + fname = os.path.join(self._thp_db_root, "pkg-status", name+".status") + fexists2 = os.path.exists(fname) + + status = "" + if fexists2: + contents = open(fname, "r").read() + metadata = json.loads(contents) + status = metadata['status'] + + return fexists, version, status def statusInProgress(self, pkg): thandy.util.replaceFile(os.path.join(self._thp_db_root, "pkg-status", @@ -94,30 +104,35 @@ class ThpChecker(PS.Checker): def getInstalledVersions(self): versions = [] - (exists, version) = self._db.exists(self._name) + (exists, version, status) = self._db.exists(self._name) if exists: versions.append(version) - return versions + return versions, status def isInstalled(self): - return self._version in self.getInstalledVersions() + versions, status = self.getInstalledVersions() + # if status = IN_PROGRESS a previous installation failed + # we need to reinstall + return (status != "IN_PROGRESS" and self._version in versions) class ThpTransaction(object): - def __init__(self, packages, repoRoot): + def __init__(self, packages, alreadyInstalled, repoRoot): self._raw_packages = packages self._repo_root = repoRoot self._installers = [] + self._alreadyInstalled = alreadyInstalled self._db = ThpDB() self._process() def _process(self): for package in self._raw_packages.keys(): - self._installers.append(ThpInstaller(self._raw_packages[package]['files'][0][0], - self._db, - self._repo_root)) + if not (self._raw_packages[package]['files'][0][0] in self._alreadyInstalled): + self._installers.append(ThpInstaller(self._raw_packages[package]['files'][0][0], + self._db, + self._repo_root)) def _orderByDep(self): """ Orders packages with a topological order by its dependencies """ @@ -148,6 +163,7 @@ class ThpTransaction(object): except LockFailed: print "Can't acquire lock on %s" % lockfile finally: + logging.info("Releasing lock...") lock.release() def remote(self): @@ -174,10 +190,11 @@ class ThpInstaller(PS.Installer): destPath = os.path.join(self._thp_root, self._pkg.get("package_name")) logging.info("Destination directory: %s" % destPath) - if self._db.exists(self._pkg.get("package_name")): + (exists, _, _) = self._db.exists(self._pkg.get("package_name")) + if exists: logging.info("%s is already installed, switching to upgrade mode." % self._pkg.get("package_name")) self._db.startUpgrade() - + pkg_metadata = self._pkg.getAll() self._db.insert(pkg_metadata) self._db.statusInProgress(pkg_metadata) diff --git a/lib/thandy/repository.py b/lib/thandy/repository.py index 20a1e9a..0b45c8d 100755 --- a/lib/thandy/repository.py +++ b/lib/thandy/repository.py @@ -5,6 +5,8 @@ import thandy.util import thandy.packagesys.PackageSystem import thandy.bt_compat +from thandy.util import logCtrl + json = thandy.util.importJSON() import logging @@ -300,7 +302,7 @@ class LocalRepository: def getFilesToUpdate(self, now=None, trackingBundles=(), hashDict=None, lengthDict=None, usePackageSystem=True, installableDict=None, btMetadataDict=None, - thpTransactionDict=None): + thpTransactionDict=None, alreadyInstalledSet=None): """Return a set of relative paths for all files that we need to fetch, and True if we're fetching actual files to install instead of metadata. Assumes that we care about the bundles @@ -331,6 +333,9 @@ class LocalRepository: need = set() + if alreadyInstalledSet == None: + alreadyInstalledSet = set() + # Fetch missing metafiles. for f in self._metaFiles: try: @@ -526,7 +531,6 @@ class LocalRepository: for pfile in packages.values(): package = pfile.get() - alreadyInstalled = set() pkgItems = {} if usePackageSystem: @@ -541,7 +545,9 @@ class LocalRepository: else: try: if item.getChecker().isInstalled(): - alreadyInstalled.add(item.getRelativePath()) + alreadyInstalledSet.add(item.getRelativePath()) + if item.getRelativePath() in thpTransactionDict.keys(): + thpTransactionDict.pop(item.getRelativePath()) except thandy.CheckNotSupported, err: logging.warn("Can't check installed-ness of %s: %s", f[0], err) @@ -550,9 +556,10 @@ class LocalRepository: for f in package['files']: rp, h = f[:2] - if rp in alreadyInstalled: + if rp in alreadyInstalledSet: logging.info("%s is already installed; no need to download", rp) + logCtrl("ALREADY", PKG=rp) continue h_expected = thandy.formats.parseHash(h)

1 0

[thandy/master] Add the LICENSE file for the lockfile library
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit 23aa3ccd0dc33e158e8ea1d65cbf055552025d67 Author: Tomás Touceda <chiiph(a)torproject.org> Date: Wed Sep 7 11:18:48 2011 -0300 Add the LICENSE file for the lockfile library --- lib/thandy/lockfile/LICENSE | 21 +++++++++++++++++++++ 1 files changed, 21 insertions(+), 0 deletions(-) diff --git a/lib/thandy/lockfile/LICENSE b/lib/thandy/lockfile/LICENSE new file mode 100644 index 0000000..610c079 --- /dev/null +++ b/lib/thandy/lockfile/LICENSE @@ -0,0 +1,21 @@ +This is the MIT license: http://www.opensource.org/licenses/mit-license.php + +Copyright (c) 2007 Skip Montanaro. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to +deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE.

1 0

[thandy/master] Try to install only if isReady
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit c412100ebd0001c02920a9a31f5ea26f7c94d262 Author: Tomás Touceda <chiiph(a)torproject.org> Date: Sat Sep 3 22:54:00 2011 -0300 Try to install only if isReady Also, it removes a bunch of code for non-thp packages. --- lib/thandy/ClientCLI.py | 23 +++-------------------- 1 files changed, 3 insertions(+), 20 deletions(-) diff --git a/lib/thandy/ClientCLI.py b/lib/thandy/ClientCLI.py index 828d853..aa60205 100755 --- a/lib/thandy/ClientCLI.py +++ b/lib/thandy/ClientCLI.py @@ -138,33 +138,16 @@ def update(args): files.add("/meta/timestamp.txt") forceCheck = False - if (thpTransactions or installable) and not files: - # for p, d in installable.items(): - # for n, i in d.items(): - # if i.canInstall(): - # logCtrl("CAN_INSTALL", PKG=p, ITEM=n) - # else: - # logCtrl("NO_INSTALL", PKG=p, ITEM=n) - # i.setCacheRoot(repoRoot) - - # logging.info("Ready to install packages for files: %s", - # ", ".join(sorted(installable.keys()))) - # if install: - # # XXXX handle ordering - # for p in installable.values(): - # for h in p.values(): - # i = h.getInstaller() - # if i != None: - # i.install() - + if thpTransactions and not files: for bundle in thpTransactions: tr = thandy.packagesys.ThpPackages.ThpTransaction(thpTransactions[bundle], alreadyInstalled, repoRoot) if tr.isReady(): logCtrl("READY", BUNDLE=bundle) - if install: + if install and tr.isReady(): tr.install() + logCtrl("INSTALLED", BUNDLE=bundle) return

1 0

[thandy/master] Improve handling of updates to thp bundles
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit 0efda8ed80c76b5cdd1cca3a06b857fc1eb2809e Author: Tomás Touceda <chiiph(a)torproject.org> Date: Sat Sep 3 22:21:34 2011 -0300 Improve handling of updates to thp bundles It only installs thp packages that aren't already installed as they should be. It also outputs READY for bundles that have at least one package in them that's installable under this restrictions. --- lib/thandy/ClientCLI.py | 11 ++++++----- lib/thandy/packagesys/ThpPackages.py | 3 +++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/thandy/ClientCLI.py b/lib/thandy/ClientCLI.py index f568a2c..828d853 100755 --- a/lib/thandy/ClientCLI.py +++ b/lib/thandy/ClientCLI.py @@ -157,13 +157,14 @@ def update(args): # if i != None: # i.install() - logCtrl("READY", BUNDLE=",".join(thpTransactions.keys())) - for bundle in thpTransactions: + tr = thandy.packagesys.ThpPackages.ThpTransaction(thpTransactions[bundle], + alreadyInstalled, + repoRoot) + if tr.isReady(): + logCtrl("READY", BUNDLE=bundle) if install: - thandy.packagesys.ThpPackages.ThpTransaction(thpTransactions[bundle], - alreadyInstalled, - repoRoot).install() + tr.install() return diff --git a/lib/thandy/packagesys/ThpPackages.py b/lib/thandy/packagesys/ThpPackages.py index dd1d399..49ce19e 100755 --- a/lib/thandy/packagesys/ThpPackages.py +++ b/lib/thandy/packagesys/ThpPackages.py @@ -134,6 +134,9 @@ class ThpTransaction(object): self._db, self._repo_root)) + def isReady(self): + return (len(self._installers) > 0) + def _orderByDep(self): """ Orders packages with a topological order by its dependencies """ return self._installers

1 0