- tor-commits - lists.torproject.org

[thandy/master] Adds installation with subdirs and upgrade based on the spec
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit b7bf4a7a18332669cf5f0c1442819db74d76d4f2 Author: Tomas Touceda <chiiph(a)gentoo.org> Date: Sat Jun 25 22:13:29 2011 -0300 Adds installation with subdirs and upgrade based on the spec --- lib/thandy/ClientCLI.py | 2 - lib/thandy/packagesys/ThpPackages.py | 48 +++++++++++++++++++++++++++++---- 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/lib/thandy/ClientCLI.py b/lib/thandy/ClientCLI.py index 92f038a..75aa347 100644 --- a/lib/thandy/ClientCLI.py +++ b/lib/thandy/ClientCLI.py @@ -155,9 +155,7 @@ def update(args): if i != None: i.install() - print "Bundles with all THP packages:" for bundle in thpTransactions: - # TODO: ThpTransaction goes here! thandy.packagesys.ThpPackages.ThpTransaction(thpTransactions[bundle], repoRoot).install() diff --git a/lib/thandy/packagesys/ThpPackages.py b/lib/thandy/packagesys/ThpPackages.py index 7edb42b..bf9029b 100644 --- a/lib/thandy/packagesys/ThpPackages.py +++ b/lib/thandy/packagesys/ThpPackages.py @@ -18,6 +18,7 @@ json = thandy.util.importJSON() class ThpDB(object): def __init__(self): + self._upgrade = False self._thp_db_root = os.environ.get("THP_DB_ROOT") if self._thp_db_root is None: raise Exception("There is no THP_DB_ROOT variable set") @@ -25,9 +26,24 @@ class ThpDB(object): def getPath(self): return self._thp_db_root + def startUpgrade(self): + self._upgrade = True + + def finishUpgrade(self, name): + fname = os.path.join(self._thp_db_root, "pkg-status", name+".json") + shutil.move(fname+".new", fname) + self._upgrade = False + + def isUpgrading(self): + return self._upgrade + def insert(self, pkg): - thandy.util.replaceFile(os.path.join(self._thp_db_root, "pkg-status", - pkg['package_name'])+".json", + fname = os.path.join(self._thp_db_root, "pkg-status", + pkg['package_name']+".json") + if self._upgrade: + fname += ".new" + + thandy.util.replaceFile(fname, json.dumps(pkg)) def delete(self, pkg): @@ -144,6 +160,10 @@ class ThpInstaller(PS.Installer): destPath = os.path.join(self._thp_root, self._pkg.get("package_name")) print "Destination directory:", destPath + if self._db.exists(self._pkg.get("package_name")): + print "%s is already installed, switching to upgrade mode." % self._pkg.get("package_name") + self._db.startUpgrade() + pkg_metadata = self._pkg.getAll() self._db.insert(pkg_metadata) self._db.statusInProgress(pkg_metadata) @@ -158,9 +178,20 @@ class ThpInstaller(PS.Installer): if file['is_config']: print "Ignoring file:", file else: - print "Processing file:", file - shutil.copyfile(os.path.join(self._pkg.getTmpPath(), "content", file['name']), - os.path.join(destPath, file['name'])); + print "Processing file:", file + try: + # Create all the needed dirs + os.makedirs(os.sep.join((os.path.join(destPath, file['name']) + .split(os.path.sep)[:-1]))) + except: + # Ignore if it already exists + pass + shutil.copy(os.path.join(self._pkg.getTmpPath(), "content", file['name']), + os.path.join(destPath, file['name'])); + + if self._db.isUpgrading(): + print "Finishing upgrade." + self._db.finishUpgrade(self._pkg.get('package_name')) self._db.statusInstalled(pkg_metadata) @@ -193,7 +224,11 @@ class ThpPackage(object): thpFile.extractall(self._tmp_path) contents = open(os.path.join(self._tmp_path, "meta", "package.json")).read() self._metadata = json.loads(contents) - print self._validateFiles(self._tmp_path) + (allOk, where) = self._validateFiles(self._tmp_path) + + if not allOk: + print "These files have different digests:" + print where def get(self, key): if self._metadata: @@ -222,3 +257,4 @@ class ThpPackage(object): f.close() if newdigest != digest: return (False, [name, digest, newdigest]) + return (True, None)

1 0

[thandy/master] Correct some silly errors
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit 762297dd1a1ab28709b551a93260e5dea824dd3d Author: Tomas Touceda <chiiph(a)gentoo.org> Date: Fri Jun 24 09:38:06 2011 -0300 Correct some silly errors --- lib/thandy/ClientCLI.py | 4 ++-- lib/thandy/repository.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/thandy/ClientCLI.py b/lib/thandy/ClientCLI.py index 97951b4..6a78bf6 100644 --- a/lib/thandy/ClientCLI.py +++ b/lib/thandy/ClientCLI.py @@ -128,7 +128,7 @@ def update(args): lengthDict=lengths, usePackageSystem=use_packagesys, installableDict=installable, - btMetadataDict=btMetadata + btMetadataDict=btMetadata, thpTransactionDict=thpTransactions) if forceCheck: @@ -157,7 +157,7 @@ def update(args): print "Bundles with all THP packages:" for bundle in thpTransactions: # TODO: ThpTransaction goes here! - print bundle + print thpTransactions[bundle] return diff --git a/lib/thandy/repository.py b/lib/thandy/repository.py index 1bc1ce4..ab1c356 100644 --- a/lib/thandy/repository.py +++ b/lib/thandy/repository.py @@ -456,7 +456,7 @@ class LocalRepository: # Okay. So we have some bundles. See if we have their packages. packages = {} - thpbundle = False + thpBundle = False for bfile in bundles.values(): bundle = bfile.get() for pkginfo in bundle['packages']: @@ -489,7 +489,7 @@ class LocalRepository: # We assume that if there is one thp package then all the rest # are thp too. But we continue with the loop to check every # package digest and signature - if thpBunle or pfile.get("format") == "thp": + if thpBundle or pfile.get()["format"] == "thp": thpBundle = True continue @@ -578,7 +578,7 @@ class LocalRepository: # value and just use installableDict from the caller. if pkgItems.has_key(rp): if pkgItems[rp]: - installableDict.setdefault(pkg_rp, {})[rp] = pkgItems[rp] + installableDict.setdefault(pkg_rp, {})[rp] = pkgItems[rp] # Okay; these are the files we need.

1 0

[thandy/master] Start scripts part
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit a7b51a462a22c8e593090b1098a19c3fd11136b6 Author: Tomas Touceda <chiiph(a)gentoo.org> Date: Sun Jun 26 00:32:41 2011 -0300 Start scripts part --- lib/thandy/packagesys/ThpPackages.py | 52 ++++++++++++++++++++++++++++++++++ 1 files changed, 52 insertions(+), 0 deletions(-) diff --git a/lib/thandy/packagesys/ThpPackages.py b/lib/thandy/packagesys/ThpPackages.py index bf9029b..4c3d8a8 100644 --- a/lib/thandy/packagesys/ThpPackages.py +++ b/lib/thandy/packagesys/ThpPackages.py @@ -6,6 +6,8 @@ import zipfile import tempfile import time import shutil +import subprocess +import sys from lockfile import LockFile, AlreadyLocked, LockFailed @@ -127,7 +129,14 @@ class ThpTransaction(object): print "Lock acquired" order = self._orderByDep() for pkg in order: + if pkg.run('checkinst') != 0: + print "Check inst failed for %s" % pkg + sys.exit(1) + for pkg in order: print "Starting installation using", pkg + if pkg.run('preinst') != 0: + print "Preinst script for %s failed" % pkg + sys.exit(1) pkg.install() except AlreadyLocked: print "You can't run more than one instance of Thandy" @@ -201,12 +210,16 @@ class ThpInstaller(PS.Installer): def getDeps(self): return self._pkg.getDeps() + def run(self, key): + return self._pkg.run(key) + class ThpPackage(object): def __init__(self, thp_path): self._thp_path = thp_path self._metadata = None self._valid = False self._tmp_path = "" + self._scripts = {} self._process() @@ -222,6 +235,7 @@ class ThpPackage(object): thpFile = zipfile.ZipFile(self._thp_path) thpFile.extractall(self._tmp_path) + json_file = os.path.join(self._tmp_path, "meta", "package.json") contents = open(os.path.join(self._tmp_path, "meta", "package.json")).read() self._metadata = json.loads(contents) (allOk, where) = self._validateFiles(self._tmp_path) @@ -229,6 +243,29 @@ class ThpPackage(object): if not allOk: print "These files have different digests:" print where + sys.exit(1) + + if "scripts" in self._metadata: + if "python2" in self._metadata['scripts']: + for script in self._metadata['scripts']['python2']: + env = {} + env['THP_PACKAGE_NAME'] = self._metadata['package_name'] + env['THP_OLD_VERSION'] = "" + env['THP_NEW_VERSION'] = self._metadata['package_version'] + env['THP_OLD_INSTALL_ROOT'] = "" + env['THP_INSTALL_ROOT'] = os.getenv("THP_INSTALL_ROOT") + env['THP_JSON_FILE'] = json_file + env['THP_VERBOSE'] = 1 + env['THP_PURGE'] = 0 + env['THP_TEMP_DIR'] = self._tmp_path + + sw = ScriptWrapper(os.path.join(self._tmp_path, "meta", + "scripts", script[0]), env) + + for type in script[1]: + self._scripts[type] = sw + else: + sys.exit(1) def get(self, key): if self._metadata: @@ -258,3 +295,18 @@ class ThpPackage(object): if newdigest != digest: return (False, [name, digest, newdigest]) return (True, None) + + def run(self, key): + if key in self._scripts.keys(): + return self._scripts[key].run() + return 0 + +class ScriptWrapper(object): + def __init__(self, path = None, env = None): + self._path = path + self._env = None + + def run(self): + self._process = subprocess.Popen(["python", self._path], + env=self._env) + return self._process.returncode

1 0

[thandy/master] Adds the ThpTransaction class and its basic structure
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit 4e9e40f156aee7b4da7519ae1f71d3b1c32e375a Author: Tomas Touceda <chiiph(a)gentoo.org> Date: Fri Jun 24 10:23:13 2011 -0300 Adds the ThpTransaction class and its basic structure --- lib/thandy/packagesys/ThpPackages.py | 70 ++++++++++++++++++++++++++-------- lib/thandy/repository.py | 2 +- 2 files changed, 55 insertions(+), 17 deletions(-) diff --git a/lib/thandy/packagesys/ThpPackages.py b/lib/thandy/packagesys/ThpPackages.py index 3ca9d52..c976d50 100644 --- a/lib/thandy/packagesys/ThpPackages.py +++ b/lib/thandy/packagesys/ThpPackages.py @@ -21,6 +21,9 @@ class ThpDB(object): if self._thp_db_root is None: raise Exception("There is no THP_DB_ROOT variable set") + def dbPath(self): + return self._thp_db_root + def insert(self, pkg): thandy.util.replaceFile(os.path.join(self._thp_db_root, "pkg-status", pkg['package_name'])+".json", @@ -79,39 +82,70 @@ class ThpChecker(PS.Checker): def isInstalled(self): return self._version in self.getInstalledVersions() -class ThpInstaller(PS.Installer): - def __init__(self, relPath): - PS.Installer.__init__(self, relPath) +class ThpTransaction(object): + def __init__(self, packages): + self._raw_packages = packages + self._installers = [] self._db = ThpDB() - def __repr__(self): - return "ThpInstaller(%r)" %(self._relPath) + self._process() - def install(self): - print "Running thp installer", self._cacheRoot, self._relPath - self._thp_root = os.environ.get("THP_INSTALL_ROOT") - if self._thp_root is None: - raise Exception("There is no THP_INSTALL_ROOT variable set") + def _process(self): + for package in self._raw_packages: + self._installers.append(ThpInstaller(package['path'])) + + def _orderByDep(self): + """ Orders packages with a topological order by its dependencies """ + pass - lockfile = os.path.join(self._thp_db_root, ".lock") + def install(self): + lockfile = os.path.join(self._db.getPath(), ".lock") lock = LockFile(lockfile) try: lock.acquire() - pkg = ThpPackage(os.path.join(self._cacheRoot, self._relPath[1:])) - shutil.copytree() + order = self._orderByDep(self) + for pkg in order: + pkg.install() except AlreadyLocked: print "You can't run more than one instance of Thandy" except LockFailed: print "Can't acquire lock on %s" % lockfile + + lock.release() + + def remote(self): + raise NotImplemented() + +class ThpInstaller(PS.Installer): + def __init__(self, relPath, db = None): + PS.Installer.__init__(self, relPath) + self._db = db + if db is None: + self._db = ThpDB() + self._pkg = ThpPackage(os.path.join(self._cacheRoot, self._relPath[1:])) + + def __repr__(self): + return "ThpInstaller(%r)" %(self._relPath) + + def install(self): + print "Running thp installer", self._cacheRoot, self._relPath + self._thp_root = os.environ.get("THP_INSTALL_ROOT") + if self._thp_root is None: + raise Exception("There is no THP_INSTALL_ROOT variable set") + +# shutil.copytree() + # self._db.insert(pkg.getAll()) # self._db.statusInstalled(pkg.getAll()) # self._db.delete(pkg.getAll()) -# print self._db.exists(pkg.get("package_name")) def remove(self): print "Running thp remover" + def getDeps(self): + return self._pkg.getDeps() + class ThpPackage(object): def __init__(self, thp_path): self._thp_path = thp_path @@ -125,7 +159,7 @@ class ThpPackage(object): def _process(self): tmpPath = tempfile.mkdtemp(suffix=str(time.time()), - prefix="thp") + prefix="thp") thpFile = zipfile.ZipFile(self._thp_path) thpFile.extractall(tmpPath) @@ -137,11 +171,15 @@ class ThpPackage(object): def get(self, key): if self._metadata: - return self._metadata.get(key) + return self._metadata.get(key) def getAll(self): return self._metadata + def getDeps(self): + if 'require_packages' in self._metadata.keys(): + return self._metadata['require_packages'] + def isValid(self): return self._valid diff --git a/lib/thandy/repository.py b/lib/thandy/repository.py index ab1c356..d082afb 100644 --- a/lib/thandy/repository.py +++ b/lib/thandy/repository.py @@ -496,7 +496,7 @@ class LocalRepository: packages[rp] = pfile if thpBundle: - thpTransactionDict[bundle['name']] = [bundle['packages']] + thpTransactionDict[bundle['name']] = bundle['packages'] thpBundle = False # We have the packages. If we're downloading via bittorrent, we need

1 0

[thandy/master] Execute postinst script and serialize script execution
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit 93cc09c9d8f44f4f9e27e74cf9937d4db001b9c7 Author: Tomas Touceda <chiiph(a)gentoo.org> Date: Mon Jun 27 10:44:30 2011 -0300 Execute postinst script and serialize script execution --- lib/thandy/packagesys/ThpPackages.py | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diff --git a/lib/thandy/packagesys/ThpPackages.py b/lib/thandy/packagesys/ThpPackages.py index 4c3d8a8..0579b5c 100644 --- a/lib/thandy/packagesys/ThpPackages.py +++ b/lib/thandy/packagesys/ThpPackages.py @@ -138,6 +138,8 @@ class ThpTransaction(object): print "Preinst script for %s failed" % pkg sys.exit(1) pkg.install() + if pkg.run('postinst') != 0: + print "WARN: postinst script failed" except AlreadyLocked: print "You can't run more than one instance of Thandy" except LockFailed: @@ -309,4 +311,5 @@ class ScriptWrapper(object): def run(self): self._process = subprocess.Popen(["python", self._path], env=self._env) + self._process.wait() return self._process.returncode

1 0

[thandy/master] Update lockfile to work on OSX
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit c288511871e2f90016ae7782d55451cbcffe5d5e Author: Tomás Touceda <chiiph(a)torproject.org> Date: Tue Jul 26 12:47:45 2011 -0300 Update lockfile to work on OSX This changes the file modes, it seems OSX needs it this way. --- lib/thandy/lockfile/__init__.py | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 diff --git a/LICENSE b/LICENSE old mode 100644 new mode 100755 diff --git a/Makefile b/Makefile old mode 100644 new mode 100755 diff --git a/TODO b/TODO old mode 100644 new mode 100755 diff --git a/config/license-win32-bundle.cfg b/config/license-win32-bundle.cfg old mode 100644 new mode 100755 diff --git a/config/license-win32-package.cfg b/config/license-win32-package.cfg old mode 100644 new mode 100755 diff --git a/config/polipo-win32-bundle.cfg b/config/polipo-win32-bundle.cfg old mode 100644 new mode 100755 diff --git a/config/polipo-win32-package.cfg b/config/polipo-win32-package.cfg old mode 100644 new mode 100755 diff --git a/config/tor-src-bundle.cfg b/config/tor-src-bundle.cfg old mode 100644 new mode 100755 diff --git a/config/tor-src-package.cfg b/config/tor-src-package.cfg old mode 100644 new mode 100755 diff --git a/config/tor-win32-bundle.cfg b/config/tor-win32-bundle.cfg old mode 100644 new mode 100755 diff --git a/config/tor-win32-package.cfg b/config/tor-win32-package.cfg old mode 100644 new mode 100755 diff --git a/config/torbutton-win32-bundle.cfg b/config/torbutton-win32-bundle.cfg old mode 100644 new mode 100755 diff --git a/config/torbutton-win32-package.cfg b/config/torbutton-win32-package.cfg old mode 100644 new mode 100755 diff --git a/config/torvm-win32-bundle.cfg b/config/torvm-win32-bundle.cfg old mode 100644 new mode 100755 diff --git a/config/torvm-win32-package.cfg b/config/torvm-win32-package.cfg old mode 100644 new mode 100755 diff --git a/config/vidalia-marble-win32-bundle.cfg b/config/vidalia-marble-win32-bundle.cfg old mode 100644 new mode 100755 diff --git a/config/vidalia-marble-win32-package.cfg b/config/vidalia-marble-win32-package.cfg old mode 100644 new mode 100755 diff --git a/config/vidalia-win32-bundle.cfg b/config/vidalia-win32-bundle.cfg old mode 100644 new mode 100755 diff --git a/config/vidalia-win32-package.cfg b/config/vidalia-win32-package.cfg old mode 100644 new mode 100755 diff --git a/doc/HOWTO b/doc/HOWTO old mode 100644 new mode 100755 diff --git a/doc/interface.txt b/doc/interface.txt old mode 100644 new mode 100755 diff --git a/lib/thandy/ClientCLI.py b/lib/thandy/ClientCLI.py old mode 100644 new mode 100755 diff --git a/lib/thandy/ServerCLI.py b/lib/thandy/ServerCLI.py old mode 100644 new mode 100755 diff --git a/lib/thandy/SignerCLI.py b/lib/thandy/SignerCLI.py old mode 100644 new mode 100755 diff --git a/lib/thandy/__init__.py b/lib/thandy/__init__.py old mode 100644 new mode 100755 diff --git a/lib/thandy/bt_compat.py b/lib/thandy/bt_compat.py old mode 100644 new mode 100755 diff --git a/lib/thandy/checkJson.py b/lib/thandy/checkJson.py old mode 100644 new mode 100755 diff --git a/lib/thandy/download.py b/lib/thandy/download.py old mode 100644 new mode 100755 diff --git a/lib/thandy/encodeToXML.py b/lib/thandy/encodeToXML.py old mode 100644 new mode 100755 diff --git a/lib/thandy/formats.py b/lib/thandy/formats.py old mode 100644 new mode 100755 diff --git a/lib/thandy/keys.py b/lib/thandy/keys.py old mode 100644 new mode 100755 diff --git a/lib/thandy/lockfile/__init__.py b/lib/thandy/lockfile/__init__.py old mode 100644 new mode 100755 index a167cd8..6ed31d5 --- a/lib/thandy/lockfile/__init__.py +++ b/lib/thandy/lockfile/__init__.py @@ -168,6 +168,8 @@ class LockBase: # Thread objects in Python 2.4 and earlier do not have ident # attrs. Worm around that. ident = getattr(t, "ident", hash(t)) + if ident is None: + ident = hash(t) self.tname = "-%x" % (ident & 0xffffffff) else: self.tname = "" diff --git a/lib/thandy/lockfile/linklockfile.py b/lib/thandy/lockfile/linklockfile.py old mode 100644 new mode 100755 diff --git a/lib/thandy/lockfile/mkdirlockfile.py b/lib/thandy/lockfile/mkdirlockfile.py old mode 100644 new mode 100755 diff --git a/lib/thandy/lockfile/pidlockfile.py b/lib/thandy/lockfile/pidlockfile.py old mode 100644 new mode 100755 diff --git a/lib/thandy/lockfile/sqlitelockfile.py b/lib/thandy/lockfile/sqlitelockfile.py old mode 100644 new mode 100755 diff --git a/lib/thandy/master_keys.py b/lib/thandy/master_keys.py old mode 100644 new mode 100755 diff --git a/lib/thandy/packagesys/ExePackages.py b/lib/thandy/packagesys/ExePackages.py old mode 100644 new mode 100755 diff --git a/lib/thandy/packagesys/PackageDB.py b/lib/thandy/packagesys/PackageDB.py old mode 100644 new mode 100755 diff --git a/lib/thandy/packagesys/PackageSystem.py b/lib/thandy/packagesys/PackageSystem.py old mode 100644 new mode 100755 diff --git a/lib/thandy/packagesys/RPMPackages.py b/lib/thandy/packagesys/RPMPackages.py old mode 100644 new mode 100755 diff --git a/lib/thandy/packagesys/ThpPackages.py b/lib/thandy/packagesys/ThpPackages.py old mode 100644 new mode 100755 diff --git a/lib/thandy/packagesys/__init__.py b/lib/thandy/packagesys/__init__.py old mode 100644 new mode 100755 diff --git a/lib/thandy/repository.py b/lib/thandy/repository.py old mode 100644 new mode 100755 diff --git a/lib/thandy/socksurls.py b/lib/thandy/socksurls.py old mode 100644 new mode 100755 diff --git a/lib/thandy/tests.py b/lib/thandy/tests.py old mode 100644 new mode 100755 diff --git a/lib/thandy/util.py b/lib/thandy/util.py old mode 100644 new mode 100755 diff --git a/samples/example-bundle.cfg b/samples/example-bundle.cfg old mode 100644 new mode 100755 diff --git a/samples/example-mirrors.txt b/samples/example-mirrors.txt old mode 100644 new mode 100755 diff --git a/samples/example-package.cfg b/samples/example-package.cfg old mode 100644 new mode 100755 diff --git a/samples/example-thp-package.cfg b/samples/example-thp-package.cfg old mode 100644 new mode 100755 diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 diff --git a/specs/thandy-spec.txt b/specs/thandy-spec.txt old mode 100644 new mode 100755

1 0

[thandy/master] Run transactions only if --install is given, and create pkg-status if it doesn't exist
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit 40b3a86a82054c88f5865cef9b8d7fff8486f768 Author: Tomás Touceda <chiiph(a)torproject.org> Date: Fri Sep 2 19:26:42 2011 -0300 Run transactions only if --install is given, and create pkg-status if it doesn't exist --- lib/thandy/ClientCLI.py | 5 +++-- lib/thandy/packagesys/ThpPackages.py | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/thandy/ClientCLI.py b/lib/thandy/ClientCLI.py index 75aa347..2987918 100755 --- a/lib/thandy/ClientCLI.py +++ b/lib/thandy/ClientCLI.py @@ -156,8 +156,9 @@ def update(args): i.install() for bundle in thpTransactions: - thandy.packagesys.ThpPackages.ThpTransaction(thpTransactions[bundle], - repoRoot).install() + if install: + thandy.packagesys.ThpPackages.ThpTransaction(thpTransactions[bundle], + repoRoot).install() return diff --git a/lib/thandy/packagesys/ThpPackages.py b/lib/thandy/packagesys/ThpPackages.py index 090fe55..f359957 100755 --- a/lib/thandy/packagesys/ThpPackages.py +++ b/lib/thandy/packagesys/ThpPackages.py @@ -24,6 +24,9 @@ class ThpDB(object): self._thp_db_root = os.environ.get("THP_DB_ROOT") if self._thp_db_root is None: raise Exception("There is no THP_DB_ROOT variable set") + dbpath = os.path.join(self._thp_db_root, "pkg-status") + if not os.path.exists(dbpath): + os.mkdir(dbpat) def getPath(self): return self._thp_db_root

1 0

[thandy/master] Fix behavior for bundles with more than one thp and add thp
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit f04c3429dc33c9bcadfe8a3707f8646690838db8 Author: Tomás Touceda <chiiph(a)torproject.org> Date: Fri Sep 2 20:44:53 2011 -0300 Fix behavior for bundles with more than one thp and add thp --- lib/thandy/formats.py | 9 +++++++-- lib/thandy/packagesys/ThpPackages.py | 2 +- lib/thandy/repository.py | 4 ++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/thandy/formats.py b/lib/thandy/formats.py index f57ab46..0a73ff4 100755 --- a/lib/thandy/formats.py +++ b/lib/thandy/formats.py @@ -746,6 +746,7 @@ def makePackageObj(config_fname, package_fname): 'exe_registry_ent', 'db_key', 'db_val', 'command_install', 'command_remove', + 'thp_name', 'thp_version', ], preload) f = open(package_fname, 'rb') @@ -783,8 +784,12 @@ def makePackageObj(config_fname, package_fname): elif format == 'thp': extra['check_type'] = 'thp' extra['install_type'] = 'thp' - extra['thp_name'] = r['name'] - extra['thp_version'] = r['version'] + if not r.get('thp_name'): + raise thandy.FormatException("missing thp_name value") + extra['thp_name'] = r['thp_name'] + if not r.get('thp_version'): + raise thandy.FormatException("missing thp_version value") + extra['thp_version'] = r['thp_version'] if r.get('command_install'): extra['install_type'] = 'command' diff --git a/lib/thandy/packagesys/ThpPackages.py b/lib/thandy/packagesys/ThpPackages.py index f359957..493a94a 100755 --- a/lib/thandy/packagesys/ThpPackages.py +++ b/lib/thandy/packagesys/ThpPackages.py @@ -26,7 +26,7 @@ class ThpDB(object): raise Exception("There is no THP_DB_ROOT variable set") dbpath = os.path.join(self._thp_db_root, "pkg-status") if not os.path.exists(dbpath): - os.mkdir(dbpat) + os.mkdir(dbpath) def getPath(self): return self._thp_db_root diff --git a/lib/thandy/repository.py b/lib/thandy/repository.py index bb7d2f6..20a1e9a 100755 --- a/lib/thandy/repository.py +++ b/lib/thandy/repository.py @@ -493,9 +493,9 @@ class LocalRepository: packages[rp] = pfile if pfile_data["format"] == "thp": - thpTransactionDict[bundle['name']] = {} + if not bundle['name'] in thpTransactionDict.keys(): + thpTransactionDict[bundle['name']] = {} thpTransactionDict[bundle['name']][pfile_data['name']] = pfile_data - continue # We have the packages. If we're downloading via bittorrent, we need # the .torrent metafiles, as well.

1 0

[thandy/master] Log with loggin instead of print, and add thp files to download when needed
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit f3602b42ad6db5918323c3b3be1c0d4c335e9c61 Author: Tomás Touceda <chiiph(a)torproject.org> Date: Thu Sep 1 23:50:54 2011 -0300 Log with loggin instead of print, and add thp files to download when needed --- lib/thandy/packagesys/PackageSystem.py | 9 +++++---- lib/thandy/packagesys/ThpPackages.py | 30 +++++++++++++++--------------- lib/thandy/repository.py | 7 ++++--- 3 files changed, 24 insertions(+), 22 deletions(-) diff --git a/lib/thandy/packagesys/PackageSystem.py b/lib/thandy/packagesys/PackageSystem.py index d1060d0..9fb878f 100755 --- a/lib/thandy/packagesys/PackageSystem.py +++ b/lib/thandy/packagesys/PackageSystem.py @@ -18,8 +18,11 @@ def getItemsFromPackage(pkg): extra = {} checker = getChecker(relPath, extra, defaultFormat=format, package=pkg) - installer = getInstaller(relPath, extra, - defaultFormat=format, package=pkg) + # we don't need this + # installer = getInstaller(relPath, extra, + # defaultFormat=format, package=pkg) + installer = None + result[relPath] = PackageItem(relPath, checker, installer) return result @@ -70,7 +73,6 @@ def getChecker(relPath, extra, defaultFormat, package): k,v=extra['registry_ent'] return thandy.packagesys.ExePackages.RegistryChecker(k,v) elif checkType == 'thp': - print "Getting thp checker" import thandy.packagesys.ThpPackages return thandy.packagesys.ThpPackages.ThpChecker( extra['thp_name'], extra['thp_version']) @@ -109,7 +111,6 @@ def getInstaller(relPath, extra, defaultFormat, package): installer = thandy.packagesys.ExePackages.CommandInstaller( relPath, extra['cmd_install'], extra.get('cmd_remove')) elif installType == 'thp': - print "Getting thp installer" import thandy.packagesys.ThpPackages installer = thandy.packagesys.ThpPackages.ThpInstaller( relPath) diff --git a/lib/thandy/packagesys/ThpPackages.py b/lib/thandy/packagesys/ThpPackages.py index 0579b5c..090fe55 100755 --- a/lib/thandy/packagesys/ThpPackages.py +++ b/lib/thandy/packagesys/ThpPackages.py @@ -124,22 +124,22 @@ class ThpTransaction(object): lockfile = os.path.join(self._db.getPath(), "db") lock = LockFile(lockfile) try: - print "Acquiring lock..." + logging.info("Acquiring lock...") lock.acquire() - print "Lock acquired" + logging.info("Lock acquired") order = self._orderByDep() for pkg in order: if pkg.run('checkinst') != 0: - print "Check inst failed for %s" % pkg + logging.info("Check inst failed for %s" % pkg) sys.exit(1) for pkg in order: - print "Starting installation using", pkg + logging.info("Starting installation using %s" % pkg) if pkg.run('preinst') != 0: - print "Preinst script for %s failed" % pkg + logging.info("Preinst script for %s failed" % pkg) sys.exit(1) pkg.install() if pkg.run('postinst') != 0: - print "WARN: postinst script failed" + logging.info("postinst script failed") except AlreadyLocked: print "You can't run more than one instance of Thandy" except LockFailed: @@ -163,16 +163,16 @@ class ThpInstaller(PS.Installer): return "ThpInstaller(%r)" %(self._relPath) def install(self): - print "Running thp installer", self._cacheRoot, self._relPath + logging.info("Running thp installer %s %s" % (self._cacheRoot, self._relPath)) self._thp_root = os.environ.get("THP_INSTALL_ROOT") if self._thp_root is None: raise Exception("There is no THP_INSTALL_ROOT variable set") destPath = os.path.join(self._thp_root, self._pkg.get("package_name")) - print "Destination directory:", destPath + logging.info("Destination directory: %s" % destPath) if self._db.exists(self._pkg.get("package_name")): - print "%s is already installed, switching to upgrade mode." % self._pkg.get("package_name") + logging.info("%s is already installed, switching to upgrade mode." % self._pkg.get("package_name")) self._db.startUpgrade() pkg_metadata = self._pkg.getAll() @@ -183,13 +183,13 @@ class ThpInstaller(PS.Installer): try: os.mkdir(dir) except: - print "%s: Already exists, using it." % dir + logging.info("%s: Already exists, using it." % dir) for file in self._pkg.get('manifest'): if file['is_config']: - print "Ignoring file:", file + logging.info("Ignoring file: %s" % file) else: - print "Processing file:", file + logging.info("Processing file: %s" % file) try: # Create all the needed dirs os.makedirs(os.sep.join((os.path.join(destPath, file['name']) @@ -201,7 +201,7 @@ class ThpInstaller(PS.Installer): os.path.join(destPath, file['name'])); if self._db.isUpgrading(): - print "Finishing upgrade." + logging.info("Finishing upgrade.") self._db.finishUpgrade(self._pkg.get('package_name')) self._db.statusInstalled(pkg_metadata) @@ -243,8 +243,8 @@ class ThpPackage(object): (allOk, where) = self._validateFiles(self._tmp_path) if not allOk: - print "These files have different digests:" - print where + logging.info("These files have different digests:") + logging.info(where) sys.exit(1) if "scripts" in self._metadata: diff --git a/lib/thandy/repository.py b/lib/thandy/repository.py index f0b51b8..bb7d2f6 100755 --- a/lib/thandy/repository.py +++ b/lib/thandy/repository.py @@ -489,13 +489,14 @@ class LocalRepository: # are thp too. But we continue with the loop to check every # package digest and signature pfile_data = pfile.get() + + packages[rp] = pfile + if pfile_data["format"] == "thp": thpTransactionDict[bundle['name']] = {} thpTransactionDict[bundle['name']][pfile_data['name']] = pfile_data continue - packages[rp] = pfile - # We have the packages. If we're downloading via bittorrent, we need # the .torrent metafiles, as well. if thandy.bt_compat.BtCompat.shouldUseBt(): @@ -524,7 +525,7 @@ class LocalRepository: # files? for pfile in packages.values(): package = pfile.get() - + alreadyInstalled = set() pkgItems = {}

1 0

[thandy/master] Improve re-install behavior
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit 7d87a55747e82c7b23daa48581135c35b553aeaf Author: Tomás Touceda <chiiph(a)torproject.org> Date: Sat Sep 3 21:58:47 2011 -0300 Improve re-install behavior Also add more controller oriented output --- lib/thandy/ClientCLI.py | 41 +++++++++++++++++++--------------- lib/thandy/download.py | 3 ++ lib/thandy/packagesys/ThpPackages.py | 37 ++++++++++++++++++++++-------- lib/thandy/repository.py | 15 +++++++++--- 4 files changed, 64 insertions(+), 32 deletions(-) diff --git a/lib/thandy/ClientCLI.py b/lib/thandy/ClientCLI.py index 2987918..f568a2c 100755 --- a/lib/thandy/ClientCLI.py +++ b/lib/thandy/ClientCLI.py @@ -122,6 +122,7 @@ def update(args): installable = {} btMetadata = {} thpTransactions = {} + alreadyInstalled = set() logging.info("Checking for files to update.") files, downloadingFiles = repo.getFilesToUpdate( trackingBundles=args, @@ -130,34 +131,38 @@ def update(args): usePackageSystem=use_packagesys, installableDict=installable, btMetadataDict=btMetadata, - thpTransactionDict=thpTransactions) + thpTransactionDict=thpTransactions, + alreadyInstalledSet=alreadyInstalled) if forceCheck: files.add("/meta/timestamp.txt") forceCheck = False if (thpTransactions or installable) and not files: - for p, d in installable.items(): - for n, i in d.items(): - if i.canInstall(): - logCtrl("CAN_INSTALL", PKG=p, ITEM=n) - else: - logCtrl("NO_INSTALL", PKG=p, ITEM=n) - i.setCacheRoot(repoRoot) - - logging.info("Ready to install packages for files: %s", - ", ".join(sorted(installable.keys()))) - if install: - # XXXX handle ordering - for p in installable.values(): - for h in p.values(): - i = h.getInstaller() - if i != None: - i.install() + # for p, d in installable.items(): + # for n, i in d.items(): + # if i.canInstall(): + # logCtrl("CAN_INSTALL", PKG=p, ITEM=n) + # else: + # logCtrl("NO_INSTALL", PKG=p, ITEM=n) + # i.setCacheRoot(repoRoot) + + # logging.info("Ready to install packages for files: %s", + # ", ".join(sorted(installable.keys()))) + # if install: + # # XXXX handle ordering + # for p in installable.values(): + # for h in p.values(): + # i = h.getInstaller() + # if i != None: + # i.install() + + logCtrl("READY", BUNDLE=",".join(thpTransactions.keys())) for bundle in thpTransactions: if install: thandy.packagesys.ThpPackages.ThpTransaction(thpTransactions[bundle], + alreadyInstalled, repoRoot).install() return diff --git a/lib/thandy/download.py b/lib/thandy/download.py index 52c6fe4..c2a9e61 100755 --- a/lib/thandy/download.py +++ b/lib/thandy/download.py @@ -13,6 +13,8 @@ import thandy.util import thandy.socksurls import thandy.checkJson +from thandy.util import logCtrl + class BadCompoundData(thandy.DownloadError): """DOCDOC""" pass @@ -514,6 +516,7 @@ class DownloadJob: total += len(c) logging.debug("Got %s/%s bytes from %s", total, expectLength, url) + logCtrl("DOWNLOAD", TOTAL=str(total), EXPECT=str(expectLength), URL=url) if self._wantLength != None and total > self._wantLength: logging.warn("Read too many bytes from %s; got %s, but " "wanted %s", url, total, self._wantLength) diff --git a/lib/thandy/packagesys/ThpPackages.py b/lib/thandy/packagesys/ThpPackages.py index 493a94a..dd1d399 100755 --- a/lib/thandy/packagesys/ThpPackages.py +++ b/lib/thandy/packagesys/ThpPackages.py @@ -70,7 +70,17 @@ class ThpDB(object): contents = open(fname, "r").read() metadata = json.loads(contents) version = metadata['package_version'] - return fexists, version + + fname = os.path.join(self._thp_db_root, "pkg-status", name+".status") + fexists2 = os.path.exists(fname) + + status = "" + if fexists2: + contents = open(fname, "r").read() + metadata = json.loads(contents) + status = metadata['status'] + + return fexists, version, status def statusInProgress(self, pkg): thandy.util.replaceFile(os.path.join(self._thp_db_root, "pkg-status", @@ -94,30 +104,35 @@ class ThpChecker(PS.Checker): def getInstalledVersions(self): versions = [] - (exists, version) = self._db.exists(self._name) + (exists, version, status) = self._db.exists(self._name) if exists: versions.append(version) - return versions + return versions, status def isInstalled(self): - return self._version in self.getInstalledVersions() + versions, status = self.getInstalledVersions() + # if status = IN_PROGRESS a previous installation failed + # we need to reinstall + return (status != "IN_PROGRESS" and self._version in versions) class ThpTransaction(object): - def __init__(self, packages, repoRoot): + def __init__(self, packages, alreadyInstalled, repoRoot): self._raw_packages = packages self._repo_root = repoRoot self._installers = [] + self._alreadyInstalled = alreadyInstalled self._db = ThpDB() self._process() def _process(self): for package in self._raw_packages.keys(): - self._installers.append(ThpInstaller(self._raw_packages[package]['files'][0][0], - self._db, - self._repo_root)) + if not (self._raw_packages[package]['files'][0][0] in self._alreadyInstalled): + self._installers.append(ThpInstaller(self._raw_packages[package]['files'][0][0], + self._db, + self._repo_root)) def _orderByDep(self): """ Orders packages with a topological order by its dependencies """ @@ -148,6 +163,7 @@ class ThpTransaction(object): except LockFailed: print "Can't acquire lock on %s" % lockfile finally: + logging.info("Releasing lock...") lock.release() def remote(self): @@ -174,10 +190,11 @@ class ThpInstaller(PS.Installer): destPath = os.path.join(self._thp_root, self._pkg.get("package_name")) logging.info("Destination directory: %s" % destPath) - if self._db.exists(self._pkg.get("package_name")): + (exists, _, _) = self._db.exists(self._pkg.get("package_name")) + if exists: logging.info("%s is already installed, switching to upgrade mode." % self._pkg.get("package_name")) self._db.startUpgrade() - + pkg_metadata = self._pkg.getAll() self._db.insert(pkg_metadata) self._db.statusInProgress(pkg_metadata) diff --git a/lib/thandy/repository.py b/lib/thandy/repository.py index 20a1e9a..0b45c8d 100755 --- a/lib/thandy/repository.py +++ b/lib/thandy/repository.py @@ -5,6 +5,8 @@ import thandy.util import thandy.packagesys.PackageSystem import thandy.bt_compat +from thandy.util import logCtrl + json = thandy.util.importJSON() import logging @@ -300,7 +302,7 @@ class LocalRepository: def getFilesToUpdate(self, now=None, trackingBundles=(), hashDict=None, lengthDict=None, usePackageSystem=True, installableDict=None, btMetadataDict=None, - thpTransactionDict=None): + thpTransactionDict=None, alreadyInstalledSet=None): """Return a set of relative paths for all files that we need to fetch, and True if we're fetching actual files to install instead of metadata. Assumes that we care about the bundles @@ -331,6 +333,9 @@ class LocalRepository: need = set() + if alreadyInstalledSet == None: + alreadyInstalledSet = set() + # Fetch missing metafiles. for f in self._metaFiles: try: @@ -526,7 +531,6 @@ class LocalRepository: for pfile in packages.values(): package = pfile.get() - alreadyInstalled = set() pkgItems = {} if usePackageSystem: @@ -541,7 +545,9 @@ class LocalRepository: else: try: if item.getChecker().isInstalled(): - alreadyInstalled.add(item.getRelativePath()) + alreadyInstalledSet.add(item.getRelativePath()) + if item.getRelativePath() in thpTransactionDict.keys(): + thpTransactionDict.pop(item.getRelativePath()) except thandy.CheckNotSupported, err: logging.warn("Can't check installed-ness of %s: %s", f[0], err) @@ -550,9 +556,10 @@ class LocalRepository: for f in package['files']: rp, h = f[:2] - if rp in alreadyInstalled: + if rp in alreadyInstalledSet: logging.info("%s is already installed; no need to download", rp) + logCtrl("ALREADY", PKG=rp) continue h_expected = thandy.formats.parseHash(h)

1 0