November 2011 - tor-commits - lists.torproject.org

[thandy/master] Add a WAIT-FOR logCtrl message
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit 2402546ac7cd26fd404af94b09e93907e8b85837 Author: Tomás Touceda <chiiph(a)torproject.org> Date: Sat Sep 3 23:19:37 2011 -0300 Add a WAIT-FOR logCtrl message --- lib/thandy/ClientCLI.py | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/lib/thandy/ClientCLI.py b/lib/thandy/ClientCLI.py index aa60205..f3192f9 100755 --- a/lib/thandy/ClientCLI.py +++ b/lib/thandy/ClientCLI.py @@ -185,6 +185,7 @@ def update(args): delay = int(waitTill - now) + 1 logging.info("Waiting another %s seconds before we are willing " "to retry any mirror.", delay) + logCtrl("WAIT", FOR="MIRROR") time.sleep(delay) continue

1 0

[thandy/master] Adds README, RELEASE-NOTES and ACKS files for lockfile
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit 0bdd881ffb8dc1923c1b308e4f8557b4a46393f0 Author: Tomás Touceda <chiiph(a)torproject.org> Date: Wed Sep 7 11:30:58 2011 -0300 Adds README, RELEASE-NOTES and ACKS files for lockfile --- lib/thandy/lockfile/ACKS | 6 ++++ lib/thandy/lockfile/README | 27 ++++++++++++++++++++ lib/thandy/lockfile/RELEASE-NOTES | 50 +++++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 0 deletions(-) diff --git a/lib/thandy/lockfile/ACKS b/lib/thandy/lockfile/ACKS new file mode 100644 index 0000000..44519d1 --- /dev/null +++ b/lib/thandy/lockfile/ACKS @@ -0,0 +1,6 @@ +Thanks to the following people for help with lockfile. + + Scott Dial + Ben Finney + Frank Niessink + Konstantin Veretennicov diff --git a/lib/thandy/lockfile/README b/lib/thandy/lockfile/README new file mode 100644 index 0000000..128b1f3 --- /dev/null +++ b/lib/thandy/lockfile/README @@ -0,0 +1,27 @@ +The lockfile package exports a LockFile class which provides a simple API for +locking files. Unlike the Windows msvcrt.locking function, the fcntl.lockf +and flock functions, and the deprecated posixfile module, the API is +identical across both Unix (including Linux and Mac) and Windows platforms. +The lock mechanism relies on the atomic nature of the link (on Unix) and +mkdir (on Windows) system calls. An implementation based on SQLite is also +provided, more as a demonstration of the possibilities it provides than as +production-quality code. + +Note: In version 0.9 the API changed in two significant ways: + + * It changed from a module defining several classes to a package containing + several modules, each defining a single class. + + * Where classes had been named SomethingFileLock before the last two words + have been reversed, so that class is now SomethingLockFile. + +The previous module-level definitions of LinkFileLock, MkdirFileLock and +SQLiteFileLock will be retained until the 1.0 release. + +Download from: + + http://code.google.com/p/pylockfile/ + +To install: + + python setup.py install diff --git a/lib/thandy/lockfile/RELEASE-NOTES b/lib/thandy/lockfile/RELEASE-NOTES new file mode 100644 index 0000000..8b452ed --- /dev/null +++ b/lib/thandy/lockfile/RELEASE-NOTES @@ -0,0 +1,50 @@ +Version 0.9.1 +============= + +* This release moves the source location to Google Code. + +* Threaded support is currently broken. (It might not actually be broken. + It might just be the tests which are broken.) + +Version 0.9 +=========== + +* The lockfile module was reorganized into a package. + +* The names of the three main classes have changed as follows: + + LinkFileLock -> LinkLockFile + MkdirFileLock -> MkdirLockFile + SQLiteFileLock -> SQLiteLockFile + +* A PIDLockFile class was added. + +Version 0.3 +=========== + +* Fix 2.4.diff file error. + +* More documentation updates. + +Version 0.2 +=========== + +* Added 2.4.diff file to patch lockfile to work with Python 2.4 (removes use + of with statement). + +* Renamed _FileLock base class to LockBase to expose it (and its docstrings) + to pydoc. + +* Got rid of time.sleep() calls in tests (thanks to Konstantin + Veretennicov). + +* Use thread.get_ident() as the thread discriminator. + +* Updated documentation a bit. + +* Added RELEASE-NOTES. + +Version 0.1 +=========== + +* First release - All basic functionality there.

1 0

[thandy/master] Handle Thp package creation in a separate CLI from Signer
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit 48cdba812c660fccd16052f7f2119b867a736ec2 Author: Tomás Touceda <chiiph(a)torproject.org> Date: Thu Oct 20 09:54:30 2011 -0300 Handle Thp package creation in a separate CLI from Signer --- lib/thandy/SignerCLI.py | 85 +----------------------------------- lib/thandy/ThpCLI.py | 111 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 83 deletions(-) diff --git a/lib/thandy/SignerCLI.py b/lib/thandy/SignerCLI.py index fd22c69..06536aa 100644 --- a/lib/thandy/SignerCLI.py +++ b/lib/thandy/SignerCLI.py @@ -3,10 +3,6 @@ import os import getopt import sys -import tempfile -import time -import shutil -import zipfile import thandy.keys import thandy.formats @@ -97,81 +93,6 @@ def makepackage(args): f.write(metaFile) f.close() -def makethppackage(args): - options, args = getopt.getopt(args, "", "keyid=") - keyid = None - scriptsPath = None - for o,v in options: - if o == "--keyid": - keyid = v - - if len(args) < 3: - usage() - - tmpPath = tempfile.mkdtemp(suffix=str(time.time()), - prefix="thp") - - print "Using temporary directory: %s" % tmpPath - - configFile = args[0] - dataPath = args[1] - thpPath = args[2] - if len(args) > 3: - scriptsPath = args[3] - - print "Generating package metadata..." - metadata = thandy.formats.makeThpPackageObj(configFile, dataPath) - - print "Generating directory structure..." - try: - os.mkdir(os.path.join(tmpPath, "meta")); - except Exception as e: - print e - thandy.util.deltree(tmpPath) - sys.exit(1) - - thandy.util.replaceFile(os.path.join(tmpPath, "meta", "package.json"), - json.dumps(metadata, indent=3)) - - shutil.copytree(dataPath, os.path.join(tmpPath, "content")) - - if "scripts" in metadata: - try: - os.mkdir(os.path.join(tmpPath, "meta", "scripts")) - except Exception as e: - print e - thandy.util.deltree(tmpPath) - sys.exit(1) - for lang in metadata["scripts"]: - for script in metadata['scripts'][lang]: - shutil.copyfile(os.path.join(scriptsPath, script[0]), - os.path.join(tmpPath, "meta", "scripts", script[0])) - - thpFileName = "%s-%s.thp" % (metadata['package_name'], - metadata['package_version']) - - print "Generating thp file in %s" % thpFileName - thpFile = zipfile.ZipFile(os.path.join(thpPath, - thpFileName), "w") - - for file in metadata['manifest']: - thpFile.write(os.path.join(tmpPath, "content", file['name']), - os.path.join("content", file['name'])) - - if "scripts" in metadata: - for lang in metadata["scripts"]: - for script in metadata['scripts'][lang]: - thpFile.write(os.path.join(tmpPath, "meta", "scripts", script[0]), - os.path.join("meta", "scripts", script[0])) - - thpFile.write(os.path.join(tmpPath, "meta", "package.json"), - os.path.join("meta", "package.json")) - - thpFile.close() - - print "All done. Cleaning tmp directory..." - thandy.util.deltree(tmpPath) - def makebundle(args): options, args = getopt.getopt(args, "", "keyid=") keyid = None @@ -386,7 +307,6 @@ def usage(): print " delrole keyid role path" print " dumpkey [--include-secret] keyid" print " makepackage config datafile" - print " makethppackage config datapath thpPath scriptsPath" print " makebundle config packagefile ..." print " signkeylist keylist" print " makekeylist keylist" @@ -399,9 +319,8 @@ def main(): cmd = sys.argv[1] args = sys.argv[2:] if cmd in [ "keygen", "listkeys", "addrole", "delrole", "chpass", - "dumpkey", "makepackage", "makebundle", "makethppackage", - "signkeylist", "makekeylist", "signkeylist", - "makemirrorlist", ]: + "dumpkey", "makepackage", "makebundle", "signkeylist", + "makekeylist", "signkeylist", "makemirrorlist", ]: try: globals()[cmd](args) except thandy.BadPassword: diff --git a/lib/thandy/ThpCLI.py b/lib/thandy/ThpCLI.py new file mode 100644 index 0000000..52938d6 --- /dev/null +++ b/lib/thandy/ThpCLI.py @@ -0,0 +1,111 @@ +# Copyright 2011 The Tor Project, Inc. See LICENSE for licensing information. + +import sys +import os +import getopt +import tempfile +import time +import shutil +import zipfile + +import thandy.keys +import thandy.util +import thandy.formats + +json = thandy.util.importJSON() + +def makethppackage(args): + options, args = getopt.getopt(args, "", "keyid=") + keyid = None + scriptsPath = None + for o,v in options: + if o == "--keyid": + keyid = v + + if len(args) < 3: + usage() + + tmpPath = tempfile.mkdtemp(suffix=str(time.time()), + prefix="thp") + + print "Using temporary directory: %s" % tmpPath + + configFile = args[0] + dataPath = args[1] + thpPath = args[2] + if len(args) > 3: + scriptsPath = args[3] + + print "Generating package metadata..." + metadata = thandy.formats.makeThpPackageObj(configFile, dataPath) + + print "Generating directory structure..." + try: + os.mkdir(os.path.join(tmpPath, "meta")); + except Exception as e: + print e + thandy.util.deltree(tmpPath) + sys.exit(1) + + thandy.util.replaceFile(os.path.join(tmpPath, "meta", "package.json"), + json.dumps(metadata, indent=3)) + + shutil.copytree(dataPath, os.path.join(tmpPath, "content")) + + if "scripts" in metadata: + try: + os.mkdir(os.path.join(tmpPath, "meta", "scripts")) + except Exception as e: + print e + thandy.util.deltree(tmpPath) + sys.exit(1) + for lang in metadata["scripts"]: + for script in metadata['scripts'][lang]: + shutil.copyfile(os.path.join(scriptsPath, script[0]), + os.path.join(tmpPath, "meta", "scripts", script[0])) + + thpFileName = "%s-%s.thp" % (metadata['package_name'], + metadata['package_version']) + + print "Generating thp file in %s" % thpFileName + thpFile = zipfile.ZipFile(os.path.join(thpPath, + thpFileName), "w") + + for file in metadata['manifest']: + thpFile.write(os.path.join(tmpPath, "content", file['name']), + os.path.join("content", file['name'])) + + if "scripts" in metadata: + for lang in metadata["scripts"]: + for script in metadata['scripts'][lang]: + thpFile.write(os.path.join(tmpPath, "meta", "scripts", script[0]), + os.path.join("meta", "scripts", script[0])) + + thpFile.write(os.path.join(tmpPath, "meta", "package.json"), + os.path.join("meta", "package.json")) + + thpFile.close() + + print "All done. Cleaning tmp directory..." + thandy.util.deltree(tmpPath) + +def usage(): + print "Known commands:" + print " makethppackage config datapath thpPath scriptsPath" + sys.exit(1) + +def main(): + if len(sys.argv) < 2: + usage() + cmd = sys.argv[1] + args = sys.argv[2:] + if cmd in [ "makethppackage", ]: + try: + globals()[cmd](args) + except thandy.BadPassword: + print >>sys.stderr, "Password incorrect." + else: + usage() + +if __name__ == '__main__': + main()

1 0

[thandy/master] Fix file permissions
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit 2d5be8cb7b4fe527df86751bb3e1f27d8cf9cd65 Author: Tomás Touceda <chiiph(a)torproject.org> Date: Thu Oct 20 09:30:35 2011 -0300 Fix file permissions --- 0 files changed, 0 insertions(+), 0 deletions(-) diff --git a/.gitignore b/.gitignore old mode 100755 new mode 100644 diff --git a/LICENSE b/LICENSE old mode 100755 new mode 100644 diff --git a/Makefile b/Makefile old mode 100755 new mode 100644 diff --git a/TODO b/TODO old mode 100755 new mode 100644 diff --git a/config/license-win32-bundle.cfg b/config/license-win32-bundle.cfg old mode 100755 new mode 100644 diff --git a/config/license-win32-package.cfg b/config/license-win32-package.cfg old mode 100755 new mode 100644 diff --git a/config/polipo-win32-bundle.cfg b/config/polipo-win32-bundle.cfg old mode 100755 new mode 100644 diff --git a/config/polipo-win32-package.cfg b/config/polipo-win32-package.cfg old mode 100755 new mode 100644 diff --git a/config/tor-src-bundle.cfg b/config/tor-src-bundle.cfg old mode 100755 new mode 100644 diff --git a/config/tor-src-package.cfg b/config/tor-src-package.cfg old mode 100755 new mode 100644 diff --git a/config/tor-win32-bundle.cfg b/config/tor-win32-bundle.cfg old mode 100755 new mode 100644 diff --git a/config/tor-win32-package.cfg b/config/tor-win32-package.cfg old mode 100755 new mode 100644 diff --git a/config/torbutton-win32-bundle.cfg b/config/torbutton-win32-bundle.cfg old mode 100755 new mode 100644 diff --git a/config/torbutton-win32-package.cfg b/config/torbutton-win32-package.cfg old mode 100755 new mode 100644 diff --git a/config/torvm-win32-bundle.cfg b/config/torvm-win32-bundle.cfg old mode 100755 new mode 100644 diff --git a/config/torvm-win32-package.cfg b/config/torvm-win32-package.cfg old mode 100755 new mode 100644 diff --git a/config/vidalia-marble-win32-bundle.cfg b/config/vidalia-marble-win32-bundle.cfg old mode 100755 new mode 100644 diff --git a/config/vidalia-marble-win32-package.cfg b/config/vidalia-marble-win32-package.cfg old mode 100755 new mode 100644 diff --git a/config/vidalia-win32-bundle.cfg b/config/vidalia-win32-bundle.cfg old mode 100755 new mode 100644 diff --git a/config/vidalia-win32-package.cfg b/config/vidalia-win32-package.cfg old mode 100755 new mode 100644 diff --git a/doc/HOWTO b/doc/HOWTO old mode 100755 new mode 100644 diff --git a/doc/interface.txt b/doc/interface.txt old mode 100755 new mode 100644 diff --git a/lib/thandy/ClientCLI.py b/lib/thandy/ClientCLI.py old mode 100755 new mode 100644 diff --git a/lib/thandy/ServerCLI.py b/lib/thandy/ServerCLI.py old mode 100755 new mode 100644 diff --git a/lib/thandy/SignerCLI.py b/lib/thandy/SignerCLI.py old mode 100755 new mode 100644 diff --git a/lib/thandy/__init__.py b/lib/thandy/__init__.py old mode 100755 new mode 100644 diff --git a/lib/thandy/bt_compat.py b/lib/thandy/bt_compat.py old mode 100755 new mode 100644 diff --git a/lib/thandy/checkJson.py b/lib/thandy/checkJson.py old mode 100755 new mode 100644 diff --git a/lib/thandy/download.py b/lib/thandy/download.py old mode 100755 new mode 100644 diff --git a/lib/thandy/encodeToXML.py b/lib/thandy/encodeToXML.py old mode 100755 new mode 100644 diff --git a/lib/thandy/formats.py b/lib/thandy/formats.py old mode 100755 new mode 100644 diff --git a/lib/thandy/keys.py b/lib/thandy/keys.py old mode 100755 new mode 100644 diff --git a/lib/thandy/lockfile/__init__.py b/lib/thandy/lockfile/__init__.py old mode 100755 new mode 100644 diff --git a/lib/thandy/lockfile/linklockfile.py b/lib/thandy/lockfile/linklockfile.py old mode 100755 new mode 100644 diff --git a/lib/thandy/lockfile/mkdirlockfile.py b/lib/thandy/lockfile/mkdirlockfile.py old mode 100755 new mode 100644 diff --git a/lib/thandy/lockfile/pidlockfile.py b/lib/thandy/lockfile/pidlockfile.py old mode 100755 new mode 100644 diff --git a/lib/thandy/lockfile/sqlitelockfile.py b/lib/thandy/lockfile/sqlitelockfile.py old mode 100755 new mode 100644 diff --git a/lib/thandy/master_keys.py b/lib/thandy/master_keys.py old mode 100755 new mode 100644 diff --git a/lib/thandy/packagesys/ExePackages.py b/lib/thandy/packagesys/ExePackages.py old mode 100755 new mode 100644 diff --git a/lib/thandy/packagesys/PackageDB.py b/lib/thandy/packagesys/PackageDB.py old mode 100755 new mode 100644 diff --git a/lib/thandy/packagesys/PackageSystem.py b/lib/thandy/packagesys/PackageSystem.py old mode 100755 new mode 100644 diff --git a/lib/thandy/packagesys/RPMPackages.py b/lib/thandy/packagesys/RPMPackages.py old mode 100755 new mode 100644 diff --git a/lib/thandy/packagesys/ThpPackages.py b/lib/thandy/packagesys/ThpPackages.py old mode 100755 new mode 100644 diff --git a/lib/thandy/packagesys/__init__.py b/lib/thandy/packagesys/__init__.py old mode 100755 new mode 100644 diff --git a/lib/thandy/repository.py b/lib/thandy/repository.py old mode 100755 new mode 100644 diff --git a/lib/thandy/socksurls.py b/lib/thandy/socksurls.py old mode 100755 new mode 100644 diff --git a/lib/thandy/tests.py b/lib/thandy/tests.py old mode 100755 new mode 100644 diff --git a/lib/thandy/util.py b/lib/thandy/util.py old mode 100755 new mode 100644 diff --git a/samples/example-bundle.cfg b/samples/example-bundle.cfg old mode 100755 new mode 100644 diff --git a/samples/example-mirrors.txt b/samples/example-mirrors.txt old mode 100755 new mode 100644 diff --git a/samples/example-package.cfg b/samples/example-package.cfg old mode 100755 new mode 100644 diff --git a/setup.py b/setup.py old mode 100755 new mode 100644 diff --git a/specs/thandy-spec.txt b/specs/thandy-spec.txt old mode 100755 new mode 100644

1 0

[thandy/master] Use the tmp dir just for the metadata
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit 15de7549fbc0d9ea111710e6a3d46c74cd76ab67 Author: Tomás Touceda <chiiph(a)torproject.org> Date: Thu Oct 20 10:06:56 2011 -0300 Use the tmp dir just for the metadata --- lib/thandy/ThpCLI.py | 30 +++++------------------------- 1 files changed, 5 insertions(+), 25 deletions(-) diff --git a/lib/thandy/ThpCLI.py b/lib/thandy/ThpCLI.py index 52938d6..77d9be5 100644 --- a/lib/thandy/ThpCLI.py +++ b/lib/thandy/ThpCLI.py @@ -40,29 +40,9 @@ def makethppackage(args): metadata = thandy.formats.makeThpPackageObj(configFile, dataPath) print "Generating directory structure..." - try: - os.mkdir(os.path.join(tmpPath, "meta")); - except Exception as e: - print e - thandy.util.deltree(tmpPath) - sys.exit(1) - - thandy.util.replaceFile(os.path.join(tmpPath, "meta", "package.json"), - json.dumps(metadata, indent=3)) - - shutil.copytree(dataPath, os.path.join(tmpPath, "content")) - if "scripts" in metadata: - try: - os.mkdir(os.path.join(tmpPath, "meta", "scripts")) - except Exception as e: - print e - thandy.util.deltree(tmpPath) - sys.exit(1) - for lang in metadata["scripts"]: - for script in metadata['scripts'][lang]: - shutil.copyfile(os.path.join(scriptsPath, script[0]), - os.path.join(tmpPath, "meta", "scripts", script[0])) + thandy.util.replaceFile(os.path.join(tmpPath, "package.json"), + json.dumps(metadata, indent=3)) thpFileName = "%s-%s.thp" % (metadata['package_name'], metadata['package_version']) @@ -72,16 +52,16 @@ def makethppackage(args): thpFileName), "w") for file in metadata['manifest']: - thpFile.write(os.path.join(tmpPath, "content", file['name']), + thpFile.write(os.path.join(dataPath, file['name']), os.path.join("content", file['name'])) if "scripts" in metadata: for lang in metadata["scripts"]: for script in metadata['scripts'][lang]: - thpFile.write(os.path.join(tmpPath, "meta", "scripts", script[0]), + thpFile.write(os.path.join(scriptsPath, script[0]), os.path.join("meta", "scripts", script[0])) - thpFile.write(os.path.join(tmpPath, "meta", "package.json"), + thpFile.write(os.path.join(tmpPath, "package.json"), os.path.join("meta", "package.json")) thpFile.close()

1 0

[thandy/master] Remove unused bit of code in getChecker
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit 87470801fdc396b54bf0f7f8bfe73f05431aa080 Author: Tomás Touceda <chiiph(a)torproject.org> Date: Tue Nov 1 13:19:15 2011 -0300 Remove unused bit of code in getChecker --- lib/thandy/packagesys/PackageSystem.py | 3 --- 1 files changed, 0 insertions(+), 3 deletions(-) diff --git a/lib/thandy/packagesys/PackageSystem.py b/lib/thandy/packagesys/PackageSystem.py index 9fb878f..d255bef 100644 --- a/lib/thandy/packagesys/PackageSystem.py +++ b/lib/thandy/packagesys/PackageSystem.py @@ -54,9 +54,6 @@ def getChecker(relPath, extra, defaultFormat, package): import thandy.packagesys.PackageDB return thandy.packagesys.PackageDB.DBChecker( package['name'], package['version']) - elif defaultFormat == 'thp': - # TODO: create checker here! - pass else: return None elif checkType == 'rpm':

1 0

[thandy/master] Documents the ThpPackages code
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit ae54ecfe477546cff95d3c6b2ffafbf7314d0554 Author: Tomás Touceda <chiiph(a)torproject.org> Date: Thu Oct 20 10:32:56 2011 -0300 Documents the ThpPackages code Also fixes a typo in a still unused method. --- lib/thandy/packagesys/ThpPackages.py | 49 ++++++++++++++++++++++++++++++++- 1 files changed, 47 insertions(+), 2 deletions(-) diff --git a/lib/thandy/packagesys/ThpPackages.py b/lib/thandy/packagesys/ThpPackages.py index 49ce19e..e4e12f6 100644 --- a/lib/thandy/packagesys/ThpPackages.py +++ b/lib/thandy/packagesys/ThpPackages.py @@ -19,6 +19,8 @@ import thandy.packagesys.PackageDB as PDB json = thandy.util.importJSON() class ThpDB(object): + """ Represents the installed (or not) Thp Packages in the system """ + def __init__(self): self._upgrade = False self._thp_db_root = os.environ.get("THP_DB_ROOT") @@ -29,12 +31,17 @@ class ThpDB(object): os.mkdir(dbpath) def getPath(self): + """ Returns the path to the database root """ return self._thp_db_root def startUpgrade(self): + """ Called when an upgrade of a package starts """ self._upgrade = True def finishUpgrade(self, name): + """ Called when an upgrade of a package finishes. + If something goes wrong, this isn't called, and it will be + detected next time """ fname = os.path.join(self._thp_db_root, "pkg-status", name+".json") shutil.move(fname+".new", fname) self._upgrade = False @@ -43,6 +50,9 @@ class ThpDB(object): return self._upgrade def insert(self, pkg): + """ Adds an entry in the database for the package. If it's an + upgrade, it means that there's already an entry for this + package, so it adds a ".new" to the name. """ fname = os.path.join(self._thp_db_root, "pkg-status", pkg['package_name']+".json") if self._upgrade: @@ -52,6 +62,7 @@ class ThpDB(object): json.dumps(pkg)) def delete(self, pkg): + """ Removes an entry from the database. """ try: os.unlink(os.path.join(self._thp_db_root, "pkg-status", pkg['package_name'])+".json") @@ -62,6 +73,11 @@ class ThpDB(object): self.insert(pkg) def exists(self, name): + """ Returns a tuple (fexists, version, status). Which + represent a boolean that is true if the entry exists, the + version of the package installed, and the status of the + installation (in case the previous installation went wrong + somehow) respectively. """ fname = os.path.join(self._thp_db_root, "pkg-status", name+".json") fexists = os.path.exists(fname) @@ -83,16 +99,19 @@ class ThpDB(object): return fexists, version, status def statusInProgress(self, pkg): + """ Updates the status of a package entry to "IN-PROGRESS". """ thandy.util.replaceFile(os.path.join(self._thp_db_root, "pkg-status", pkg['package_name']+".status"), json.dumps({ "status" : "IN-PROGRESS" })) def statusInstalled(self, pkg): + """ Updates the status of a package entry to "INSTALLED". """ thandy.util.replaceFile(os.path.join(self._thp_db_root, "pkg-status", pkg['package_name']+".status"), json.dumps({ "status" : "INSTALLED" })) class ThpChecker(PS.Checker): + """ Installation checker for Thp packages. """ def __init__(self, name, version): PS.Checker.__init__(self) self._name = name @@ -103,6 +122,9 @@ class ThpChecker(PS.Checker): return "ThpChecker(%r, %r)"%(self._name, self._version) def getInstalledVersions(self): + """ Returns a tuple (version, status). Which represent a list + of versions of the installed packages and the status of the + installation. """ versions = [] (exists, version, status) = self._db.exists(self._name) @@ -112,12 +134,14 @@ class ThpChecker(PS.Checker): return versions, status def isInstalled(self): + """ Returns true if the package is installed properly. """ versions, status = self.getInstalledVersions() # if status = IN_PROGRESS a previous installation failed # we need to reinstall return (status != "IN_PROGRESS" and self._version in versions) class ThpTransaction(object): + """ Represents the installation of a bundle that contains thp packages. """ def __init__(self, packages, alreadyInstalled, repoRoot): self._raw_packages = packages self._repo_root = repoRoot @@ -128,6 +152,8 @@ class ThpTransaction(object): self._process() def _process(self): + """ Generates a list of ThpInstallers depending on whether the + package is already installed or not. """ for package in self._raw_packages.keys(): if not (self._raw_packages[package]['files'][0][0] in self._alreadyInstalled): self._installers.append(ThpInstaller(self._raw_packages[package]['files'][0][0], @@ -135,13 +161,17 @@ class ThpTransaction(object): self._repo_root)) def isReady(self): + """ A transaction is ready to be installed if it has at least + one installer to be used. """ return (len(self._installers) > 0) def _orderByDep(self): - """ Orders packages with a topological order by its dependencies """ + """ Orders packages with a topological order by its + dependencies (not implemented, obviously). """ return self._installers def install(self): + """ Installs the packages that belong to the transaction. """ lockfile = os.path.join(self._db.getPath(), "db") lock = LockFile(lockfile) try: @@ -169,10 +199,11 @@ class ThpTransaction(object): logging.info("Releasing lock...") lock.release() - def remote(self): + def remove(self): raise NotImplemented() class ThpInstaller(PS.Installer): + """ Represents an installer for an individual Thp package. """ def __init__(self, relPath, db = None, repoRoot = None): PS.Installer.__init__(self, relPath) self._db = db @@ -185,6 +216,7 @@ class ThpInstaller(PS.Installer): return "ThpInstaller(%r)" %(self._relPath) def install(self): + """ Installs the Thp package (copies the file structure to the installation root). """ logging.info("Running thp installer %s %s" % (self._cacheRoot, self._relPath)) self._thp_root = os.environ.get("THP_INSTALL_ROOT") if self._thp_root is None: @@ -239,6 +271,7 @@ class ThpInstaller(PS.Installer): return self._pkg.run(key) class ThpPackage(object): + """ Represents a Thp Package. """ def __init__(self, thp_path): self._thp_path = thp_path self._metadata = None @@ -255,6 +288,8 @@ class ThpPackage(object): print "ThpPackage(%s)" % self._thp_path def _process(self): + """ Extracts the package, validates its files and creates the + ScriptWrappers. """ self._tmp_path = tempfile.mkdtemp(suffix=str(time.time()), prefix="thp") @@ -293,13 +328,16 @@ class ThpPackage(object): sys.exit(1) def get(self, key): + """ Accessor for a metadata key. """ if self._metadata: return self._metadata.get(key) def getAll(self): + """ Accessor to all the metadata. """ return self._metadata def getDeps(self): + """ Return the dependencies especified in the metadata. """ if 'require_packages' in self._metadata.keys(): return self._metadata['require_packages'] @@ -310,6 +348,9 @@ class ThpPackage(object): return self._tmp_path def _validateFiles(self, tmpPath): + """ Validates the files based on the manifest + information. Returns a list of invalid files, in case they are + found. """ for manifest in self._metadata['manifest']: name = manifest['name'] digest = manifest['digest'] @@ -322,16 +363,20 @@ class ThpPackage(object): return (True, None) def run(self, key): + """ Executes a script from the package. """ if key in self._scripts.keys(): return self._scripts[key].run() return 0 class ScriptWrapper(object): + """ Wrapper for the scripts in the Thp package. """ def __init__(self, path = None, env = None): self._path = path self._env = None def run(self): + """ Abstracts how a script is executed. In this case, another + python instance is executed with the necessary env. """ self._process = subprocess.Popen(["python", self._path], env=self._env) self._process.wait()

1 0

[thandy/master] Paths inside a Thp cfg will use / from now on
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit 78e260013786d048589172a74dbe6288ef4edba6 Author: Tomás Touceda <chiiph(a)torproject.org> Date: Thu Oct 20 10:55:51 2011 -0300 Paths inside a Thp cfg will use / from now on --- lib/thandy/packagesys/ThpPackages.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/thandy/packagesys/ThpPackages.py b/lib/thandy/packagesys/ThpPackages.py index da76c6b..84e55c5 100644 --- a/lib/thandy/packagesys/ThpPackages.py +++ b/lib/thandy/packagesys/ThpPackages.py @@ -249,8 +249,8 @@ class ThpInstaller(PS.Installer): logging.info("Processing file: %s" % file) try: # Create all the needed dirs - os.makedirs(os.sep.join((os.path.join(destPath, file['name']) - .split(os.path.sep)[:-1]))) + os.makedirs(os.path.join((os.path.join(destPath, file['name']) + .split("/")[:-1]))) except: # Ignore if it already exists pass

1 0

[thandy/master] Fix possible race condition with os.path.exists
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit 53e9b2b3c0b10b3b0075da6acc59c032553813b7 Author: Tomás Touceda <chiiph(a)torproject.org> Date: Thu Oct 20 10:54:16 2011 -0300 Fix possible race condition with os.path.exists --- lib/thandy/packagesys/ThpPackages.py | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diff --git a/lib/thandy/packagesys/ThpPackages.py b/lib/thandy/packagesys/ThpPackages.py index e4e12f6..da76c6b 100644 --- a/lib/thandy/packagesys/ThpPackages.py +++ b/lib/thandy/packagesys/ThpPackages.py @@ -27,8 +27,10 @@ class ThpDB(object): if self._thp_db_root is None: raise Exception("There is no THP_DB_ROOT variable set") dbpath = os.path.join(self._thp_db_root, "pkg-status") - if not os.path.exists(dbpath): + try: os.mkdir(dbpath) + except OSError, e: + pass def getPath(self): """ Returns the path to the database root """

1 0

[thandy/master] Make transactions not Thp specific
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit 3e752ddc9435831c5d74181e070ec8c5ba1b714e Author: Tomás Touceda <chiiph(a)torproject.org> Date: Tue Nov 1 13:20:49 2011 -0300 Make transactions not Thp specific --- lib/thandy/ClientCLI.py | 25 +++++++++++------------ lib/thandy/packagesys/PackageSystem.py | 29 +++++++++++++++++++++++++++ lib/thandy/packagesys/ThpPackages.py | 8 +++++- lib/thandy/repository.py | 34 ++++++++++++++----------------- 4 files changed, 62 insertions(+), 34 deletions(-) diff --git a/lib/thandy/ClientCLI.py b/lib/thandy/ClientCLI.py index f3192f9..d3bcaf0 100644 --- a/lib/thandy/ClientCLI.py +++ b/lib/thandy/ClientCLI.py @@ -15,7 +15,6 @@ import thandy.repository import thandy.download import thandy.master_keys import thandy.packagesys.PackageSystem -import thandy.packagesys.ThpPackages import thandy.socksurls import thandy.encodeToXML @@ -121,7 +120,6 @@ def update(args): lengths = {} installable = {} btMetadata = {} - thpTransactions = {} alreadyInstalled = set() logging.info("Checking for files to update.") files, downloadingFiles = repo.getFilesToUpdate( @@ -131,23 +129,24 @@ def update(args): usePackageSystem=use_packagesys, installableDict=installable, btMetadataDict=btMetadata, - thpTransactionDict=thpTransactions, - alreadyInstalledSet=alreadyInstalled) + alreadyInstalledSet=alreadyInstalled, + cacheRoot=repoRoot) if forceCheck: files.add("/meta/timestamp.txt") forceCheck = False - if thpTransactions and not files: - for bundle in thpTransactions: - tr = thandy.packagesys.ThpPackages.ThpTransaction(thpTransactions[bundle], - alreadyInstalled, - repoRoot) - if tr.isReady(): + if installable and not files: + for bundle, transaction in installable.items(): + if transaction.isReady(): logCtrl("READY", BUNDLE=bundle) - if install and tr.isReady(): - tr.install() - logCtrl("INSTALLED", BUNDLE=bundle) + logging.info("Ready to install packages for files: %s", + ", ".join(sorted(installable.keys()))) + + if install: + for p in installable.values(): + if p.isReady(): + p.install() return diff --git a/lib/thandy/packagesys/PackageSystem.py b/lib/thandy/packagesys/PackageSystem.py index d255bef..b2e66a1 100644 --- a/lib/thandy/packagesys/PackageSystem.py +++ b/lib/thandy/packagesys/PackageSystem.py @@ -121,6 +121,16 @@ def getInstaller(relPath, extra, defaultFormat, package): return installer +def getTransaction(transactionType, packages, alreadyInstalledSet, cacheRoot): + """ Return a transaction from the type transactionType considering + the alreadyInstalledSet and that works in the cacheRoot """ + if transactionType == "thp": + import thandy.packagesys.ThpPackages + return thandy.packagesys.ThpPackages.ThpTransaction(packages, + alreadyInstalledSet, + cacheRoot) + return None + class PackageItem: """Represents a single item from a package.""" def __init__(self, relativePath, checker, installer): @@ -237,4 +247,23 @@ class Installer: "DOCDOC params, manifest" return None, None +class Transaction(object): + """ Abstract base class. A Transaction knows how to install or + remove a bundle. """ + def __init__(self, packages, alreadyInstalled, repoRoot): + self._raw_packages = packages + self._repo_root = repoRoot + self._alreadyInstalled = alreadyInstalled + def isReady(self): + """ Returns True if the transaction is ready to start installing + or removing a bundle. """ + raise NotImplemented() + + def install(self): + """ Installs the packages that belong to this transaction. """ + raise NotImplemented() + + def remove(self): + """ Removes the packages that belong to this transaction. """ + raise NotImplemented() diff --git a/lib/thandy/packagesys/ThpPackages.py b/lib/thandy/packagesys/ThpPackages.py index b558d59..f82415e 100644 --- a/lib/thandy/packagesys/ThpPackages.py +++ b/lib/thandy/packagesys/ThpPackages.py @@ -142,7 +142,7 @@ class ThpChecker(PS.Checker): # we need to reinstall return (status == "INSTALLED" and self._version in versions) -class ThpTransaction(object): +class ThpTransaction(PS.Transaction): """ Represents the installation of a bundle that contains thp packages. """ def __init__(self, packages, alreadyInstalled, repoRoot): self._raw_packages = packages @@ -254,8 +254,12 @@ class ThpInstaller(PS.Installer): except: # Ignore if it already exists pass + + if "/" in file["name"]: + os.makedirs("/".join([destPath] + file["name"].split("/")[:-1])) + shutil.copy(os.path.join(self._pkg.getTmpPath(), "content", file['name']), - os.path.join(destPath, file['name'])); + os.path.join(destPath, file['name'])); if self._db.isUpgrading(): logging.info("Finishing upgrade.") diff --git a/lib/thandy/repository.py b/lib/thandy/repository.py index 0b45c8d..b294018 100644 --- a/lib/thandy/repository.py +++ b/lib/thandy/repository.py @@ -302,12 +302,13 @@ class LocalRepository: def getFilesToUpdate(self, now=None, trackingBundles=(), hashDict=None, lengthDict=None, usePackageSystem=True, installableDict=None, btMetadataDict=None, - thpTransactionDict=None, alreadyInstalledSet=None): + alreadyInstalledSet=None, + cacheRoot=None): """Return a set of relative paths for all files that we need to fetch, and True if we're fetching actual files to install instead of metadata. Assumes that we care about the bundles 'trackingBundles'. - DOCDOC installableDict, hashDict, usePackageSystem, thpTransaction + DOCDOC installableDict, hashDict, usePackageSystem """ if now == None: @@ -326,10 +327,8 @@ class LocalRepository: if btMetadataDict == None: btMetadataDict = {} - if thpTransactionDict == None: - thpTransactionDict = {} - pkgItems = None + transactions = {} need = set() @@ -497,10 +496,8 @@ class LocalRepository: packages[rp] = pfile - if pfile_data["format"] == "thp": - if not bundle['name'] in thpTransactionDict.keys(): - thpTransactionDict[bundle['name']] = {} - thpTransactionDict[bundle['name']][pfile_data['name']] = pfile_data + transactions.setdefault(pfile_data["format"], {})\ + .setdefault(bundle["name"], {})[pfile_data["name"]] = pfile_data # We have the packages. If we're downloading via bittorrent, we need # the .torrent metafiles, as well. @@ -546,8 +543,6 @@ class LocalRepository: try: if item.getChecker().isInstalled(): alreadyInstalledSet.add(item.getRelativePath()) - if item.getRelativePath() in thpTransactionDict.keys(): - thpTransactionDict.pop(item.getRelativePath()) except thandy.CheckNotSupported, err: logging.warn("Can't check installed-ness of %s: %s", f[0], err) @@ -577,14 +572,15 @@ class LocalRepository: if h_got != h_expected: logging.info("Hash for %s not as expected; must load.", rp) need.add(rp) - else: - # XXX What if not? Maybe this should always be true. - # if that works, we can get rid of the second return - # value and just use installableDict from the caller. - if pkgItems.has_key(rp): - if pkgItems[rp]: - installableDict.setdefault(pkg_rp, {})[rp] = pkgItems[rp] - + if len(need) == 0: + # We have done everything, lets see if we have thp bundles, + # and create the transaction for it as the installable item + for transaction_type in transactions: + for bundle in transactions[transaction_type]: + installableDict[bundle] = thandy.packagesys.PackageSystem.getTransaction(transaction_type, + transactions[transaction_type][bundle], + alreadyInstalledSet, + cacheRoot) # Okay; these are the files we need. return need, True

1 0