- tor-commits - lists.torproject.org

[thandy/master] Paths inside a Thp cfg will use / from now on
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit 78e260013786d048589172a74dbe6288ef4edba6 Author: Tomás Touceda <chiiph(a)torproject.org> Date: Thu Oct 20 10:55:51 2011 -0300 Paths inside a Thp cfg will use / from now on --- lib/thandy/packagesys/ThpPackages.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/thandy/packagesys/ThpPackages.py b/lib/thandy/packagesys/ThpPackages.py index da76c6b..84e55c5 100644 --- a/lib/thandy/packagesys/ThpPackages.py +++ b/lib/thandy/packagesys/ThpPackages.py @@ -249,8 +249,8 @@ class ThpInstaller(PS.Installer): logging.info("Processing file: %s" % file) try: # Create all the needed dirs - os.makedirs(os.sep.join((os.path.join(destPath, file['name']) - .split(os.path.sep)[:-1]))) + os.makedirs(os.path.join((os.path.join(destPath, file['name']) + .split("/")[:-1]))) except: # Ignore if it already exists pass

1 0

[thandy/master] Fix possible race condition with os.path.exists
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit 53e9b2b3c0b10b3b0075da6acc59c032553813b7 Author: Tomás Touceda <chiiph(a)torproject.org> Date: Thu Oct 20 10:54:16 2011 -0300 Fix possible race condition with os.path.exists --- lib/thandy/packagesys/ThpPackages.py | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diff --git a/lib/thandy/packagesys/ThpPackages.py b/lib/thandy/packagesys/ThpPackages.py index e4e12f6..da76c6b 100644 --- a/lib/thandy/packagesys/ThpPackages.py +++ b/lib/thandy/packagesys/ThpPackages.py @@ -27,8 +27,10 @@ class ThpDB(object): if self._thp_db_root is None: raise Exception("There is no THP_DB_ROOT variable set") dbpath = os.path.join(self._thp_db_root, "pkg-status") - if not os.path.exists(dbpath): + try: os.mkdir(dbpath) + except OSError, e: + pass def getPath(self): """ Returns the path to the database root """

1 0

[thandy/master] Make transactions not Thp specific
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit 3e752ddc9435831c5d74181e070ec8c5ba1b714e Author: Tomás Touceda <chiiph(a)torproject.org> Date: Tue Nov 1 13:20:49 2011 -0300 Make transactions not Thp specific --- lib/thandy/ClientCLI.py | 25 +++++++++++------------ lib/thandy/packagesys/PackageSystem.py | 29 +++++++++++++++++++++++++++ lib/thandy/packagesys/ThpPackages.py | 8 +++++- lib/thandy/repository.py | 34 ++++++++++++++----------------- 4 files changed, 62 insertions(+), 34 deletions(-) diff --git a/lib/thandy/ClientCLI.py b/lib/thandy/ClientCLI.py index f3192f9..d3bcaf0 100644 --- a/lib/thandy/ClientCLI.py +++ b/lib/thandy/ClientCLI.py @@ -15,7 +15,6 @@ import thandy.repository import thandy.download import thandy.master_keys import thandy.packagesys.PackageSystem -import thandy.packagesys.ThpPackages import thandy.socksurls import thandy.encodeToXML @@ -121,7 +120,6 @@ def update(args): lengths = {} installable = {} btMetadata = {} - thpTransactions = {} alreadyInstalled = set() logging.info("Checking for files to update.") files, downloadingFiles = repo.getFilesToUpdate( @@ -131,23 +129,24 @@ def update(args): usePackageSystem=use_packagesys, installableDict=installable, btMetadataDict=btMetadata, - thpTransactionDict=thpTransactions, - alreadyInstalledSet=alreadyInstalled) + alreadyInstalledSet=alreadyInstalled, + cacheRoot=repoRoot) if forceCheck: files.add("/meta/timestamp.txt") forceCheck = False - if thpTransactions and not files: - for bundle in thpTransactions: - tr = thandy.packagesys.ThpPackages.ThpTransaction(thpTransactions[bundle], - alreadyInstalled, - repoRoot) - if tr.isReady(): + if installable and not files: + for bundle, transaction in installable.items(): + if transaction.isReady(): logCtrl("READY", BUNDLE=bundle) - if install and tr.isReady(): - tr.install() - logCtrl("INSTALLED", BUNDLE=bundle) + logging.info("Ready to install packages for files: %s", + ", ".join(sorted(installable.keys()))) + + if install: + for p in installable.values(): + if p.isReady(): + p.install() return diff --git a/lib/thandy/packagesys/PackageSystem.py b/lib/thandy/packagesys/PackageSystem.py index d255bef..b2e66a1 100644 --- a/lib/thandy/packagesys/PackageSystem.py +++ b/lib/thandy/packagesys/PackageSystem.py @@ -121,6 +121,16 @@ def getInstaller(relPath, extra, defaultFormat, package): return installer +def getTransaction(transactionType, packages, alreadyInstalledSet, cacheRoot): + """ Return a transaction from the type transactionType considering + the alreadyInstalledSet and that works in the cacheRoot """ + if transactionType == "thp": + import thandy.packagesys.ThpPackages + return thandy.packagesys.ThpPackages.ThpTransaction(packages, + alreadyInstalledSet, + cacheRoot) + return None + class PackageItem: """Represents a single item from a package.""" def __init__(self, relativePath, checker, installer): @@ -237,4 +247,23 @@ class Installer: "DOCDOC params, manifest" return None, None +class Transaction(object): + """ Abstract base class. A Transaction knows how to install or + remove a bundle. """ + def __init__(self, packages, alreadyInstalled, repoRoot): + self._raw_packages = packages + self._repo_root = repoRoot + self._alreadyInstalled = alreadyInstalled + def isReady(self): + """ Returns True if the transaction is ready to start installing + or removing a bundle. """ + raise NotImplemented() + + def install(self): + """ Installs the packages that belong to this transaction. """ + raise NotImplemented() + + def remove(self): + """ Removes the packages that belong to this transaction. """ + raise NotImplemented() diff --git a/lib/thandy/packagesys/ThpPackages.py b/lib/thandy/packagesys/ThpPackages.py index b558d59..f82415e 100644 --- a/lib/thandy/packagesys/ThpPackages.py +++ b/lib/thandy/packagesys/ThpPackages.py @@ -142,7 +142,7 @@ class ThpChecker(PS.Checker): # we need to reinstall return (status == "INSTALLED" and self._version in versions) -class ThpTransaction(object): +class ThpTransaction(PS.Transaction): """ Represents the installation of a bundle that contains thp packages. """ def __init__(self, packages, alreadyInstalled, repoRoot): self._raw_packages = packages @@ -254,8 +254,12 @@ class ThpInstaller(PS.Installer): except: # Ignore if it already exists pass + + if "/" in file["name"]: + os.makedirs("/".join([destPath] + file["name"].split("/")[:-1])) + shutil.copy(os.path.join(self._pkg.getTmpPath(), "content", file['name']), - os.path.join(destPath, file['name'])); + os.path.join(destPath, file['name'])); if self._db.isUpgrading(): logging.info("Finishing upgrade.") diff --git a/lib/thandy/repository.py b/lib/thandy/repository.py index 0b45c8d..b294018 100644 --- a/lib/thandy/repository.py +++ b/lib/thandy/repository.py @@ -302,12 +302,13 @@ class LocalRepository: def getFilesToUpdate(self, now=None, trackingBundles=(), hashDict=None, lengthDict=None, usePackageSystem=True, installableDict=None, btMetadataDict=None, - thpTransactionDict=None, alreadyInstalledSet=None): + alreadyInstalledSet=None, + cacheRoot=None): """Return a set of relative paths for all files that we need to fetch, and True if we're fetching actual files to install instead of metadata. Assumes that we care about the bundles 'trackingBundles'. - DOCDOC installableDict, hashDict, usePackageSystem, thpTransaction + DOCDOC installableDict, hashDict, usePackageSystem """ if now == None: @@ -326,10 +327,8 @@ class LocalRepository: if btMetadataDict == None: btMetadataDict = {} - if thpTransactionDict == None: - thpTransactionDict = {} - pkgItems = None + transactions = {} need = set() @@ -497,10 +496,8 @@ class LocalRepository: packages[rp] = pfile - if pfile_data["format"] == "thp": - if not bundle['name'] in thpTransactionDict.keys(): - thpTransactionDict[bundle['name']] = {} - thpTransactionDict[bundle['name']][pfile_data['name']] = pfile_data + transactions.setdefault(pfile_data["format"], {})\ + .setdefault(bundle["name"], {})[pfile_data["name"]] = pfile_data # We have the packages. If we're downloading via bittorrent, we need # the .torrent metafiles, as well. @@ -546,8 +543,6 @@ class LocalRepository: try: if item.getChecker().isInstalled(): alreadyInstalledSet.add(item.getRelativePath()) - if item.getRelativePath() in thpTransactionDict.keys(): - thpTransactionDict.pop(item.getRelativePath()) except thandy.CheckNotSupported, err: logging.warn("Can't check installed-ness of %s: %s", f[0], err) @@ -577,14 +572,15 @@ class LocalRepository: if h_got != h_expected: logging.info("Hash for %s not as expected; must load.", rp) need.add(rp) - else: - # XXX What if not? Maybe this should always be true. - # if that works, we can get rid of the second return - # value and just use installableDict from the caller. - if pkgItems.has_key(rp): - if pkgItems[rp]: - installableDict.setdefault(pkg_rp, {})[rp] = pkgItems[rp] - + if len(need) == 0: + # We have done everything, lets see if we have thp bundles, + # and create the transaction for it as the installable item + for transaction_type in transactions: + for bundle in transactions[transaction_type]: + installableDict[bundle] = thandy.packagesys.PackageSystem.getTransaction(transaction_type, + transactions[transaction_type][bundle], + alreadyInstalledSet, + cacheRoot) # Okay; these are the files we need. return need, True

1 0

[thandy/master] Fix minor bug in the ThpCheck: broken status wont mark it as not installed
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit 31d99e2446f2d9ddc3832ba3652c317724dcd6c2 Author: Tomás Touceda <chiiph(a)torproject.org> Date: Tue Nov 1 13:15:48 2011 -0300 Fix minor bug in the ThpCheck: broken status wont mark it as not installed --- lib/thandy/packagesys/ThpPackages.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/lib/thandy/packagesys/ThpPackages.py b/lib/thandy/packagesys/ThpPackages.py index 84e55c5..b558d59 100644 --- a/lib/thandy/packagesys/ThpPackages.py +++ b/lib/thandy/packagesys/ThpPackages.py @@ -140,7 +140,7 @@ class ThpChecker(PS.Checker): versions, status = self.getInstalledVersions() # if status = IN_PROGRESS a previous installation failed # we need to reinstall - return (status != "IN_PROGRESS" and self._version in versions) + return (status == "INSTALLED" and self._version in versions) class ThpTransaction(object): """ Represents the installation of a bundle that contains thp packages. """

1 0

[thandy/master] Fix typo in RPMChecker
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit ba87d47619e73663d3a1475f954451c5ac8047ca Author: Tomas Touceda <chiiph(a)gentoo.org> Date: Tue Jun 21 21:22:18 2011 -0300 Fix typo in RPMChecker --- lib/thandy/packagesys/RPMPackages.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/lib/thandy/packagesys/RPMPackages.py b/lib/thandy/packagesys/RPMPackages.py index 2c793e3..331ef8e 100644 --- a/lib/thandy/packagesys/RPMPackages.py +++ b/lib/thandy/packagesys/RPMPackages.py @@ -125,7 +125,7 @@ def checkRPMInstall(name, version, ts=None): return found and all_ok -class RPMChacker(PS.Checker): +class RPMChecker(PS.Checker): def __init__(self, rpmname, rpmversion): PS.Checker.__init__(self) self._name = rpmname

1 0

[tor/master] Fix a memleak when fetching descriptors for bridges in ExcludeNodes.
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit 3ae96845d33b11ded74d0e569a859d8f138b70d9 Author: George Kadianakis <desnacked(a)gmail.com> Date: Thu Nov 3 17:16:31 2011 +0100 Fix a memleak when fetching descriptors for bridges in ExcludeNodes. --- changes/bug4383 | 5 +++++ src/or/circuitbuild.c | 3 ++- 2 files changed, 7 insertions(+), 1 deletions(-) diff --git a/changes/bug4383 b/changes/bug4383 new file mode 100644 index 0000000..e618b8c --- /dev/null +++ b/changes/bug4383 @@ -0,0 +1,5 @@ + o Minor bugfixes: + - Fix a memleak in launch_direct_bridge_descriptor_fetch() that + occured when a client tried to fetch a descriptor for a bridge + in ExcludeNodes. Fixes #4383; bugfix on 0.2.2.25-alpha. + diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c index c864fd2..72ec9e4 100644 --- a/src/or/circuitbuild.c +++ b/src/or/circuitbuild.c @@ -4670,7 +4670,6 @@ launch_direct_bridge_descriptor_fetch(bridge_info_t *bridge) DIR_PURPOSE_FETCH_SERVERDESC)) return; /* it's already on the way */ - address = tor_dup_addr(&bridge->addr); if (routerset_contains_bridge(options->ExcludeNodes, bridge)) { download_status_mark_impossible(&bridge->fetch_status); log_warn(LD_APP, "Not using bridge at %s: it is in ExcludeNodes.", @@ -4678,6 +4677,8 @@ launch_direct_bridge_descriptor_fetch(bridge_info_t *bridge) return; } + address = tor_dup_addr(&bridge->addr); + directory_initiate_command(address, &bridge->addr, bridge->port, 0, 0, /* does not matter */

1 0

[tor/master] Merge branch 'bug4383_nm' into maint-0.2.2
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit 972564630e8f85499e611f1eeb7174d0c0230731 Merge: 16dbcaf 3ae9684 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Nov 3 13:23:59 2011 -0400 Merge branch 'bug4383_nm' into maint-0.2.2 changes/bug4383 | 5 +++++ src/or/circuitbuild.c | 3 ++- 2 files changed, 7 insertions(+), 1 deletions(-)

1 0

[tor/master] Merge remote-tracking branch 'origin/maint-0.2.2'
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit c1005dd6d34a40080af478ef4cb85860d04f3844 Merge: d029c32 9725646 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Nov 3 13:24:46 2011 -0400 Merge remote-tracking branch 'origin/maint-0.2.2' changes/bug4383 | 5 +++++ src/or/circuitbuild.c | 3 ++- 2 files changed, 7 insertions(+), 1 deletions(-)

1 0

[tor/maint-0.2.2] Fix a memleak when fetching descriptors for bridges in ExcludeNodes.
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit 3ae96845d33b11ded74d0e569a859d8f138b70d9 Author: George Kadianakis <desnacked(a)gmail.com> Date: Thu Nov 3 17:16:31 2011 +0100 Fix a memleak when fetching descriptors for bridges in ExcludeNodes. --- changes/bug4383 | 5 +++++ src/or/circuitbuild.c | 3 ++- 2 files changed, 7 insertions(+), 1 deletions(-) diff --git a/changes/bug4383 b/changes/bug4383 new file mode 100644 index 0000000..e618b8c --- /dev/null +++ b/changes/bug4383 @@ -0,0 +1,5 @@ + o Minor bugfixes: + - Fix a memleak in launch_direct_bridge_descriptor_fetch() that + occured when a client tried to fetch a descriptor for a bridge + in ExcludeNodes. Fixes #4383; bugfix on 0.2.2.25-alpha. + diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c index c864fd2..72ec9e4 100644 --- a/src/or/circuitbuild.c +++ b/src/or/circuitbuild.c @@ -4670,7 +4670,6 @@ launch_direct_bridge_descriptor_fetch(bridge_info_t *bridge) DIR_PURPOSE_FETCH_SERVERDESC)) return; /* it's already on the way */ - address = tor_dup_addr(&bridge->addr); if (routerset_contains_bridge(options->ExcludeNodes, bridge)) { download_status_mark_impossible(&bridge->fetch_status); log_warn(LD_APP, "Not using bridge at %s: it is in ExcludeNodes.", @@ -4678,6 +4677,8 @@ launch_direct_bridge_descriptor_fetch(bridge_info_t *bridge) return; } + address = tor_dup_addr(&bridge->addr); + directory_initiate_command(address, &bridge->addr, bridge->port, 0, 0, /* does not matter */

1 0

[tor/maint-0.2.2] Merge branch 'bug4383_nm' into maint-0.2.2
by nickm＠torproject.org 03 Nov '11

03 Nov '11

commit 972564630e8f85499e611f1eeb7174d0c0230731 Merge: 16dbcaf 3ae9684 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Nov 3 13:23:59 2011 -0400 Merge branch 'bug4383_nm' into maint-0.2.2 changes/bug4383 | 5 +++++ src/or/circuitbuild.c | 3 ++- 2 files changed, 7 insertions(+), 1 deletions(-)

1 0