[or-cvs] r21185: {projects} Change rsync procedures, fix cronjob installation, put split (projects/gettor/lib/gettor)

kaner at seul.org kaner at seul.org
Sun Dec 13 16:47:55 UTC 2009


Author: kaner
Date: 2009-12-13 11:47:55 -0500 (Sun, 13 Dec 2009)
New Revision: 21185

Modified:
   projects/gettor/lib/gettor/packages.py
   projects/gettor/lib/gettor/responses.py
   projects/gettor/lib/gettor/utils.py
Log:
Change rsync procedures, fix cronjob installation, put split package build in a separate routine


Modified: projects/gettor/lib/gettor/packages.py
===================================================================
--- projects/gettor/lib/gettor/packages.py	2009-12-13 14:59:45 UTC (rev 21184)
+++ projects/gettor/lib/gettor/packages.py	2009-12-13 16:47:55 UTC (rev 21185)
@@ -59,9 +59,10 @@
                      #"torbutton": "torbutton-current.xpi$",
                    }
 
-    def __init__(self, config):
+    def __init__(self, config, mirror="rsync.torproject.org", silent=False):
         self.packageList = {}
         self.distDir = config.getDistDir()
+        self.initRsync(mirror, silent)
         try:
             entry = os.stat(self.distDir)
         except OSError, e:
@@ -95,46 +96,19 @@
         return self.packageList
 
     def buildPackages(self):
-        # Reverse this: Loop through package regex and look for match 
         for (pack, (regex_single, regex_split)) in self.packageRegex.items():
             for filename in os.listdir(self.distDir):
                 # Splitfile hacks. XXX: Refactor
                 if re.compile(regex_split).match(filename):
-                    packSplitDir = None
-                    try:
-                        packSplitDir = self.packDir + "/" + pack + ".split"
-                        if not os.access(packSplitDir, os.R_OK):
-                            os.mkdir(packSplitDir)
-                    except OSError, e:
-                        log.error("Could not create dir %s: %s" \
-                                        % (packSplitDir, e))
-                    # Loop through split dir, look if every partXX.ZZZ has a 
-                    # matching signature, pack them together in a .z
-                    splitdir = self.distDir + "/" + filename
-                    for splitfile in os.listdir(splitdir):
-                        # Skip signature files
-                        if splitfile.endswith(".asc"):
-                            continue
-                        if re.compile(".*split.part.*").match(splitfile):
-                            ascfile = splitdir + "/signatures/" + splitfile + ".asc"
-                            file = splitdir + "/" + splitfile
-                            zipFileName = packSplitDir + "/" + splitfile + ".z"
-                            if os.access(ascfile, os.R_OK) and os.access(file, os.R_OK):
-                                print "ok: ", zipFileName
-                                zip = zipfile.ZipFile(zipFileName, "w")
-                                zip.write(splitdir + "/" + splitfile, os.path.basename(file))
-                                zip.write(ascfile, os.path.basename(ascfile))
-                                zip.close()
-                            else:
-                                log.error("Uhm, expected signature file for %s to be: %s" % (file, ascfile))
-                                return False
+                    if not self.buildSplitFiles(pack, filename):
+                        log.error("Could not build split files packages")
+                        return False
                 if re.compile(regex_single).match(filename):
                     file = self.distDir + "/" + filename
                     ascfile = file + ".asc"
                     zipFileName  = self.packDir + "/" + pack + ".z"
                     # If .asc file is there, build Zip file
                     if os.access(ascfile, os.R_OK):
-		    	print "ok: ", zipFileName
                         zip = zipfile.ZipFile(zipFileName, "w")
                         zip.write(file, os.path.basename(file))
                         zip.write(ascfile, os.path.basename(ascfile))
@@ -147,66 +121,112 @@
             log.error("Failed to build packages")
             return False
 
-    def syncWithMirror(self, mirror, silent):
-        rsync = ["rsync"]
-        rsync.append("-a")
-        # Exclude stuff we don't need
-        rsync.append("--exclude=*current*")
-        rsync.append("--exclude=*osx*")
-        rsync.append("--exclude=*rpm*")
-        rsync.append("--exclude=*privoxy*")
-        rsync.append("--exclude=*alpha*")
-        rsync.append("--exclude=*vidalia-bundles*")
+    def initRsync(self, mirror="rsync.torproject.org", silent=False):
+        # Rsync command 1
+        self.rsync = "rsync -a" 
+        self.rsync += " "
+        self.rsync += "--exclude='*current*'"
+        self.rsync += " "
+        self.rsync += "--exclude='*osx*'"
+        self.rsync += " "
+        self.rsync += "--exclude='*rpm*'"
+        self.rsync += " "
+        self.rsync += "--exclude='*privoxy*'"
+        self.rsync += " "
+        self.rsync += "--exclude='*alpha*'"
+        self.rsync += " "
+        self.rsync += "--exclude='*vidalia-bundles*'"
+        self.rsync += " "
         if not silent:
-            rsync.append("--progress")
-        rsync.append("rsync://%s/tor/dist/" % mirror)
-        # XXX HACK :) will be fixed soon
-        rsync2 = ["rsync"]
-        rsync2.append("-a")
-        # Exclude stuff we don't need
-        rsync2.append("--exclude=*current*")
-        rsync2.append("--exclude=*osx*")
-        rsync2.append("--exclude=*rpm*")
-        rsync2.append("--exclude=*privoxy*")
-        rsync2.append("--exclude=*alpha*")
-        rsync2.append("--exclude=*vidalia-bundles*")
+            self.rsync += "--progress"
+            self.rsync += " "
+        self.rsync += "rsync://%s/tor/dist/" % mirror
+        self.rsync += " "
+        self.rsync += self.distDir
+        self.rsync += " "
+        self.rsync += "&&"
+        self.rsync += " "
+        # Rsync command 2
+        self.rsync += "rsync -a"
+        self.rsync += " "
+        self.rsync += "--exclude='*current*'"
+        self.rsync += " "
+        self.rsync += "--exclude='*osx*'"
+        self.rsync += " "
+        self.rsync += "--exclude='*rpm*'"
+        self.rsync += " "
+        self.rsync += "--exclude='*privoxy*'"
+        self.rsync += " "
+        self.rsync += "--exclude='*alpha*'"
+        self.rsync += " "
+        self.rsync += "--exclude='*vidalia-bundles*'"
+        self.rsync += " "
         if not silent:
-            rsync2.append("--progress")
-        rsync2.append("rsync://%s/tor/torbrowser/dist/" % mirror)
-        rsync2.append(self.distDir)
-        rsync3 = ["rsync"]
-        rsync3.append("-a")
-        rsync3.append("--exclude=*alpha*")
+            self.rsync += "--progress"
+            self.rsync += " "
+        self.rsync += "rsync://%s/tor/torbrowser/dist/" % mirror
+        self.rsync += " "
+        self.rsync += self.distDir
+        self.rsync += " "
+        self.rsync += "&&"
+        self.rsync += " "
+        # Rsync command 3
+        self.rsync += "rsync -a"
+        self.rsync += " "
+        self.rsync += "--exclude='*alpha*'"
+        self.rsync += " "
         if not silent:
-            rsync3.append("--progress")
-        rsync3.append("rsync://%s/tor/dist/vidalia-bundles/" % mirror)
-        rsync3.append(self.distDir)
-        rsync.append(self.distDir)
-        process = subprocess.Popen(rsync)
+            self.rsync += "--progress"
+            self.rsync += " "
+        self.rsync += "rsync://%s/tor/dist/vidalia-bundles/" % mirror
+        self.rsync += " "
+        self.rsync += self.distDir
+        self.rsync += " "
+
+    def syncWithMirror(self):
+        process = subprocess.Popen(self.rsync, shell=True)
         process.wait()
-        process = subprocess.Popen(rsync2)
-        process.wait()
-        process = subprocess.Popen(rsync3)
-        process.wait()
+
         return process.returncode
 
     def getCommandToStr(self, mirror, silent):
         """This is useful for cronjob installations
         """
-        rsync = ["rsync"]
-        rsync.append("-a")
-        rsync.append("--exclude=*current*")
-        rsync.append("--exclude=*osx*")
-        rsync.append("--exclude=*rpm*")
-        rsync.append("--exclude=*privoxy*")
-        rsync.append("--exclude=*alpha*")
-        rsync.append("--exclude=*vidalia-bundles*")
-        if not silent:
-            rsync.append("--progress")
-        rsync.append("rsync://%s/tor/dist/current/" % mirror)
-        rsync.append(self.distDir)
-        return ''.join(self.rsync)
+        return self.rsync
 
+    def buildSplitFiles(self, pack, filename):
+        log.info("Building split files..")
+        packSplitDir = None
+        try:
+            packSplitDir = self.packDir + "/" + pack + ".split"
+            if not os.access(packSplitDir, os.R_OK):
+                os.mkdir(packSplitDir)
+        except OSError, e:
+            log.error("Could not create dir %s: %s" \
+                            % (packSplitDir, e))
+        # Loop through split dir, look if every partXX.ZZZ has a 
+        # matching signature, pack them together in a .z
+        splitdir = self.distDir + "/" + filename
+        for splitfile in os.listdir(splitdir):
+            # Skip signature files
+            if splitfile.endswith(".asc"):
+                continue
+            if re.compile(".*split.part.*").match(splitfile):
+                ascfile = splitdir + "/signatures/" + splitfile + ".asc"
+                file = splitdir + "/" + splitfile
+                zipFileName = packSplitDir + "/" + splitfile + ".z"
+                if os.access(ascfile, os.R_OK) and os.access(file, os.R_OK):
+                    zip = zipfile.ZipFile(zipFileName, "w")
+                    zip.write(splitdir + "/" + splitfile, os.path.basename(file))
+                    zip.write(ascfile, os.path.basename(ascfile))
+                    zip.close()
+                else:
+                    log.error("Uhm, expected signature file for %s to be: %s" % (file, ascfile))
+                    return False
+
+        log.info("Done.")
+        return True
+
 if __name__ == "__main__" :
     c = gettor_config.Config()
     p = gettorPackages("rsync.torproject.org", c)

Modified: projects/gettor/lib/gettor/responses.py
===================================================================
--- projects/gettor/lib/gettor/responses.py	2009-12-13 14:59:45 UTC (rev 21184)
+++ projects/gettor/lib/gettor/responses.py	2009-12-13 16:47:55 UTC (rev 21185)
@@ -149,6 +149,8 @@
             package = self.constructMessage(message, subj, fullPath)
             try:
                 status = self.sendMessage(package)
+                log.info("Sent out split package [%02d / %02d]. Status: %s" \
+                        % (num, nFiles, status))
             except:
                 log.error("Could not send package %s to user" % filename)
                 # XXX What now? Keep on sending? Bail out? Use might have 

Modified: projects/gettor/lib/gettor/utils.py
===================================================================
--- projects/gettor/lib/gettor/utils.py	2009-12-13 14:59:45 UTC (rev 21184)
+++ projects/gettor/lib/gettor/utils.py	2009-12-13 16:47:55 UTC (rev 21185)
@@ -12,6 +12,7 @@
 '''
 
 import os
+import sys
 import subprocess
 import hashlib
 import datetime
@@ -112,11 +113,11 @@
     """Fetch Tor packages from a mirror"""
     log.info("Fetching package files..")
     try:
-        packs = gettor.packages.Packages(conf)
+        packs = gettor.packages.Packages(conf, mirror, False)
     except IOError:
         log.error("Error initiating package list.")
         return False
-    if packs.syncWithMirror(mirror, False) != 0:
+    if packs.syncWithMirror() != 0:
         log.error("Syncing Tor packages failed.")
         return False
     else:



More information about the tor-commits mailing list