[or-cvs] r17478: {projects} Patch from kaner - added whitelisting, cleaned up blacklisti (projects/gettor)

ioerror at seul.org ioerror at seul.org
Thu Dec 4 09:54:09 UTC 2008


Author: ioerror
Date: 2008-12-04 04:54:09 -0500 (Thu, 04 Dec 2008)
New Revision: 17478

Modified:
   projects/gettor/gettor.py
   projects/gettor/gettor_blacklist.py
   projects/gettor/gettor_config.py
   projects/gettor/gettor_log.py
   projects/gettor/gettor_opt.py
   projects/gettor/gettor_packages.py
   projects/gettor/gettor_responses.py
Log:
Patch from kaner - added whitelisting, cleaned up blacklisting, some fixes / improvements and is lightly broken :)


Modified: projects/gettor/gettor.py
===================================================================
--- projects/gettor/gettor.py	2008-12-03 20:54:49 UTC (rev 17477)
+++ projects/gettor/gettor.py	2008-12-04 09:54:09 UTC (rev 17478)
@@ -69,9 +69,11 @@
 
 
 # Switch language to 'newlocale'. Return default if language is not supported.
-# XXX: There should be a more elegant way to switch languages during runtime.
 def switchLocale(newlocale):
-    trans = gettext.translation("gettor", "/usr/share/locale", [newlocale], fallback=True)
+    trans = gettext.translation("gettor", 
+                                "/usr/share/locale", 
+                                [newlocale], 
+                                fallback=True)
     trans.install()
 
 def runTests():
@@ -87,11 +89,13 @@
     cronProc.communicate()[0]
     return cronProc.returncode
 
-def processMail(conf, log, logLang, packageList):
-    if len(packageList) < 1:
+def processMail(conf, log, logLang, packageList, blackList, whiteList):
+    if packageList is None or len(packageList) < 1:
         log.error(_("Sorry, your package list is unusable."))
+        log.error(_("Try running with --fetch-packages --prep-packages."))
         return False
-    # Receive mail
+
+    # Receive mail from stdin
     rmail = gettor_requests.requestMail(packageList)
     rawMessage = rmail.getRawMessage()
     if not rawMessage:
@@ -101,8 +105,8 @@
     if not parsedMessage:
         log.error(_("No parsed message. Dropping message."))
         return False
-    # XXX: We should add a blacklist check here so that for exmaple ReplyTo can't be our own 
-    #      address (DoS) (in case we have DKIM) 
+    # XXX: We should add a blacklist check here so that for exmaple ReplyTo 
+    # can't be our own address (DoS) (in case we have DKIM) 
     replyTo = rmail.getReplyTo()
     if not replyTo:
         log.error(_("No help dispatched. Invalid reply address for user."))
@@ -110,26 +114,27 @@
     replyLang = rmail.getLocale()
     if not replyLang:
         replyLang = logLang
+
     # Initialize response
     srcEmail = conf.getSrcEmail()
     resp = gettor_responses.gettorResponse(replyLang, logLang)
     signature = rmail.hasVerifiedSignature()
     log.info(_("Signature is: %s") % str(signature))
-    if not signature:
+    if not signature and not whiteList.lookupListEntry(replyTo):
         # Check to see if we've helped them to understand that they need DKIM
         # in the past
-        previouslyHelped = gettor_blacklist.blackList(replyTo)
+        previouslyHelped = blackList.lookupListEntry(replyTo)
         if previouslyHelped:
             log.info(_("Unsigned messaged to gettor by blacklisted user dropped."))
             return False
         else:
             # Reply with some help and bail out
-            gettor_blacklist.blackList(replyTo, True)
+            blackList.createListEntry(replyTo)
             resp.sendHelp(srcEmail, replyTo)
             log.info(_("Unsigned messaged to gettor. We issued some help."))
             return True
     else:
-        log.info(_("Signed messaged to gettor."))
+        log.info(_("Good message to gettor."))
         package = rmail.getPackage()
         if package != None:
             log.info(_("Package: %s selected.") % str(package))
@@ -140,53 +145,95 @@
 
     return True
 
-if __name__ == "__main__":
+def main():
+    success = None
     # Parse command line, setup config, logging and language
     options, arguments = gettor_opt.parseOpts()
     conf = gettor_config.gettorConf(options.configfile)
-    logger  = gettor_log.gettorLogger()
-    log = logger.getLogger()
+    gettor_log.initialize()
+    log = gettor_log.getLogger()
     logLang = conf.getLocale()
     switchLocale(logLang)
     distDir = conf.getDistDir()
     if not os.path.isdir(distDir):
-        log.info(_("Sorry, %s is not a directory.") % distDir)
-        exit(1)
+        log.error(_("Sorry, %s is not a directory.") % distDir)
+        return False
+    packs = gettor_packages.gettorPackages(options.mirror, conf)
+    whiteList = gettor_blacklist.BWList("/tmp/whitelist")
+    blackList = gettor_blacklist.BWList("/tmp/blacklist")
 
-    packs = gettor_packages.gettorPackages(options.mirror, conf)
-    # Action
     if options.fetchpackages:
         if packs.syncWithMirror() != 0:
             log.error(_("Syncing Tor packages failed."))
-            exit(1)
+            return False
         else:
             log.info(_("Syncing Tor packages done."))
-            exit(0)
+            success = True
     if options.preppackages:
         if not packs.buildPackages():
             log.error(_("Building packages failed."))
-            exit(1)
+            return False
         else:
             log.info(_("Building packages done."))
-            exit(0)
+            success = True
     if options.runtests:
         if not runTests():
             log.error(_("Tests failed."))
-            exit(1)
+            return False
         else:
             log.info(_("Tests passed."))
-            exit(0)
+            success = True
     if options.installcron:
         if installCron(packs.getCommandToStr()) != 0:
             log.error(_("Installing cron failed"))
-            exit(1)
+            return False
         else:
             log.info(_("Installing cron done."))
-            exit(0)
+            success = True
+    if options.whitelist:
+        if not whiteList.createListEntry(options.whitelist):
+            log.error(_("Creating whitelist entry failed."))
+            return False
+        else:
+            log.info(_("Creating whitelist entry ok."))
+            success = True
+    if options.blacklist:
+        if not blackList.createListEntry(options.blacklist):
+            log.error(_("Creating blacklist entry failed."))
+            return False
+        else:
+            log.info(_("Creating blacklist entry ok."))
+            success = True
+    if options.clearwl:
+        if not whiteList.clearAll():
+            log.error(_("Deleting whitelist failed."))
+            return false
+        else:
+            log.info(_("Deleting whitelist done."))
+            success = True
+    if options.clearbl:
+        if not blackList.clearAll():
+            log.error(_("Deleting blacklist failed."))
+            return false
+        else:
+            log.info(_("Deleting blacklist done."))
+            success = True
+
+    # Break here if preparation work has been done
+    if success is not None:
+        return success
     
     # Main loop
-    if not processMail(conf, log, logLang, packs.getPackageList()):
+    if not processMail(conf, log, logLang, packs.getPackageList(), blackList,
+                       whiteList):
         log.error(_("Processing mail failed."))
+        return False
+
+    return True
+
+if __name__ == "__main__":
+    if not main():
+        print >> sys.stderr, _("Main loop exited with errors.")
         exit(1)
-
-    exit(0)
+    else:
+        exit(0)

Modified: projects/gettor/gettor_blacklist.py
===================================================================
--- projects/gettor/gettor_blacklist.py	2008-12-03 20:54:49 UTC (rev 17477)
+++ projects/gettor/gettor_blacklist.py	2008-12-04 09:54:09 UTC (rev 17478)
@@ -6,110 +6,90 @@
 import hashlib
 import os
 import gettor_config
+import gettor_log
 
+log = gettor_log.getLogger()
+
 conf = gettor_config.gettorConf()
 stateDir = conf.getStateDir()
 blStateDir = conf.getBlStateDir()
 
-def blackList(address, createEntry=False):
-    """ 
-    Check to see if an address is on our blacklist. If it is - we'll return true.
-    If requested, we'll attempt to create a blacklist entry and return true if 
-    everything works out.
-    """
-    # XXX TODO: Eventually we may want to sort entries with netcom
-    # style /tmp/gettor/2-digits-of-hash/2-more-digits/rest-of-hash files
+class BWList:
+    def __init__(self, listdir):
+        self.listDir = listdir
+        if not os.path.isdir(self.listDir):
+            log.error(_("Bad dir %s.") % self.listDir)
+            # XXX Change this to something more appropriate
+            raise Exception
 
-    prepBLStateDir()
-
-    privateAddress = makeAddressPrivate(address)
-    blackListed = lookupBlackListEntry(privateAddress)
-    
-    if blackListed:
+    def lookupListEntry(self, address):
+        """Check to see if we have a list entry for the given address."""
+        entry = self.listDir + "/" + str(hashlib.sha1(address).hexdigest())
+        try:
+            entry = os.stat(entry)
+        except OSError:
+            return False
         return True
-    elif createEntry:
-        return createBlackListEntry(privateAddress)
 
-    return False
-
-def lookupBlackListEntry(privateAddress):
-    """ Check to see if we have a blacklist entry for the given address. """
-    entry = stateDir + str(privateAddress)
-    try:
-        entry = os.stat(entry)
-    except OSError:
-        return False
-    return True
-
-def createBlackListEntry(privateAddress):
-    """ Create a zero byte file that represents the address in our blacklist. """
-    entry = blStateDir + str(privateAddress)
-    stat = None
-    try:
-        stat = os.stat(entry)
-    except OSError:
-        try:
-            fd = open(entry, 'w')
-            fd.close()
-            return True
-        except:
+    def createListEntry(self, address):
+        entry = self.listDir + "/" + str(hashlib.sha1(address).hexdigest())
+        if self.lookupListEntry(address) == False:
+            try:
+                fd = open(entry, 'w')
+                fd.close
+                return True
+            except:
+                log.error(_("Creating list entry %s failed.") % entry)
+                return False
+        else:
+            # List entry already exists
             return False
-    return False
 
-def removeBlackListEntry(privateAddress):
-    """ Remove the zero byte file that represents an entry in our blacklist."""
-    entry = blStateDir + str(privateAddress)
-    stat = None
-    try:
-        entry = os.unlink(entry)
-    except OSError:
-        return False
-    return True
-
-def makeAddressPrivate(address):
-    """ Creates a unique identifier for the user. """
-    hash = hashlib.sha1(address)
-    privateAddress = hash.hexdigest()
-    return privateAddress
-
-def prepBLStateDir():
-    stat = None
-    try:
-        stat = os.stat(stateDir)
-        return True
-    except OSError:
-        try:
-            os.makedirs(stateDir)
-            return True
-        except:
+    def removeListEntry(self, address):
+        entry = self.listDir + "/" + str(hashlib.sha1(address).hexdigest())
+        if (self.lookupListEntry(address) == True):
+            try:
+                os.unlink(entry)
+            except OSError:
+                log.error(_("Could not unlink entry %s") % entry)
+                return False
+        else:
+            log.info(_("Requested removal of non-existing entry %s. Abord.") 
+                    % entry)
             return False
 
+    def removeAll(self):
+        print "Removing all entries from list!"
+
 def blackListtests(address):
     """ This is a basic evaluation of our blacklist functionality """
-    prepBLStateDir()
-    privateAddress = makeAddressPrivate(address)
-    print "We have a private address of: "  + privateAddress
-    print "Testing creation of blacklist entry: "
-    blackListEntry = createBlackListEntry(privateAddress)
-    print blackListEntry
-    print "We're testing a lookup of a known positive blacklist entry: "
-    blackListEntry = lookupBlackListEntry(privateAddress)
-    print blackListEntry
-    print "We're testing removal of a known blacklist entry: "
-    blackListEntry = removeBlackListEntry(privateAddress)
-    print blackListEntry
-    print "We're testing a lookup of a known false blacklist entry: "
-    blackListEntry = lookupBlackListEntry(privateAddress)
-    print blackListEntry
-    print "Now we'll test the higher level blacklist functionality..."
-    print "This should not find an entry in the blacklist: "
-    print blackList(address)
-    print "This should add an entry to the blacklist: "
-    print blackList(address, True)
-    print "This should find the previous addition to the blacklist: "
-    print blackList(address)
-    print "Please ensure the tests match what you would expect for your environment."
+    bwlist = BWList("/tmp/") 
+    print bwlist.createListEntry(address)
+    print bwlist.lookupListEntry(address)
+    #prepBLStateDir()
+    #privateAddress = makeAddressPrivate(address)
+    #print "We have a private address of: "  + privateAddress
+    #print "Testing creation of blacklist entry: "
+    #blackListEntry = createBlackListEntry(privateAddress)
+    #print blackListEntry
+    #print "We're testing a lookup of a known positive blacklist entry: "
+    #blackListEntry = lookupBlackListEntry(privateAddress)
+    #print blackListEntry
+    #print "We're testing removal of a known blacklist entry: "
+    #blackListEntry = removeBlackListEntry(privateAddress)
+    #print blackListEntry
+    #print "We're testing a lookup of a known false blacklist entry: "
+    #blackListEntry = lookupBlackListEntry(privateAddress)
+    #print blackListEntry
+    #print "Now we'll test the higher level blacklist functionality..."
+    #print "This should not find an entry in the blacklist: "
+    #print blackList(address)
+    #print "This should add an entry to the blacklist: "
+    #print blackList(address, True)
+    #print "This should find the previous addition to the blacklist: "
+    #print blackList(address)
+    #print "Please ensure the tests match what you would expect for your environment."
 
 if __name__ == "__main__" :
-    print "It appears that you're manually calling the blacklisting code. We'll run some tests..."
+    print "Running some tests.."
     blackListtests("requestingUser at example.com")

Modified: projects/gettor/gettor_config.py
===================================================================
--- projects/gettor/gettor_config.py	2008-12-03 20:54:49 UTC (rev 17477)
+++ projects/gettor/gettor_config.py	2008-12-04 09:54:09 UTC (rev 17478)
@@ -93,23 +93,11 @@
         # We're trying to be failsafe here
         self.outConf = ConfigParser.ConfigParser()
 
-        # See if config file is accessable
         try:
             if os.access(self.configFile, os.R_OK):
-                readableConfigFile = True
-            else:
-                readableConfigFile = False
-
-        except OSError:
-            readableConfigFile = False
-
-        if readableConfigFile: 
-            try:
-                # It's likely that a user will make a mistake in their config
-                # If they make a mistake for now we'll ignore *everything* :-)
                 self.config.read(self.configFile)
-            except:
-                pass
+        except:
+            pass
 
         # Main parser loop:
         # * Merge default values with those from the config file, if there are

Modified: projects/gettor/gettor_log.py
===================================================================
--- projects/gettor/gettor_log.py	2008-12-03 20:54:49 UTC (rev 17477)
+++ projects/gettor/gettor_log.py	2008-12-04 09:54:09 UTC (rev 17478)
@@ -25,59 +25,62 @@
 import gettor_config
 from logging import handlers
 
-__all__ = ["gettorLogger"]
+__all__ = ["initalize", "getLogger", "getLogSubSystem"]
 
 # Leave this to INFO for now
 loglevel = logging.INFO
+format = '%(asctime)-15s (%(process)d) %(message)s'
+logger = None
+logSubSystem = None
 
-class gettorLogger:
-    """A configurable logging system for gettor.
-    """
+def initialize():
+    global logger
+    global logSubSystem
+    config = gettor_config.gettorConf() 
+    logger = logging.getLogger('gettor')
+    logger.setLevel(loglevel)
+    logSubSystem = config.getLogSubSystem()
 
-    format = '%(asctime)-15s (%(process)d) %(message)s'
+    if logSubSystem == "stdout":
+        handler = logging.StreamHandler()
+    elif logSubSystem == "file":
+        # Silently fail if things are misconfigured
+        logFile = config.getLogFile()
+        try:
+            if os.access(os.path.dirname(logFile), os.W_OK):
+                handler = logging.FileHandler(logFile)
+            else:
+                logSubSystem = "nothing"
+        except:
+            logSubSystem = "nothing"
+    elif logSubSystem == "syslog":
+        handler = logging.handlers.SysLogHandler(address="/dev/log")
+    else:
+        # Failsafe fallback
+        logSubSystem = "nothing"
 
-    def __init__(self):  
-        self.config = gettor_config.gettorConf() 
-        self.logger = logging.getLogger('gettor')
-        self.logger.setLevel(loglevel)
-        self.logSubSystem = self.config.getLogSubSystem()
+    # If anything went wrong or the user doesn't want to log
+    if logSubSystem == "nothing":
+        handler = logging.FileHandler(os.devnull)
 
-        if self.logSubSystem == "stdout":
-            handler = logging.StreamHandler()
-        elif self.logSubSystem == "file":
-            # Silently fail if things are misconfigured
-            self.logFile = self.config.getLogFile()
-            try:
-                if os.access(os.path.dirname(self.logFile), os.W_OK):
-                    handler = logging.FileHandler(self.logFile)
-                else:
-                    self.logSubSystem = "nothing"
-            except:
-                self.logSubSystem = "nothing"
-        elif self.logSubSystem == "syslog":
-            handler = logging.handlers.SysLogHandler(address="/dev/log")
-        else:
-            # Failsafe fallback
-            self.logSubSystem = "nothing"
+    formatter = logging.Formatter(fmt=format)
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
 
-        # If anything went wrong or the user doesn't want to log
-        if self.logSubSystem == "nothing":
-            handler = logging.FileHandler(os.devnull)
+def getLogSubSystem():
+    global logSubSystem
+    return logSubSystem
 
-        formatter = logging.Formatter(fmt=self.format)
-        handler.setFormatter(formatter)
-        self.logger.addHandler(handler)
+def getLogger():
+    global logger
+    if logger is None:
+        initialize()
+    return logger
 
-    def getLogSubSystem(self):
-        return self.logSubSystem
-
-    def getLogger(self):
-        return self.logger
-
 if __name__ == "__main__" :
-    l = gettorLogger()
+    initialize()
     print "This is the logging module. You probably do not want to call it by hand."
     print "We'll send a test logging message now with the following subsystem: " + \
-    l.getLogSubSystem()
-    log = l.getLogger()
+    getLogSubSystem()
+    log = getLogger()
     log.info("I'm a logger, logging!")

Modified: projects/gettor/gettor_opt.py
===================================================================
--- projects/gettor/gettor_opt.py	2008-12-03 20:54:49 UTC (rev 17477)
+++ projects/gettor/gettor_opt.py	2008-12-04 09:54:09 UTC (rev 17478)
@@ -23,7 +23,8 @@
                         help="set config file to FILE", metavar="FILE")
     cmdParser.add_option("-m", "--use-mirror", dest="mirror",
                         default="rsync.torproject.org",
-                        help="set Tor package mirror to MIRROR", metavar="MIRROR")
+                        help="set Tor package mirror to MIRROR", 
+                        metavar="MIRROR")
     cmdParser.add_option("-i", "--install-crontab", dest="installcron",
                         action="store_true", default=False,
                         help="install crontab to refresh packagelist")
@@ -36,8 +37,22 @@
     cmdParser.add_option("-t", "--run-tests", dest="runtests",
                         action="store_true", default=False,
                         help="run some tests")
+    cmdParser.add_option("-w", "--whitelist", dest="whitelist",
+                         default="",
+                         help="add an email address to the whitelist",
+                         metavar="WHITELIST")
+    cmdParser.add_option("-b", "--blacklist", dest="blacklist",
+                         default="",
+                         help="add an email address to the blacklist",
+                         metavar="BLACKLIST")
+    cmdParser.add_option("-x", "--clear-whitelist", dest="clearwl",
+                        action="store_true", default=False,
+                        help="clear all entrys in the whitelist")
+    cmdParser.add_option("-y", "--clear-blacklist", dest="clearbl",
+                        action="store_true", default=False,
+                        help="clear all entrys in the blacklist")
 
     return cmdParser.parse_args()
 
 if __name__ == "__main__":
-    print >> sys.stderr "You shouldn't run this directly."
+    print >> sys.stderr, "You shouldn't run this directly."

Modified: projects/gettor/gettor_packages.py
===================================================================
--- projects/gettor/gettor_packages.py	2008-12-03 20:54:49 UTC (rev 17477)
+++ projects/gettor/gettor_packages.py	2008-12-04 09:54:09 UTC (rev 17478)
@@ -20,6 +20,8 @@
 
 __all__ = ["gettorPackages"]
 
+log = gettor_log.getLogger()
+
 class gettorPackages:
 
     packageRegex = { "windows-bundle": "vidalia-bundle-.*.exe$",
@@ -44,8 +46,13 @@
 
     def getPackageList(self):
         # Build dict like 'name': 'name.z'
-        for filename in os.listdir(self.packDir):
-            self.packageList[filename[:-2]] = self.packDir + "/" + filename
+        try:
+            for filename in os.listdir(self.packDir):
+                self.packageList[filename[:-2]] = self.packDir + "/" + filename
+        except OSError, (strerror):
+            log.error(_("Failed to build package list: %s") % strerror)
+            return None
+
         # Check sanity
         for key, val in self.packageList.items():
             # Remove invalid packages
@@ -82,10 +89,7 @@
     def getCommandToStr(self):
         """This is useful for cronjob installations
         """
-        rsyncstr = ""
-        for i,v in enumerate(self.rsync):
-            rsyncstr += self.rsync[i] + " "
-        return rsyncstr
+        return ''.join(self.rsync)
 
 if __name__ == "__main__" :
     c = gettor_config.gettorConf()

Modified: projects/gettor/gettor_responses.py
===================================================================
--- projects/gettor/gettor_responses.py	2008-12-03 20:54:49 UTC (rev 17477)
+++ projects/gettor/gettor_responses.py	2008-12-04 09:54:09 UTC (rev 17478)
@@ -19,6 +19,7 @@
 
 __all__ = ["gettorResponse"]
 
+
 class gettorResponse:
 
     def __init__(self, mailLang="en", logLang="en"):
@@ -27,8 +28,10 @@
 
     def setLang(self, language):
         # XXX: Sorta hack, have nothing better on my mind right now
-        # On every entry to a translation-needing function, call this with lang=maillang
-        # On every exit of a translation-needing function, call this with lang=loglang
+        # On every entry to a translation-needing function, call this with 
+        # lang=maillang
+        # On every exit of a translation-needing function, call this with 
+        # lang=loglang
         # :-/
         trans = gettext.translation("gettor", "/usr/share/locale", [language])
         trans.install()



More information about the tor-commits mailing list