[tor-commits] [gettor/master] Make GetTor installation more standard compliant

kaner at torproject.org kaner at torproject.org
Sat Mar 3 06:02:27 UTC 2012


commit d24f82f2d8bbb67a965596807831b6455afaaac8
Author: Christian Fromme <kaner at strace.org>
Date:   Sat Mar 3 07:01:15 2012 +0100

    Make GetTor installation more standard compliant
---
 GetTor.py     |  152 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 README        |   58 +++++++++++-----------
 lib/GetTor.py |  152 ---------------------------------------------------------
 setup.cfg     |    5 +-
 setup.py      |    2 +-
 5 files changed, 183 insertions(+), 186 deletions(-)

diff --git a/GetTor.py b/GetTor.py
new file mode 100644
index 0000000..d9c378d
--- /dev/null
+++ b/GetTor.py
@@ -0,0 +1,152 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+__program__ = 'GetTor.py'
+__url__ = 'https://gitweb.torproject.org/gettor.git'
+__author__ = 'Jacob Appelbaum <jacob at appelbaum.net>, Christian Fromme <kaner at strace.org>'
+__copyright__ = 'Copyright (c) 2008, The Tor Project'
+__license__ = 'See LICENSE for licensing information'
+
+try:
+    from future import antigravity
+except ImportError:
+    antigravity = None
+
+import os
+import sys
+import logging
+import traceback
+import gettor.opt
+import gettor.config
+import gettor.requests
+import gettor.responses
+import gettor.utils
+import gettor.filters
+from time import strftime
+
+
+def initializeLogging(cfg):
+    level = getattr(cfg, 'LOGLEVEL', 'WARNING')
+    level = getattr(logging, level)
+    extra = {}
+    logfileName = cfg.LOGFILE + "-" +  strftime("%Y-%m-%d") + ".log"
+    logDir = os.path.join(cfg.BASEDIR, "log")
+    if not gettor.utils.createDir(logDir):
+        # Fall back to /tmp in case of an error
+        logDir="/tmp"
+    extra['filename'] = os.path.join(logDir, logfileName)
+
+    print "Logfile is %s" % extra['filename']
+
+    logging.basicConfig(format='%(asctime)s [%(levelname)s] %(message)s',
+                        datefmt="%Y-%m-%d %H:%M:%S",
+                        level=level,
+                        **extra)
+
+def processFail(conf, rawMessage, failedAction, e=None):
+    """This routine gets called when something went wrong with the processing
+    """
+    logging.error("Failed to " + failedAction)
+    if e is not None:
+        logging.error("%s" % traceback.format_exc())
+        logging.error("Here is the exception I saw: %s" % sys.exc_info()[0])
+        logging.error("Detail: %s" %e)
+    if conf.DUMPFILE != "":
+        # Keep a copy of the failed email for later reference
+        logging.info("We'll keep a record of this mail.")
+        dumpFile = os.path.join(conf.BASEDIR, conf.DUMPFILE)
+        gettor.utils.dumpMessage(dumpFile, rawMessage)
+
+def processMail(conf):
+    """All mail processing happens here. Processing goes as follows:
+       - Parse request. This means: Find out everything we need to reply in 
+         an appropriate manner. Reply address, language, package name.
+         Also try to find out if the user wants split packages and if he has 
+         a valid signature on his mail.
+       - Do some filtering voodoo.
+       - Send reply. Use all information gathered from the request and pass
+         it on to the reply class/method to decide what to do.
+    """
+    rawMessage = ""
+    reqInfo = None
+    logging.debug("Processing mail..")
+    # Retrieve request from stdin and parse it
+    try:
+        request = gettor.requests.requestMail(conf)
+        rawMessage = request.getRawMessage()
+        # reqval contains all important information we need from the request
+        reqInfo = request.parseMail()
+    except Exception as e:
+        processFail(conf, rawMessage, "process request", e)
+        return False
+
+    # Do some filtering on the request
+    try:
+        reqInfo = gettor.filters.doFilter(reqInfo)
+    except Exception as e:
+        processFail(conf, rawMessage, "filter request", e)
+        return False
+
+    # See if the filtering process decided if the request was valid
+    if reqInfo['valid'] is not True:
+        logging.error("Invalid request.")
+        processFail(conf, rawMessage, "send reply")
+        return False
+
+    # Ok, request information aquired. Make it so.
+    try:
+        reply = gettor.responses.Response(conf, reqInfo)
+        if not reply.sendReply():
+            processFail(conf, rawMessage, "send reply")
+            return False
+        return True
+    except Exception as e:
+        processFail(conf, rawMessage, "send reply (got exception)", e)
+        return False
+
+def processOptions(options, conf):
+    """Do everything that's not part of parsing a mail. Prepare GetTor usage,
+       install files, fetch packages, do some black/whitelist voodoo and so on
+    """ 
+
+    if options.fetchpackages:
+        gettor.utils.fetchPackages(conf)
+    if options.preppackages:
+       gettor.utils.prepPackages(conf)
+    if options.installcron:
+        gettor.utils.installCron()
+    if options.whitelist:
+        gettor.utils.addWhitelistEntry(conf, options.whitelist)
+    if options.blacklist:
+        gettor.utils.addBlacklistEntry(conf, options.blacklist)
+    if options.lookup:
+        gettor.utils.lookupAddress(conf, options.lookup)
+    if options.clearwl:
+        gettor.utils.clearWhitelist(conf)
+    if options.days:
+        gettor.utils.clearBlacklist(conf, options.days)
+    if options.cmdpass:
+        gettor.utils.setCmdPassword(conf, options.cmdpass)
+
+def main():
+    """Parse command line, setup config and logging.
+    """
+    options, arguments = gettor.opt.parseOpts()
+    config = gettor.config.Config(options.configfile)
+    initializeLogging(config)
+
+    if sys.stdin.isatty():
+        # We separate this because we need a way to know how we reply to the 
+        # caller: Send mail/write log file or just dump to stdout/stderr.
+        processOptions(options, config)
+        print "Done."
+    else:
+        # We've got mail
+        if processMail(config):
+            logging.debug("Processing mail finished")
+        else:
+            logging.error("Processing mail failed")
+        logging.debug("Done.")
+
+if __name__ == "__main__":
+    main()
diff --git a/README b/README
index b7a2084..fc1bbf9 100644
--- a/README
+++ b/README
@@ -25,24 +25,23 @@ in case you need to.
 SETUP
 -----
 
-This setup descripton presumes that you have access to the latest gettor svn 
-trunk somewhere in your local file system.
+This setup descripton presumes that you have access to the latest gettor source
+code.
 
-    gettor at hostname:~/tmp$ svn co https://freehaven.net/svn/projects/gettor gettor
+    gettor at hostname:~/src$ git clone git://git.torproject.org/gettor.git
 
-After checking out the GetTor source code, you will need to install everything
-as follows:
+After obtaining the GetTor source code, you will need to install it as follows:
 
-    gettor at hostname:~/tmp$ cd gettor/
-    gettor at hostname:~/tmp/gettor$ python setup.py trans
-    gettor at hostname:~/tmp/gettor$ python setup.py install_data
-    gettor at hostname:~/tmp/gettor$ python setup.py install
+    gettor at hostname:~/src$ cd gettor/
+    gettor at hostname:~/src/gettor$ python setup.py trans
+    gettor at hostname:~/src/gettor$ python setup.py install_data
+    gettor at hostname:~/src/gettor$ python setup.py install
 
-This will copy necessary files to ~/opt/gettor.
+This will copy necessary files to where they're needed.
 
 Once you have everything installed, you'll want to initialize the file store:
 
-    gettor at hostname:~/opt/gettor$ python GetTor.py -fp
+    gettor at hostname:~/src/gettor$ GetTor.py -fp
 
 You should see an rsync and finally something like:
 
@@ -54,19 +53,19 @@ You should see an rsync and finally something like:
 Now you'll install the cronjob. This clears the blacklist and updates packages 
 daily:
 
-    gettor at hostname:~/opt/gettor$ python GetTor.py -i
+    gettor at hostname:~/src/gettor$ GetTor.py -i
 
 Set up a password for package forwarding commands if you wish to use that 
 feature:
 
-    gettor at hostname:~/opt/gettor$ python GetTor.py -s seCreTpAssworD
+    gettor at hostname:~/src/gettor$ GetTor.py -s seCreTpAssworD
 
-The secret password will be stored in whereever `PASSFILE' is configured in 
-the configuration.
+The secret password will be stored as a hash in whereever `PASSFILE' is 
+configured in the configuration.
 
 Finally, you need to setup email forwarding to the GetTor bot like so:
 
-    gettor at hostname:~$ echo "|python /home/g/opt/gettor/GetTor.py" > ~/.forward
+    gettor at hostname:~$ echo "| /home/gettor/bin/GetTor.py" > ~/.forward
 
 Now GetTor is installed, prepared and ready to serve files. Send it email!
 
@@ -75,8 +74,8 @@ TRANSLATION FILES
 Provided there is a working locale environment, GetTor will compile and setup
 locale files for you as follows:
 
-    gettor at hostname:~/opt/gettor$ python setup.py trans
-    gettor at hostname:~/opt/gettor$ python setup.py install_data
+    gettor at hostname:~/src/gettor$ python setup.py trans
+    gettor at hostname:~/src/gettor$ python setup.py install_data
 
 To add a new language to GetTor, one must create a new .po file in subversion
 in the proper directory. For Korean, one would create po/ko/ and add the
@@ -95,7 +94,7 @@ and should look similar to this:
     MAIL_FROM = "GetTor <gettor at torproject.org>"
 
     # Where it is all based at. Subdirs for GetTor start from here.
-    BASEDIR = "/home/gettor/gettor"
+    BASEDIR = "/home/gettor/run"
 
     # Should we send a `Your package will arrive soon, be patient' mail?
     DELAY_ALERT = True
@@ -151,7 +150,7 @@ FILE SIZE ISSUES
 
 Some email providers may not be happy with the files GetTor sends.
 Gmail provides documentation of their incoming and outgoing file sizes. It is
-currently 20MB (though we have sent around 21MB without issue):
+currently 25MB (though we have sent around 26MB without issue):
 
 	http://mail.google.com/support/bin/answer.py?hl=en&answer=8770
 
@@ -177,22 +176,21 @@ unique email address is allowed to mail GetTor the same type of mail once each
 sendSplitPackage. Blacklists are stored in the form of hashed email files under
 directories. In reality, that looks as follows:
 
-    gettor at hostname:~/opt/gettor$ ls -d /home/gettor/gettor/bl/*
-        /home/gettor/gettor/bl/general
-        /home/gettor/gettor/bl/sendPackage
-        /home/gettor/gettor/bl/sendSplitPackage
-        /home/gettor/gettor/bl/sendDelayAlert
-        /home/gettor/gettor/bl/sendPackageHelp
+    gettor at hostname:~/$ ls -d /home/gettor/run/bl/*
+        /home/gettor/run/bl/general
+        /home/gettor/run/bl/sendPackage
+        /home/gettor/run/bl/sendSplitPackage
+        /home/gettor/run/bl/sendDelayAlert
+        /home/gettor/run/bl/sendPackageHelp
 
-    gettor at hostname:~/opt/gettor$ ls /home/gettor/gettor/bl/sendDelayAlert \
-                                                                    | head -n 2
+    gettor at hostname:~/$ ls /home/gettor/run/bl/sendDelayAlert | head -n 2
      0154d8584c0afa6290e21098e7ab4cc635b7d50a
      02a33e16feece8671f1274de62de32068a67cf20
 
 In addition to this automatic blacklisting mechanism, there is the possibility 
 to add blacklist entries by hand as follows:
 
-    gettor at hostname:~/opt/gettor$ python GetTor.py -b someone at evil.org
+    gettor at hostname:~/$ GetTor.py -b someone at evil.org
 
 Email addresses that are added this way, go to the general/ directory in the 
 blacklist directory and will therefore be blocked from using GetTor in any way.
@@ -200,7 +198,7 @@ blacklist directory and will therefore be blocked from using GetTor in any way.
 Besides the blacklisting mechanism, there is a whitelisting mechanism. It works
 analogous to the manual blacklisting mechanism:
 
-    gettor at hostname:~/opt/gettor$ python GetTor.py -w someone at heaven.org
+    gettor at hostname:~/$ GetTor.py -w someone at heaven.org
 
 Whitelisting wins over blacklisting. If a user if blacklisted for X, but also
 whitelisted, he will be allowed to do X.
diff --git a/lib/GetTor.py b/lib/GetTor.py
deleted file mode 100644
index d9c378d..0000000
--- a/lib/GetTor.py
+++ /dev/null
@@ -1,152 +0,0 @@
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-
-__program__ = 'GetTor.py'
-__url__ = 'https://gitweb.torproject.org/gettor.git'
-__author__ = 'Jacob Appelbaum <jacob at appelbaum.net>, Christian Fromme <kaner at strace.org>'
-__copyright__ = 'Copyright (c) 2008, The Tor Project'
-__license__ = 'See LICENSE for licensing information'
-
-try:
-    from future import antigravity
-except ImportError:
-    antigravity = None
-
-import os
-import sys
-import logging
-import traceback
-import gettor.opt
-import gettor.config
-import gettor.requests
-import gettor.responses
-import gettor.utils
-import gettor.filters
-from time import strftime
-
-
-def initializeLogging(cfg):
-    level = getattr(cfg, 'LOGLEVEL', 'WARNING')
-    level = getattr(logging, level)
-    extra = {}
-    logfileName = cfg.LOGFILE + "-" +  strftime("%Y-%m-%d") + ".log"
-    logDir = os.path.join(cfg.BASEDIR, "log")
-    if not gettor.utils.createDir(logDir):
-        # Fall back to /tmp in case of an error
-        logDir="/tmp"
-    extra['filename'] = os.path.join(logDir, logfileName)
-
-    print "Logfile is %s" % extra['filename']
-
-    logging.basicConfig(format='%(asctime)s [%(levelname)s] %(message)s',
-                        datefmt="%Y-%m-%d %H:%M:%S",
-                        level=level,
-                        **extra)
-
-def processFail(conf, rawMessage, failedAction, e=None):
-    """This routine gets called when something went wrong with the processing
-    """
-    logging.error("Failed to " + failedAction)
-    if e is not None:
-        logging.error("%s" % traceback.format_exc())
-        logging.error("Here is the exception I saw: %s" % sys.exc_info()[0])
-        logging.error("Detail: %s" %e)
-    if conf.DUMPFILE != "":
-        # Keep a copy of the failed email for later reference
-        logging.info("We'll keep a record of this mail.")
-        dumpFile = os.path.join(conf.BASEDIR, conf.DUMPFILE)
-        gettor.utils.dumpMessage(dumpFile, rawMessage)
-
-def processMail(conf):
-    """All mail processing happens here. Processing goes as follows:
-       - Parse request. This means: Find out everything we need to reply in 
-         an appropriate manner. Reply address, language, package name.
-         Also try to find out if the user wants split packages and if he has 
-         a valid signature on his mail.
-       - Do some filtering voodoo.
-       - Send reply. Use all information gathered from the request and pass
-         it on to the reply class/method to decide what to do.
-    """
-    rawMessage = ""
-    reqInfo = None
-    logging.debug("Processing mail..")
-    # Retrieve request from stdin and parse it
-    try:
-        request = gettor.requests.requestMail(conf)
-        rawMessage = request.getRawMessage()
-        # reqval contains all important information we need from the request
-        reqInfo = request.parseMail()
-    except Exception as e:
-        processFail(conf, rawMessage, "process request", e)
-        return False
-
-    # Do some filtering on the request
-    try:
-        reqInfo = gettor.filters.doFilter(reqInfo)
-    except Exception as e:
-        processFail(conf, rawMessage, "filter request", e)
-        return False
-
-    # See if the filtering process decided if the request was valid
-    if reqInfo['valid'] is not True:
-        logging.error("Invalid request.")
-        processFail(conf, rawMessage, "send reply")
-        return False
-
-    # Ok, request information aquired. Make it so.
-    try:
-        reply = gettor.responses.Response(conf, reqInfo)
-        if not reply.sendReply():
-            processFail(conf, rawMessage, "send reply")
-            return False
-        return True
-    except Exception as e:
-        processFail(conf, rawMessage, "send reply (got exception)", e)
-        return False
-
-def processOptions(options, conf):
-    """Do everything that's not part of parsing a mail. Prepare GetTor usage,
-       install files, fetch packages, do some black/whitelist voodoo and so on
-    """ 
-
-    if options.fetchpackages:
-        gettor.utils.fetchPackages(conf)
-    if options.preppackages:
-       gettor.utils.prepPackages(conf)
-    if options.installcron:
-        gettor.utils.installCron()
-    if options.whitelist:
-        gettor.utils.addWhitelistEntry(conf, options.whitelist)
-    if options.blacklist:
-        gettor.utils.addBlacklistEntry(conf, options.blacklist)
-    if options.lookup:
-        gettor.utils.lookupAddress(conf, options.lookup)
-    if options.clearwl:
-        gettor.utils.clearWhitelist(conf)
-    if options.days:
-        gettor.utils.clearBlacklist(conf, options.days)
-    if options.cmdpass:
-        gettor.utils.setCmdPassword(conf, options.cmdpass)
-
-def main():
-    """Parse command line, setup config and logging.
-    """
-    options, arguments = gettor.opt.parseOpts()
-    config = gettor.config.Config(options.configfile)
-    initializeLogging(config)
-
-    if sys.stdin.isatty():
-        # We separate this because we need a way to know how we reply to the 
-        # caller: Send mail/write log file or just dump to stdout/stderr.
-        processOptions(options, config)
-        print "Done."
-    else:
-        # We've got mail
-        if processMail(config):
-            logging.debug("Processing mail finished")
-        else:
-            logging.error("Processing mail failed")
-        logging.debug("Done.")
-
-if __name__ == "__main__":
-    main()
diff --git a/setup.cfg b/setup.cfg
index aa9f4ae..9fa8c68 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,4 +1,3 @@
 [install]
-install-purelib=$HOME/opt/gettor
-install-scripts=$HOME/opt/gettor
-install-data=$HOME/opt/gettor
+prefix=$HOME
+install-data=$HOME
diff --git a/setup.py b/setup.py
index a88aed5..28064ff 100644
--- a/setup.py
+++ b/setup.py
@@ -104,7 +104,7 @@ setup(name='GetTor',
       url='https://www.torproject.org/gettor/',
       package_dir={'': 'lib'},
       packages=['gettor'],
-      scripts = ["MakeStat.py"],
+      scripts = ["GetTor.py", "MakeStat.py"],
       py_modules=['GetTor'],
       long_description = """Really long text here.""",
       cmdclass={'trans': createTrans,



More information about the tor-commits mailing list