[tor-commits] [gettor/master] Make GetTor installation more standard compliant

commit d24f82f2d8bbb67a965596807831b6455afaaac8 Author: Christian Fromme kaner@strace.org Date: Sat Mar 3 07:01:15 2012 +0100

Make GetTor installation more standard compliant --- GetTor.py | 152 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ README | 58 +++++++++++----------- lib/GetTor.py | 152 --------------------------------------------------------- setup.cfg | 5 +- setup.py | 2 +- 5 files changed, 183 insertions(+), 186 deletions(-)

diff --git a/GetTor.py b/GetTor.py new file mode 100644 index 0000000..d9c378d --- /dev/null +++ b/GetTor.py @@ -0,0 +1,152 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +__program__ = 'GetTor.py' +__url__ = 'https://gitweb.torproject.org/gettor.git' +__author__ = 'Jacob Appelbaum jacob@appelbaum.net, Christian Fromme kaner@strace.org' +__copyright__ = 'Copyright (c) 2008, The Tor Project' +__license__ = 'See LICENSE for licensing information' + +try: + from future import antigravity +except ImportError: + antigravity = None + +import os +import sys +import logging +import traceback +import gettor.opt +import gettor.config +import gettor.requests +import gettor.responses +import gettor.utils +import gettor.filters +from time import strftime + + +def initializeLogging(cfg): + level = getattr(cfg, 'LOGLEVEL', 'WARNING') + level = getattr(logging, level) + extra = {} + logfileName = cfg.LOGFILE + "-" + strftime("%Y-%m-%d") + ".log" + logDir = os.path.join(cfg.BASEDIR, "log") + if not gettor.utils.createDir(logDir): + # Fall back to /tmp in case of an error + logDir="/tmp" + extra['filename'] = os.path.join(logDir, logfileName) + + print "Logfile is %s" % extra['filename'] + + logging.basicConfig(format='%(asctime)s [%(levelname)s] %(message)s', + datefmt="%Y-%m-%d %H:%M:%S", + level=level, + **extra) + +def processFail(conf, rawMessage, failedAction, e=None): + """This routine gets called when something went wrong with the processing + """ + logging.error("Failed to " + failedAction) + if e is not None: + logging.error("%s" % traceback.format_exc()) + logging.error("Here is the exception I saw: %s" % sys.exc_info()[0]) + logging.error("Detail: %s" %e) + if conf.DUMPFILE != "": + # Keep a copy of the failed email for later reference + logging.info("We'll keep a record of this mail.") + dumpFile = os.path.join(conf.BASEDIR, conf.DUMPFILE) + gettor.utils.dumpMessage(dumpFile, rawMessage) + +def processMail(conf): + """All mail processing happens here. Processing goes as follows: + - Parse request. This means: Find out everything we need to reply in + an appropriate manner. Reply address, language, package name. + Also try to find out if the user wants split packages and if he has + a valid signature on his mail. + - Do some filtering voodoo. + - Send reply. Use all information gathered from the request and pass + it on to the reply class/method to decide what to do. + """ + rawMessage = "" + reqInfo = None + logging.debug("Processing mail..") + # Retrieve request from stdin and parse it + try: + request = gettor.requests.requestMail(conf) + rawMessage = request.getRawMessage() + # reqval contains all important information we need from the request + reqInfo = request.parseMail() + except Exception as e: + processFail(conf, rawMessage, "process request", e) + return False + + # Do some filtering on the request + try: + reqInfo = gettor.filters.doFilter(reqInfo) + except Exception as e: + processFail(conf, rawMessage, "filter request", e) + return False + + # See if the filtering process decided if the request was valid + if reqInfo['valid'] is not True: + logging.error("Invalid request.") + processFail(conf, rawMessage, "send reply") + return False + + # Ok, request information aquired. Make it so. + try: + reply = gettor.responses.Response(conf, reqInfo) + if not reply.sendReply(): + processFail(conf, rawMessage, "send reply") + return False + return True + except Exception as e: + processFail(conf, rawMessage, "send reply (got exception)", e) + return False + +def processOptions(options, conf): + """Do everything that's not part of parsing a mail. Prepare GetTor usage, + install files, fetch packages, do some black/whitelist voodoo and so on + """ + + if options.fetchpackages: + gettor.utils.fetchPackages(conf) + if options.preppackages: + gettor.utils.prepPackages(conf) + if options.installcron: + gettor.utils.installCron() + if options.whitelist: + gettor.utils.addWhitelistEntry(conf, options.whitelist) + if options.blacklist: + gettor.utils.addBlacklistEntry(conf, options.blacklist) + if options.lookup: + gettor.utils.lookupAddress(conf, options.lookup) + if options.clearwl: + gettor.utils.clearWhitelist(conf) + if options.days: + gettor.utils.clearBlacklist(conf, options.days) + if options.cmdpass: + gettor.utils.setCmdPassword(conf, options.cmdpass) + +def main(): + """Parse command line, setup config and logging. + """ + options, arguments = gettor.opt.parseOpts() + config = gettor.config.Config(options.configfile) + initializeLogging(config) + + if sys.stdin.isatty(): + # We separate this because we need a way to know how we reply to the + # caller: Send mail/write log file or just dump to stdout/stderr. + processOptions(options, config) + print "Done." + else: + # We've got mail + if processMail(config): + logging.debug("Processing mail finished") + else: + logging.error("Processing mail failed") + logging.debug("Done.") + +if __name__ == "__main__": + main() diff --git a/README b/README index b7a2084..fc1bbf9 100644 --- a/README +++ b/README @@ -25,24 +25,23 @@ in case you need to. SETUP -----

-This setup descripton presumes that you have access to the latest gettor svn -trunk somewhere in your local file system. +This setup descripton presumes that you have access to the latest gettor source +code.

- gettor@hostname:~/tmp$ svn co https://freehaven.net/svn/projects/gettor gettor + gettor@hostname:~/src$ git clone git://git.torproject.org/gettor.git

-After checking out the GetTor source code, you will need to install everything -as follows: +After obtaining the GetTor source code, you will need to install it as follows:

- gettor@hostname:~/tmp$ cd gettor/ - gettor@hostname:~/tmp/gettor$ python setup.py trans - gettor@hostname:~/tmp/gettor$ python setup.py install_data - gettor@hostname:~/tmp/gettor$ python setup.py install + gettor@hostname:~/src$ cd gettor/ + gettor@hostname:~/src/gettor$ python setup.py trans + gettor@hostname:~/src/gettor$ python setup.py install_data + gettor@hostname:~/src/gettor$ python setup.py install

-This will copy necessary files to ~/opt/gettor. +This will copy necessary files to where they're needed.

Once you have everything installed, you'll want to initialize the file store:

- gettor@hostname:~/opt/gettor$ python GetTor.py -fp + gettor@hostname:~/src/gettor$ GetTor.py -fp

You should see an rsync and finally something like:

@@ -54,19 +53,19 @@ You should see an rsync and finally something like: Now you'll install the cronjob. This clears the blacklist and updates packages daily:

- gettor@hostname:~/opt/gettor$ python GetTor.py -i + gettor@hostname:~/src/gettor$ GetTor.py -i

Set up a password for package forwarding commands if you wish to use that feature:

- gettor@hostname:~/opt/gettor$ python GetTor.py -s seCreTpAssworD + gettor@hostname:~/src/gettor$ GetTor.py -s seCreTpAssworD

-The secret password will be stored in whereever `PASSFILE' is configured in -the configuration. +The secret password will be stored as a hash in whereever `PASSFILE' is +configured in the configuration.

Finally, you need to setup email forwarding to the GetTor bot like so:

- gettor@hostname:~$ echo "|python /home/g/opt/gettor/GetTor.py" > ~/.forward + gettor@hostname:~$ echo "| /home/gettor/bin/GetTor.py" > ~/.forward

Now GetTor is installed, prepared and ready to serve files. Send it email!

@@ -75,8 +74,8 @@ TRANSLATION FILES Provided there is a working locale environment, GetTor will compile and setup locale files for you as follows:

- gettor@hostname:~/opt/gettor$ python setup.py trans - gettor@hostname:~/opt/gettor$ python setup.py install_data + gettor@hostname:~/src/gettor$ python setup.py trans + gettor@hostname:~/src/gettor$ python setup.py install_data

To add a new language to GetTor, one must create a new .po file in subversion in the proper directory. For Korean, one would create po/ko/ and add the @@ -95,7 +94,7 @@ and should look similar to this: MAIL_FROM = "GetTor gettor@torproject.org"

# Where it is all based at. Subdirs for GetTor start from here. - BASEDIR = "/home/gettor/gettor" + BASEDIR = "/home/gettor/run"

# Should we send a `Your package will arrive soon, be patient' mail? DELAY_ALERT = True @@ -151,7 +150,7 @@ FILE SIZE ISSUES

Some email providers may not be happy with the files GetTor sends. Gmail provides documentation of their incoming and outgoing file sizes. It is -currently 20MB (though we have sent around 21MB without issue): +currently 25MB (though we have sent around 26MB without issue):

http://mail.google.com/support/bin/answer.py?hl=en&answer=8770

@@ -177,22 +176,21 @@ unique email address is allowed to mail GetTor the same type of mail once each sendSplitPackage. Blacklists are stored in the form of hashed email files under directories. In reality, that looks as follows:

- gettor@hostname:~/opt/gettor$ ls -d /home/gettor/gettor/bl/* - /home/gettor/gettor/bl/general - /home/gettor/gettor/bl/sendPackage - /home/gettor/gettor/bl/sendSplitPackage - /home/gettor/gettor/bl/sendDelayAlert - /home/gettor/gettor/bl/sendPackageHelp + gettor@hostname:~/$ ls -d /home/gettor/run/bl/* + /home/gettor/run/bl/general + /home/gettor/run/bl/sendPackage + /home/gettor/run/bl/sendSplitPackage + /home/gettor/run/bl/sendDelayAlert + /home/gettor/run/bl/sendPackageHelp

- gettor@hostname:~/opt/gettor$ ls /home/gettor/gettor/bl/sendDelayAlert \ - | head -n 2 + gettor@hostname:~/$ ls /home/gettor/run/bl/sendDelayAlert | head -n 2 0154d8584c0afa6290e21098e7ab4cc635b7d50a 02a33e16feece8671f1274de62de32068a67cf20

In addition to this automatic blacklisting mechanism, there is the possibility to add blacklist entries by hand as follows:

- gettor@hostname:~/opt/gettor$ python GetTor.py -b someone@evil.org + gettor@hostname:~/$ GetTor.py -b someone@evil.org

Email addresses that are added this way, go to the general/ directory in the blacklist directory and will therefore be blocked from using GetTor in any way. @@ -200,7 +198,7 @@ blacklist directory and will therefore be blocked from using GetTor in any way. Besides the blacklisting mechanism, there is a whitelisting mechanism. It works analogous to the manual blacklisting mechanism:

- gettor@hostname:~/opt/gettor$ python GetTor.py -w someone@heaven.org + gettor@hostname:~/$ GetTor.py -w someone@heaven.org

Whitelisting wins over blacklisting. If a user if blacklisted for X, but also whitelisted, he will be allowed to do X. diff --git a/lib/GetTor.py b/lib/GetTor.py deleted file mode 100644 index d9c378d..0000000 --- a/lib/GetTor.py +++ /dev/null @@ -1,152 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -__program__ = 'GetTor.py' -__url__ = 'https://gitweb.torproject.org/gettor.git' -__author__ = 'Jacob Appelbaum jacob@appelbaum.net, Christian Fromme kaner@strace.org' -__copyright__ = 'Copyright (c) 2008, The Tor Project' -__license__ = 'See LICENSE for licensing information' - -try: - from future import antigravity -except ImportError: - antigravity = None - -import os -import sys -import logging -import traceback -import gettor.opt -import gettor.config -import gettor.requests -import gettor.responses -import gettor.utils -import gettor.filters -from time import strftime - - -def initializeLogging(cfg): - level = getattr(cfg, 'LOGLEVEL', 'WARNING') - level = getattr(logging, level) - extra = {} - logfileName = cfg.LOGFILE + "-" + strftime("%Y-%m-%d") + ".log" - logDir = os.path.join(cfg.BASEDIR, "log") - if not gettor.utils.createDir(logDir): - # Fall back to /tmp in case of an error - logDir="/tmp" - extra['filename'] = os.path.join(logDir, logfileName) - - print "Logfile is %s" % extra['filename'] - - logging.basicConfig(format='%(asctime)s [%(levelname)s] %(message)s', - datefmt="%Y-%m-%d %H:%M:%S", - level=level, - **extra) - -def processFail(conf, rawMessage, failedAction, e=None): - """This routine gets called when something went wrong with the processing - """ - logging.error("Failed to " + failedAction) - if e is not None: - logging.error("%s" % traceback.format_exc()) - logging.error("Here is the exception I saw: %s" % sys.exc_info()[0]) - logging.error("Detail: %s" %e) - if conf.DUMPFILE != "": - # Keep a copy of the failed email for later reference - logging.info("We'll keep a record of this mail.") - dumpFile = os.path.join(conf.BASEDIR, conf.DUMPFILE) - gettor.utils.dumpMessage(dumpFile, rawMessage) - -def processMail(conf): - """All mail processing happens here. Processing goes as follows: - - Parse request. This means: Find out everything we need to reply in - an appropriate manner. Reply address, language, package name. - Also try to find out if the user wants split packages and if he has - a valid signature on his mail. - - Do some filtering voodoo. - - Send reply. Use all information gathered from the request and pass - it on to the reply class/method to decide what to do. - """ - rawMessage = "" - reqInfo = None - logging.debug("Processing mail..") - # Retrieve request from stdin and parse it - try: - request = gettor.requests.requestMail(conf) - rawMessage = request.getRawMessage() - # reqval contains all important information we need from the request - reqInfo = request.parseMail() - except Exception as e: - processFail(conf, rawMessage, "process request", e) - return False - - # Do some filtering on the request - try: - reqInfo = gettor.filters.doFilter(reqInfo) - except Exception as e: - processFail(conf, rawMessage, "filter request", e) - return False - - # See if the filtering process decided if the request was valid - if reqInfo['valid'] is not True: - logging.error("Invalid request.") - processFail(conf, rawMessage, "send reply") - return False - - # Ok, request information aquired. Make it so. - try: - reply = gettor.responses.Response(conf, reqInfo) - if not reply.sendReply(): - processFail(conf, rawMessage, "send reply") - return False - return True - except Exception as e: - processFail(conf, rawMessage, "send reply (got exception)", e) - return False - -def processOptions(options, conf): - """Do everything that's not part of parsing a mail. Prepare GetTor usage, - install files, fetch packages, do some black/whitelist voodoo and so on - """ - - if options.fetchpackages: - gettor.utils.fetchPackages(conf) - if options.preppackages: - gettor.utils.prepPackages(conf) - if options.installcron: - gettor.utils.installCron() - if options.whitelist: - gettor.utils.addWhitelistEntry(conf, options.whitelist) - if options.blacklist: - gettor.utils.addBlacklistEntry(conf, options.blacklist) - if options.lookup: - gettor.utils.lookupAddress(conf, options.lookup) - if options.clearwl: - gettor.utils.clearWhitelist(conf) - if options.days: - gettor.utils.clearBlacklist(conf, options.days) - if options.cmdpass: - gettor.utils.setCmdPassword(conf, options.cmdpass) - -def main(): - """Parse command line, setup config and logging. - """ - options, arguments = gettor.opt.parseOpts() - config = gettor.config.Config(options.configfile) - initializeLogging(config) - - if sys.stdin.isatty(): - # We separate this because we need a way to know how we reply to the - # caller: Send mail/write log file or just dump to stdout/stderr. - processOptions(options, config) - print "Done." - else: - # We've got mail - if processMail(config): - logging.debug("Processing mail finished") - else: - logging.error("Processing mail failed") - logging.debug("Done.") - -if __name__ == "__main__": - main() diff --git a/setup.cfg b/setup.cfg index aa9f4ae..9fa8c68 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,4 +1,3 @@ [install] -install-purelib=$HOME/opt/gettor -install-scripts=$HOME/opt/gettor -install-data=$HOME/opt/gettor +prefix=$HOME +install-data=$HOME diff --git a/setup.py b/setup.py index a88aed5..28064ff 100644 --- a/setup.py +++ b/setup.py @@ -104,7 +104,7 @@ setup(name='GetTor', url='https://www.torproject.org/gettor/', package_dir={'': 'lib'}, packages=['gettor'], - scripts = ["MakeStat.py"], + scripts = ["GetTor.py", "MakeStat.py"], py_modules=['GetTor'], long_description = """Really long text here.""", cmdclass={'trans': createTrans,