[or-cvs] r18534: {weather} Initial branch check in. (in weather/branches/sqlite-unstable: . apache2)
ioerror at seul.org
ioerror at seul.org
Fri Feb 13 22:43:16 UTC 2009
Author: ioerror
Date: 2009-02-13 17:43:16 -0500 (Fri, 13 Feb 2009)
New Revision: 18534
Added:
weather/branches/sqlite-unstable/apache2/
weather/branches/sqlite-unstable/apache2/tor-weather
weather/branches/sqlite-unstable/apache2/tor-weather-https
weather/branches/sqlite-unstable/setup-db.py
weather/branches/sqlite-unstable/weatherdb.py
Modified:
weather/branches/sqlite-unstable/config.py
weather/branches/sqlite-unstable/poll.py
weather/branches/sqlite-unstable/weather.py
Log:
Initial branch check in.
Added: weather/branches/sqlite-unstable/apache2/tor-weather
===================================================================
--- weather/branches/sqlite-unstable/apache2/tor-weather (rev 0)
+++ weather/branches/sqlite-unstable/apache2/tor-weather 2009-02-13 22:43:16 UTC (rev 18534)
@@ -0,0 +1,17 @@
+<VirtualHost *>
+ ServerAdmin tor-ops at torproject.org
+ ServerName weather.torproject.org
+ ServerAlias weather.torproject.org
+
+ Redirect / https://weather.torproject.org/
+
+ LogFormat "%t \"%r\" %>s %b" noip
+ CustomLog /var/log/apache2/tor-weather-https-redirect-access.log noip
+
+ # Possible values include: debug, info, notice, warn, error, crit,
+ # alert, emerg.
+ LogLevel warn
+
+ ServerSignature Off
+
+</VirtualHost>
Added: weather/branches/sqlite-unstable/apache2/tor-weather-https
===================================================================
--- weather/branches/sqlite-unstable/apache2/tor-weather-https (rev 0)
+++ weather/branches/sqlite-unstable/apache2/tor-weather-https 2009-02-13 22:43:16 UTC (rev 18534)
@@ -0,0 +1,53 @@
+<VirtualHost 209.237.247.85:443>
+ ServerAdmin tor-ops at torproject.org
+ ServerName weather.torproject.org
+ ServerAlias weather.torproject.org
+
+ SSLEngine on
+ SSLCertificateFile /etc/apache2/ssl/weather.torproject.org.cert
+ SSLCertificateKeyFile /etc/apache2/ssl/weather.torproject.org.key
+ SSLProtocol all
+ SSLCipherSuite HIGH:MEDIUM
+
+ DocumentRoot /var/www/tor-weather/
+ DirectoryIndex /weather.py/
+
+ <Directory "/var/www/tor-weather/">
+
+ # This gives us pretty and reasonable urls
+ RewriteEngine on
+ RewriteBase /
+
+ # Required for the images and css
+ RewriteRule ^(top-right.png)$ - [L]
+ RewriteRule ^(top-left.png)$ - [L]
+ RewriteRule ^(top-middle.png)$ - [L]
+ RewriteRule ^(stylesheet.css)$ - [L]
+ RewriteRule ^(favicon.ico)$ - [L]
+
+ # Required for the actual python app
+ RewriteCond %{REQUEST_URI} !^(/.*)+weather.py/
+ RewriteRule ^(.*)$ weather.py/$1 [PT]
+
+ # Disallow .htaccess trickery
+ AllowOverride None
+ Order allow,deny
+ Allow from all
+
+ # Ensure the proper handler for weather.py
+ <Files weather.py>
+ SetHandler fastcgi-script
+ </Files>
+
+ </Directory>
+
+ LogFormat "%t \"%r\" %>s %b" noip
+ CustomLog /var/log/apache2/tor-weather-https-access.log noip
+
+ # Possible values include: debug, info, notice, warn, error, crit,
+ # alert, emerg.
+ LogLevel warn
+
+ ServerSignature Off
+
+</VirtualHost>
Modified: weather/branches/sqlite-unstable/config.py
===================================================================
--- weather/branches/sqlite-unstable/config.py 2009-02-13 19:50:42 UTC (rev 18533)
+++ weather/branches/sqlite-unstable/config.py 2009-02-13 22:43:16 UTC (rev 18534)
@@ -2,12 +2,15 @@
URLbase = "https://weather.torproject.org"
-weather_storage = "/var/lib/torweather/"
+#weather_storage = "/var/lib/torweather/"
+weather_storage = "/tmp/"
authenticator = open(weather_storage + "auth_token").read().strip()
weather_email = "tor-ops at torproject.org"
+sqlitedb = weather_storage + "/weather.sqlite3"
+
# these respond to pings (for now!) and are geographically dispersed
ping_targets = ["google.com", "telstra.com.au", "yahoo.co.uk"]
Modified: weather/branches/sqlite-unstable/poll.py
===================================================================
--- weather/branches/sqlite-unstable/poll.py 2009-02-13 19:50:42 UTC (rev 18533)
+++ weather/branches/sqlite-unstable/poll.py 2009-02-13 22:43:16 UTC (rev 18534)
@@ -2,7 +2,6 @@
import socket
import sys
import os
-import gdbm
import re
import time
import threading
@@ -10,6 +9,7 @@
from traceback import print_exception
from subprocess import Popen, PIPE
import TorCtl.TorCtl as TorCtl
+import sqlite3
from config import authenticator, URLbase, weather_email, failure_threshold
from config import poll_period, ping_targets, weather_storage
@@ -19,9 +19,8 @@
import traceback
debug = 0
-
-
debugfile = open(weather_storage + "/torctl-debug","w")
+
class TorPing:
"Check to see if various tor nodes respond to SSL hanshakes"
def __init__(self, control_host = "127.0.0.1", control_port = 9051):
@@ -58,7 +57,7 @@
# ErrorReply: 552 Unrecognized key "ns/id/46D9..."
# This means that the node isn't recognized by
x = traceback.format_exc()
- fh = file("/tmp/tor-ctl-failed-ping-log", "a")
+ fh = file(weather_storage + "/tor-ctl-failed-ping-log", "a")
fh.write(x)
fh.close()
info = None
@@ -67,7 +66,7 @@
except:
# Remove this, it's a hack to debug this specific bug
x = traceback.format_exc()
- fh = file("/tmp/misc-failed-ping-log", "a")
+ fh = file(weather_storage + "/misc-failed-ping-log", "a")
fh.write(x)
fh.close()
info = None
@@ -76,30 +75,6 @@
# If we're here, we were able to fetch information about the router
return True
- # info looks like this:
- # {'ns/id/FFCB46DB1339DA84674C70D7CB586434C4370441': 'r moria1 /8tG2xM52oRnTHDXy1hkNMQ3BEE pavoLDqxMvw+T1VHR5hmmgpr9self 2007-10-10 21:12:08 128.31.0.34 9001 9031\ns Authority Fast Named Running Valid V2Dir\n'}
- ##ip,port = info[string].split()[6:8]
- # throw exceptions like confetti if this isn't legit
- ##socket.inet_aton(ip)
- # ensure port is a kosher string
- ##assert 0 < int(port) < 65536
-
- ##if debug: print "contacting node at %s:%s" % (ip,port)
-
- # XXX check: could non-blocking io be used to make us safe from
- # answer-very-slowly DOSes? or do we need to spawn threads here?
-
- ##cmd = ["openssl", "s_client", "-connect", ip + ':' + port]
- ##ssl_handshake = Popen( args = cmd, stdout = PIPE, stderr = PIPE, stdin=PIPE)
- ##ssl_handshake.stdin.close()
- ##safe_from_DOS = 10000 # moria1's response is ~1500 chars long
- ##output = ssl_handshake.stdout.read(safe_from_DOS)
- ##n = output.find("Server public key is 1024 bit")
- ##if n > 0:
- ## return True
- ##else:
- ## return False
-
def test(self):
"Check that the connection to the Tor Control port is still okay."
try:
Added: weather/branches/sqlite-unstable/setup-db.py
===================================================================
--- weather/branches/sqlite-unstable/setup-db.py (rev 0)
+++ weather/branches/sqlite-unstable/setup-db.py 2009-02-13 22:43:16 UTC (rev 18534)
@@ -0,0 +1,76 @@
+#!/usr/bin/env python2.5
+#
+# A small program to setup the database and test that things are working
+#
+
+from weatherdb import *
+from config import sqlitedb
+
+# Connect to our sqlite3 database
+con = create_database_handle(sqlitedb)
+# Setup our tables if they're not already created
+intialize_database(con)
+
+# Create a cursor
+cur = con.cursor()
+
+TABLE_NAME = 'nodes'
+SELECT = 'select * from %s' % TABLE_NAME
+
+# Execute our SELECT statement:
+cur.execute(SELECT)
+
+# Print a header.
+FIELD_MAX_WIDTH = 10
+for fieldDesc in cur.description:
+ print fieldDesc[0].ljust(FIELD_MAX_WIDTH) ,
+print # Finish the header with a newline.
+print '-' * 78
+
+# Print the contents
+fieldIndices = range(len(cur.description))
+for row in cur:
+ for fieldIndex in fieldIndices:
+ fieldValue = str(row[fieldIndex])
+ print fieldValue.ljust(FIELD_MAX_WIDTH) ,
+print
+
+#destroy_database(con)
+#exit()
+
+# Populate with basic information about a node
+fingerprint = "9B0511125922B3FAB37C84B5E695623DD107A74D"
+fingerprint1 = "9B05AA745922B3FAB37C84B5E695623DD107A74D"
+fingerprint2 = "9B02AAd45922B3FAB37C84B5E695623DD107A74D"
+fingerprint3 = "9B02AA445922B3FAB37C84B5E695623DD107A74C"
+#con.execute("INSERT INTO nodes (fingerprint, strikes) values (?,?)", (fingerprint, 0,))
+#add_node(con, fingerprint1)
+#add_node(con, fingerprint2)
+add_node(cur, fingerprint3)
+
+# Return that information in a useful way
+nodes = fetch_nodes(con)
+print "Total number of nodes we were able to fetch: " + nodes
+
+# Fetch the node strike
+strikes = fetch_node_strikes(con, fingerprint1)
+foo = str(strikes)
+print "Strikes against node: " + foo + " "
+
+# Strike a node three times
+increment_node_strike(con, fingerprint1)
+increment_node_strike(con, fingerprint1)
+increment_node_strike(con, fingerprint1)
+
+# Fetch the node striking
+strikes = fetch_node_strikes(con, fingerprint1)
+foo = str(strikes)
+print "Strikes against node: " + foo + " "
+
+#
+# Drop all the tables and destroy everything
+#destroy_database(con)
+
+# Tear that connection down and exit
+teardown_database(con)
+exit()
Property changes on: weather/branches/sqlite-unstable/setup-db.py
___________________________________________________________________
Added: svn:executable
+ *
Modified: weather/branches/sqlite-unstable/weather.py
===================================================================
--- weather/branches/sqlite-unstable/weather.py 2009-02-13 19:50:42 UTC (rev 18533)
+++ weather/branches/sqlite-unstable/weather.py 2009-02-13 22:43:16 UTC (rev 18534)
@@ -5,13 +5,16 @@
import re
import random
import sys
-import gdbm
import time
import threading
import signal # does this help with keyboard interrupts?
import base64
+import smtplib
+import datetime
+from email.mime.text import MIMEText
+from config import URLbase, weather_email, weather_storage, apache_fcgi, sqlitedb
+from database import *
-from config import URLbase, weather_email, weather_storage, apache_fcgi
debug = 0
dummy_testing = 0
@@ -32,71 +35,27 @@
# Should do something more elegant with this!
if __name__ == "__main__":
-# This is a single lock for all the gdbm write rights, to ensure that
-# different web.py threads aren't trying to write at the same time.
-
try:
+ # XXX TODO Have a mode for creating the database table, etc
+ # Create an sqlite3 database connection
+ conn = sqlite.connect('/tmp/sqlitedb')
+ conn.isolation_level = "IMMEDIATE"
+ db = conn.cursor()
+ # Initialize the database tables
+ intialize_database(db)
- gdbm_lock = threading.RLock()
+ antispam_lock = threading.RLock()
+ antispam = {} # a dict mapping IP to the number of recent unanswered requests allowed
+ # from that IP
+ antispam_min = 2
+ antispam_max = 10
- requests = gdbm.open(weather_storage + "/requests.gdbm","cs")
- print "requests:"
- for s in requests.keys():
- print s, requests[s]
- subscriptions = gdbm.open(weather_storage + "/subscriptions.gdbm","cs")
- print "subscriptions:"
- for s in subscriptions.keys():
- print s, '"'+subscriptions[s]+'"'
- unsubscriptions = gdbm.open(weather_storage + "/unsubscriptions.gdbm","cs")
- print "unsubscriptions:"
- for s in unsubscriptions.keys():
- print s, unsubscriptions[s]
-
- failures = gdbm.open(weather_storage + "/failures.gdbm", "cs")
- print "failures:"
- for s in failures.keys():
- print s, failures[s]
-
- antispam_lock = threading.RLock()
- antispam = {} # a dict mapping IP to the number of recent unanswered requests allowed
- # from that IP
- antispam_min = 2
- antispam_max = 10
-
except:
- print "Unable to get lock on database"
+ print "Unable to open database."
exit()
-# these may or may not be better than storing pickles with gdbm
-
-class DatabaseError(Exception):
- pass
-
-def parse_subscriptions(node, subs):
- "Turn a string in the db back into a list of pairs"
- words = subs[node].split()
- try:
- return [ (words[i], words[i+1]) for i in xrange(0, len(words), 2) ]
- except IndexError:
- raise DatabaseError, words
-
-def delete_sub(pair, sub, node):
- "Craziness to delete pair from a string in the subscriptions db"
- # regexps probably aren't easily made safe here
- words = sub[node].split()
- if (len(words) % 2 != 0):
- raise DatabaseError, words
- for n in xrange(len(words) / 2):
- if pair[0] == words[n*2] and pair[1] == words[n*2 + 1]:
- sub[node] = " ".join(words[:n*2] + words[n*2 + 2:])
- break
- else:
- raise DatabaseError, pair
- sub.sync()
-
def randstring():
- # This is where we sometimes return '-' and we shouldn't
- "Produce a random alphanumeric string for authentication"
+ "Produce an mua friendly random alphanumeric string for authentication"
theory = base64.urlsafe_b64encode(os.urandom(18))[:-1]
if theory[-1] == "-": # some email clients don't like URLs ending in -
theory[-1] = 'x'
@@ -120,6 +79,7 @@
def GET(self):
web.header('content-type', 'text/html')
+ # XXX TODO: we probably want to open this with an absolute path
print open("subscribe.template").read()
whitespace = re.compile("\s*")
@@ -152,18 +112,17 @@
# Leak no information about who is subscribed
print "Thank you for using Tor Weather. A confirmation request has been sent to", i.email + "."
- # node ids are 40 digit hexidecimal numbers
- node_okay = re.compile("(0x)?[a-fA-F0-9]{40}\Z")
-
- def check_node_id(self, node):
- if self.node_okay.match(node):
+ def check_node_id(self, nodeid):
+ # Node ids are 40 digit hexidecimal numbers
+ node_okay = re.compile("(0x)?[a-fA-F0-9]{40}\Z")
+ if node_okay.match(nodeid):
return True
else:
return False
- random.seed()
def allowed_to_subscribe(self,ip):
"An antispam measure!"
+ random.seed()
antispam_lock.acquire()
try:
if antispam.has_key(ip):
@@ -179,24 +138,23 @@
finally:
antispam_lock.release()
- def already_subscribed(self, address, node):
- gdbm_lock.acquire()
- try:
-
- try:
- words = subscriptions[node].split()
- if address in words:
+ def already_subscribed(db, email, nodeid):
+ try:
+ subscriptions = parse_subscriptions(db, nodeid)
+ # TODO XXX
+ # Iterate over subscriptions and return True if we match an email
+ if # TODO XXX
already = True
- else:
+ else:
already = False
- except KeyError:
- already = False
+ except KeyError:
+ already = False
- finally:
- gdbm_lock.release()
return already
def send_confirmation_email(self, address, node):
+ # XXX TODO update to use sqlite
+ # We should pull this from the database rather than generating it here
authstring = randstring()
gdbm_lock.acquire()
@@ -205,19 +163,11 @@
finally:
gdbm_lock.release()
- if dummy_testing:
- print "gotcha"
- return True
-
- #def f(s):
- # requests[authstring] = s
- #gdbm_lock.lock(f, address + " " + node)
-
- #url = web.ctx.homedomain + "/confirm-subscribe/" + authstring
url = URLbase + "/confirm-subscribe/" + authstring
- import smtplib
- from email.mime.text import MIMEText
+ # God this is ugly
+ #import smtplib
+ #from email.mime.text import MIMEText
msg= MIMEText(subscribe_text % (node, url))
s = smtplib.SMTP()
s.connect()
@@ -228,20 +178,18 @@
s.sendmail(sender, [address], msg.as_string())
s.close()
- print "Thank you for using Tor Weather. A confirmation request has been sent to", address + "."
- #print url
+ print "Thank you for using Tor Weather. A confirmation request has been sent to ", address + "."
+ def check_email(self, address):
+ "Just check that address is not something fruity"
# Section 3.4.1 of RFC 2822 is much more liberal than this!
domain_okay = re.compile("[A-Za-z0-9\-\.']*")
local_okay = re.compile("[A-Za-z0-9\-_\.\+]*")
querinator=DNS.Request(qtype='mx')
email_error = None
+ # This is wrong (see http://www.ex-parrot.com/~pdw/Mail-RFC822-Address.html)
+ # but it should prevent crazy stuff from being accepted
- def check_email(self, address):
- "Just check that address is not something fruity"
- # This is wrong (see http://www.ex-parrot.com/~pdw/Mail-RFC822-Address.html)
- # but it should prevent crazy stuff from being accepted
-
if len(address) >= 80:
self.email_error = "We declare this address too long"
return False
Added: weather/branches/sqlite-unstable/weatherdb.py
===================================================================
--- weather/branches/sqlite-unstable/weatherdb.py (rev 0)
+++ weather/branches/sqlite-unstable/weatherdb.py 2009-02-13 22:43:16 UTC (rev 18534)
@@ -0,0 +1,126 @@
+#!/usr/bin/env python2.5
+
+from pysqlite2 import dbapi2 as sqlite
+
+class DatabaseError(Exception):
+ pass
+
+def create_database_handle(sqlitedb):
+ try:
+ con = sqlite.connect(database=sqlitedb, timeout=10.0)
+ # We want to autocommit everything
+ con.isolation_level = "None"
+ except:
+ raise DatabaseError
+ return con
+
+def intialize_database(db):
+ "Initalize the database tables"
+ try:
+ db.execute('''CREATE TABLE IF NOT EXISTS nodes (
+ id integer primary key autoincrement,
+ fingerprint text,
+ strikes double not null,
+ first_seen date,
+ last_seen date )''')
+ db.execute('''CREATE TABLE IF NOT EXISTS subscriptions (
+ id integer primary key autoincrement,
+ fingerprint text,
+ email text,
+ comment text,
+ signup_date default current_timestamp,
+ auth_token text,
+ query_interval double not null,
+ confirmed integer not null,
+ alert_status integer not null )''')
+ except:
+ print "Attempts to initialize the database tables have failed."
+ raise DatabaseError
+
+def destroy_database(db):
+ "Destroy all data and all tables"
+ try:
+ db.execute('''DROP TABLE nodes''')
+ db.execute('''DROP TABLE subscriptions''')
+ except:
+ print "Attempts to destroy all data and all tables have failed."
+
+def teardown_database(db):
+ try:
+ db.close()
+ except:
+ "Unable to destroy database connection"
+ raise DatabaseError
+
+def fetch_node_subscriptions(db, fingerprint):
+ "Return a list of emails subscribed to a given node id"
+ try:
+ subscriptions = db.execute('SELECT email from subscriptions where fingerprint = ?', (fingerprint,))
+ return subscriptions.fetchall()
+ except:
+ raise DatabaseError
+
+def fetch_nodes(db):
+ "Return a list of all node fingerprints in the database"
+ try:
+ nodes = db.execute('SELECT fingerprint from nodes')
+ return nodes.fetchall()
+ except:
+ raise DatabaseError
+
+def delete_subscriptions_with_token(db, auth_token):
+ "Delete subscription from database that matches the given auth token"
+ # XXX TODO: Check that the token is valid first, if it's there delete it
+ try:
+ # XXX TODO: Ensure we erase everything related to a given subscription
+ subscriptions = db.execute('DELETE email, node from subscriptions where auth_token = ?', auth_token)
+ except:
+ raise DatabaseError
+ return True
+
+def fetch_node_subscriptions_with_token(db, auth_token):
+ "Return node id and email for a given auth token"
+ try:
+ subscriptions = db.execute('SELECT email from subscriptions where token = ?', auth_token)
+ return subscriptions.fetchall()
+ except:
+ raise DatabaseError
+
+def insert_node_subscription(db, fingerprint, email, auth_token):
+ # XXX TODO: Insert subscription into the database, ensure data is cleaned
+ "Insert node, email and token subscription information"
+ try:
+ insertion = db.execute('INSERT nodeid, email, token # TODO', token)
+ return True
+ except:
+ return False
+
+def add_node(db, fingerprint, strikes=0):
+ # XXX finish this
+ "Add a node into the database and set it's strike count to zero"
+ try:
+ print "Attempting to add fingerprint: "
+ print fingerprint + " with " + str(strikes) + " number of strikes..."
+ now = datetime.datetime.now()
+ later = datetime.datetime.now()
+ db.execute("INSERT INTO nodes (id, fingerprint, strikes, first_seen, last_seen) values (?,?,?,?,?)", (0, fingerprint, strikes, now, later))
+ except:
+ print "Unable to add node fingerprint to database"
+ raise DatabaseError
+
+def increment_node_strike(db, fingerprint):
+ "Increment number of strikes for a given fingerprint by one"
+ try:
+ strikes = db.execute('UPDATE strikes where fingerprint == ?', (fingerprint,))
+ except:
+ raise DatabaseError
+ print "Unable to strike node"
+
+def fetch_node_strikes(db, fingerprint):
+ "Fetch the number of strikes for a given fingeprint"
+ try:
+ strikes = db.execute('SELECT strikes from nodes where fingerprint=?', (fingerprint,))
+ return strikes.fetchone()
+ except:
+ print "Unable to fetch strikes for fingerprint: %s", fingerprint
+ raise DatabaseError
More information about the tor-commits
mailing list