commit 6081a27dfc38634eb1f47c462d2bf9c6aab599d8 Author: aagbsn aagbsn@extc.org Date: Mon Jul 25 17:04:14 2011 -0700

1860 - email rate limiting support

This set of changes implements email rate-limiting for the EmailBasedDistributor. Abusers are warned, and then temporarily blacklisted. --- lib/bridgedb/Dist.py | 20 +++++++++++++++++++- lib/bridgedb/I18n.py | 9 ++++++++- lib/bridgedb/Server.py | 37 +++++++++++++++++++++++++++++++++++++ lib/bridgedb/Storage.py | 37 +++++++++++++++++++++++++++++++++++++ lib/bridgedb/Tests.py | 47 ++++++++++++++++++++++++++++++++++++++++++++++- 5 files changed, 147 insertions(+), 3 deletions(-)

diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index 5394cfa..030c7fa 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -146,6 +146,10 @@ class TooSoonEmail(BadEmail): """Raised when we got a request from this address too recently.""" pass

+class IgnoreEmail(BadEmail): + """Raised when we get requests from this address after rate warning.""" + pass + def extractAddrSpec(addr): """Given an email From line, try to extract and parse the addrspec portion. Returns localpart,domain on success; raises BadEmail @@ -257,14 +261,27 @@ class EmailBasedDistributor(bridgedb.Bridges.BridgeHolder): return [] #XXXX raise an exception.

db = bridgedb.Storage.getDB() + wasWarned = db.getWarnedEmail(emailaddress)

lastSaw = db.getEmailTime(emailaddress) if lastSaw is not None and lastSaw + MAX_EMAIL_RATE >= now: + if wasWarned: + logging.warn("Got a request for bridges from %r; we already " + "sent a warning. Ignoring.", emailaddress) + raise IgnoreEmail("Client was warned", emailaddress) + else: + db.setWarnedEmail(emailaddress, True, now) + db.commit() + logging.warn("Got a request for bridges from %r; we already " - "answered one within the last %d seconds. Ignoring.", + "answered one within the last %d seconds. Warning.", emailaddress, MAX_EMAIL_RATE) raise TooSoonEmail("Too many emails; wait till later", emailaddress)

+ # warning period is over + elif wasWarned: + db.setWarnedEmail(emailaddress, False) + pos = self.emailHmac("<%s>%s" % (epoch, emailaddress)) result = self.ring.getBridges(pos, N, countryCode)

@@ -279,6 +296,7 @@ class EmailBasedDistributor(bridgedb.Bridges.BridgeHolder): db = bridgedb.Storage.getDB() try: db.cleanEmailedBridges(time.time()-MAX_EMAIL_RATE) + db.cleanWarnedBridges(time.time()-MAX_EMAIL_RATE) except: db.rollback() raise diff --git a/lib/bridgedb/I18n.py b/lib/bridgedb/I18n.py index 44adb8c..750ca4a 100644 --- a/lib/bridgedb/I18n.py +++ b/lib/bridgedb/I18n.py @@ -51,5 +51,12 @@ bridge addresses."""), # BRIDGEDB_TEXT[8] _("""(e-mail requests not currently supported)"""), # BRIDGEDB_TEXT[9] - _("""(Might be blocked)""") + _("""To receive your bridge relays, please prove you are human"""), + # BRIDGEDB_TEXT[10] + _("""You have exceeded the rate limit. Please slow down, the minimum time between +emails is: """), + # BRIDGEDB_TEXT[11] + _("""hours"""), + # BRIDGEDB_TEXT[12] + _("""All further emails will be ignored.""") ] diff --git a/lib/bridgedb/Server.py b/lib/bridgedb/Server.py index 8ec659e..9cbeb15 100644 --- a/lib/bridgedb/Server.py +++ b/lib/bridgedb/Server.py @@ -274,6 +274,35 @@ def getMailResponse(lines, ctx): clientAddr, e) return None, None

+ # Handle rate limited email + except bridgedb.Dist.TooSoonEmail, e: + logging.info("Got a mail too frequently; warning %r: %s.", + clientAddr, e) + + # Compose a warning email + f = StringIO() + w = MimeWriter.MimeWriter(f) + w.addheader("From", ctx.fromAddr) + w.addheader("To", clientAddr) + w.addheader("Message-ID", twisted.mail.smtp.messageid()) + if not subject.startswith("Re:"): subject = "Re: %s"%subject + w.addheader("Subject", subject) + if msgID: + w.addheader("In-Reply-To", msgID) + w.addheader("Date", twisted.mail.smtp.rfc822date()) + body = w.startbody("text/plain") + + # MAX_EMAIL_RATE is in seconds, convert to hours + EMAIL_MESSAGE_RATELIMIT = buildSpamWarningTemplate(t) + body.write(EMAIL_MESSAGE_RATELIMIT % (bridgedb.Dist.MAX_EMAIL_RATE / 3600)) + f.seek(0) + return clientAddr, f + + except bridgedb.Dist.IgnoreEmail, e: + logging.info("Got a mail too frequently; ignoring %r: %s.", + clientAddr, e) + return None, None + # Generate the message. f = StringIO() w = MimeWriter.MimeWriter(f) @@ -311,6 +340,14 @@ def buildMessageTemplate(t):

return msg_template

+def buildSpamWarningTemplate(t): + msg_template = t.gettext(I18n.BRIDGEDB_TEXT[5]) + "\n\n" \ + + t.gettext(I18n.BRIDGEDB_TEXT[10]) + "\n\n" \ + + "%s " \ + + t.gettext(I18n.BRIDGEDB_TEXT[11]) + "\n\n" \ + + t.gettext(I18n.BRIDGEDB_TEXT[12]) + "\n\n" + return msg_template + def replyToMail(lines, ctx): """Given a list of lines from an incoming email message, and a MailContext object, possibly send a reply. diff --git a/lib/bridgedb/Storage.py b/lib/bridgedb/Storage.py index 5d42930..da64a30 100644 --- a/lib/bridgedb/Storage.py +++ b/lib/bridgedb/Storage.py @@ -8,6 +8,7 @@ import logging import binascii import sqlite3 import time +import sha

toHex = binascii.b2a_hex fromHex = binascii.a2b_hex @@ -144,6 +145,13 @@ SCHEMA2_SCRIPT = """

CREATE INDEX BlockedBridgesBlockingCountry on BlockedBridges(hex_key);

+ CREATE TABLE WarnedEmails ( + email PRIMARY KEY NOT NULL, + when_warned + ); + + CREATE INDEX WarnedEmailsWasWarned on WarnedEmails ( email ); + INSERT INTO Config VALUES ( 'schema-version', 2 ); """

@@ -227,6 +235,7 @@ class Database: cur.execute("DELETE FROM EmailedBridges WHERE when_mailed < ?", (t,))

def getEmailTime(self, addr): + addr = sha.new(addr).hexdigest() cur = self._cur cur.execute("SELECT when_mailed FROM EmailedBridges WHERE " "email = ?", (addr,)) @@ -236,6 +245,7 @@ class Database: return strToTime(v[0])

def setEmailTime(self, addr, whenMailed): + addr = sha.new(addr).hexdigest() cur = self._cur t = timeToStr(whenMailed) cur.execute("INSERT OR REPLACE INTO EmailedBridges " @@ -321,6 +331,33 @@ class Database: return False return True

+ def getWarnedEmail(self, addr): + addr = sha.new(addr).hexdigest() + cur = self._cur + cur.execute("SELECT * FROM WarnedEmails WHERE " + " email = ?", (addr,)) + v = cur.fetchone() + if v is None: + return False + return True + + def setWarnedEmail(self, addr, warned=True, whenWarned=time.time()): + addr = sha.new(addr).hexdigest() + t = timeToStr(whenWarned) + cur = self._cur + if warned == True: + cur.execute("INSERT INTO WarnedEmails" + "(email,when_warned) VALUES (?,?)", (addr, t,)) + elif warned == False: + cur.execute("DELETE FROM WarnedEmails WHERE " + "email = ?", (addr,)) + + def cleanWarnedEmails(self, expireBefore): + cur = self._cur + t = timeToStr(expireBefore) + + cur.execute("DELETE FROM WarnedEmails WHERE when_warned < ?", (t,)) + def openDatabase(sqlite_file): conn = sqlite3.Connection(sqlite_file) cur = conn.cursor() diff --git a/lib/bridgedb/Tests.py b/lib/bridgedb/Tests.py index 558fedf..9648f86 100644 --- a/lib/bridgedb/Tests.py +++ b/lib/bridgedb/Tests.py @@ -45,6 +45,41 @@ class RhymesWith255Category: def contains(self, ip): return ip.endswith(".255")

+class EmailBridgeDistTests(unittest.TestCase): + def setUp(self): + self.fd, self.fname = tempfile.mkstemp() + self.db = bridgedb.Storage.Database(self.fname) + bridgedb.Storage.setGlobalDB(self.db) + self.cur = self.db._conn.cursor() + + def tearDown(self): + self.db.close() + os.close(self.fd) + os.unlink(self.fname) + + def testEmailRateLimit(self): + db = self.db + EMAIL_DOMAIN_MAP = {'example.com':'example.com'} + d = bridgedb.Dist.EmailBasedDistributor( + "Foo", + {'example.com': 'example.com', + 'dkim.example.com': 'dkim.example.com'}, + {'example.com': [], 'dkim.example.com': ['dkim']}) + for _ in xrange(256): + d.insert(fakeBridge()) + d.getBridgesForEmail('abc@example.com', 1, 3) + self.assertRaises(bridgedb.Dist.TooSoonEmail, + d.getBridgesForEmail, 'abc@example.com', 1, 3) + self.assertRaises(bridgedb.Dist.IgnoreEmail, + d.getBridgesForEmail, 'abc@example.com', 1, 3) + + def testUnsupportedDomain(self): + db = self.db + self.assertRaises(bridgedb.Dist.UnsupportedDomain, + bridgedb.Dist.normalizeEmail, 'bad@email.com', + {'example.com':'example.com'}, + {'example.com':[]}) + class IPBridgeDistTests(unittest.TestCase): def dumbAreaMapper(self, ip): return ip @@ -250,12 +285,22 @@ class SQLStorageTests(unittest.TestCase):

self.assertEquals(set(db.getBlockingCountries(b2.fingerprint)), set(['uk', 'cn', 'de', 'jp', 'se', 'kr'])) + self.assertEquals(db.getWarnedEmail("def@example.com"), False) + db.setWarnedEmail("def@example.com") + self.assertEquals(db.getWarnedEmail("def@example.com"), True) + db.setWarnedEmail("def@example.com", False) + self.assertEquals(db.getWarnedEmail("def@example.com"), False) + + db.setWarnedEmail("def@example.com") + self.assertEquals(db.getWarnedEmail("def@example.com"), True) + db.cleanWarnedEmails(t+200) + self.assertEquals(db.getWarnedEmail("def@example.com"), False)

def testSuite(): suite = unittest.TestSuite() loader = unittest.TestLoader()

- for klass in [ IPBridgeDistTests, DictStorageTests, SQLStorageTests ]: + for klass in [ IPBridgeDistTests, DictStorageTests, SQLStorageTests, EmailBridgeDistTests ]: suite.addTest(loader.loadTestsFromTestCase(klass))

for module in [ bridgedb.Bridges,