[tor-commits] [bridgedb/master] 1860 - email rate limiting support
arma at torproject.org
arma at torproject.org
Mon Sep 19 01:10:01 UTC 2011
commit 6081a27dfc38634eb1f47c462d2bf9c6aab599d8
Author: aagbsn <aagbsn at extc.org>
Date: Mon Jul 25 17:04:14 2011 -0700
1860 - email rate limiting support
This set of changes implements email rate-limiting for the
EmailBasedDistributor. Abusers are warned, and then temporarily
blacklisted.
---
lib/bridgedb/Dist.py | 20 +++++++++++++++++++-
lib/bridgedb/I18n.py | 9 ++++++++-
lib/bridgedb/Server.py | 37 +++++++++++++++++++++++++++++++++++++
lib/bridgedb/Storage.py | 37 +++++++++++++++++++++++++++++++++++++
lib/bridgedb/Tests.py | 47 ++++++++++++++++++++++++++++++++++++++++++++++-
5 files changed, 147 insertions(+), 3 deletions(-)
diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py
index 5394cfa..030c7fa 100644
--- a/lib/bridgedb/Dist.py
+++ b/lib/bridgedb/Dist.py
@@ -146,6 +146,10 @@ class TooSoonEmail(BadEmail):
"""Raised when we got a request from this address too recently."""
pass
+class IgnoreEmail(BadEmail):
+ """Raised when we get requests from this address after rate warning."""
+ pass
+
def extractAddrSpec(addr):
"""Given an email From line, try to extract and parse the addrspec
portion. Returns localpart,domain on success; raises BadEmail
@@ -257,14 +261,27 @@ class EmailBasedDistributor(bridgedb.Bridges.BridgeHolder):
return [] #XXXX raise an exception.
db = bridgedb.Storage.getDB()
+ wasWarned = db.getWarnedEmail(emailaddress)
lastSaw = db.getEmailTime(emailaddress)
if lastSaw is not None and lastSaw + MAX_EMAIL_RATE >= now:
+ if wasWarned:
+ logging.warn("Got a request for bridges from %r; we already "
+ "sent a warning. Ignoring.", emailaddress)
+ raise IgnoreEmail("Client was warned", emailaddress)
+ else:
+ db.setWarnedEmail(emailaddress, True, now)
+ db.commit()
+
logging.warn("Got a request for bridges from %r; we already "
- "answered one within the last %d seconds. Ignoring.",
+ "answered one within the last %d seconds. Warning.",
emailaddress, MAX_EMAIL_RATE)
raise TooSoonEmail("Too many emails; wait till later", emailaddress)
+ # warning period is over
+ elif wasWarned:
+ db.setWarnedEmail(emailaddress, False)
+
pos = self.emailHmac("<%s>%s" % (epoch, emailaddress))
result = self.ring.getBridges(pos, N, countryCode)
@@ -279,6 +296,7 @@ class EmailBasedDistributor(bridgedb.Bridges.BridgeHolder):
db = bridgedb.Storage.getDB()
try:
db.cleanEmailedBridges(time.time()-MAX_EMAIL_RATE)
+ db.cleanWarnedBridges(time.time()-MAX_EMAIL_RATE)
except:
db.rollback()
raise
diff --git a/lib/bridgedb/I18n.py b/lib/bridgedb/I18n.py
index 44adb8c..750ca4a 100644
--- a/lib/bridgedb/I18n.py
+++ b/lib/bridgedb/I18n.py
@@ -51,5 +51,12 @@ bridge addresses."""),
# BRIDGEDB_TEXT[8]
_("""(e-mail requests not currently supported)"""),
# BRIDGEDB_TEXT[9]
- _("""(Might be blocked)""")
+ _("""To receive your bridge relays, please prove you are human"""),
+ # BRIDGEDB_TEXT[10]
+ _("""You have exceeded the rate limit. Please slow down, the minimum time between
+emails is: """),
+ # BRIDGEDB_TEXT[11]
+ _("""hours"""),
+ # BRIDGEDB_TEXT[12]
+ _("""All further emails will be ignored.""")
]
diff --git a/lib/bridgedb/Server.py b/lib/bridgedb/Server.py
index 8ec659e..9cbeb15 100644
--- a/lib/bridgedb/Server.py
+++ b/lib/bridgedb/Server.py
@@ -274,6 +274,35 @@ def getMailResponse(lines, ctx):
clientAddr, e)
return None, None
+ # Handle rate limited email
+ except bridgedb.Dist.TooSoonEmail, e:
+ logging.info("Got a mail too frequently; warning %r: %s.",
+ clientAddr, e)
+
+ # Compose a warning email
+ f = StringIO()
+ w = MimeWriter.MimeWriter(f)
+ w.addheader("From", ctx.fromAddr)
+ w.addheader("To", clientAddr)
+ w.addheader("Message-ID", twisted.mail.smtp.messageid())
+ if not subject.startswith("Re:"): subject = "Re: %s"%subject
+ w.addheader("Subject", subject)
+ if msgID:
+ w.addheader("In-Reply-To", msgID)
+ w.addheader("Date", twisted.mail.smtp.rfc822date())
+ body = w.startbody("text/plain")
+
+ # MAX_EMAIL_RATE is in seconds, convert to hours
+ EMAIL_MESSAGE_RATELIMIT = buildSpamWarningTemplate(t)
+ body.write(EMAIL_MESSAGE_RATELIMIT % (bridgedb.Dist.MAX_EMAIL_RATE / 3600))
+ f.seek(0)
+ return clientAddr, f
+
+ except bridgedb.Dist.IgnoreEmail, e:
+ logging.info("Got a mail too frequently; ignoring %r: %s.",
+ clientAddr, e)
+ return None, None
+
# Generate the message.
f = StringIO()
w = MimeWriter.MimeWriter(f)
@@ -311,6 +340,14 @@ def buildMessageTemplate(t):
return msg_template
+def buildSpamWarningTemplate(t):
+ msg_template = t.gettext(I18n.BRIDGEDB_TEXT[5]) + "\n\n" \
+ + t.gettext(I18n.BRIDGEDB_TEXT[10]) + "\n\n" \
+ + "%s " \
+ + t.gettext(I18n.BRIDGEDB_TEXT[11]) + "\n\n" \
+ + t.gettext(I18n.BRIDGEDB_TEXT[12]) + "\n\n"
+ return msg_template
+
def replyToMail(lines, ctx):
"""Given a list of lines from an incoming email message, and a
MailContext object, possibly send a reply.
diff --git a/lib/bridgedb/Storage.py b/lib/bridgedb/Storage.py
index 5d42930..da64a30 100644
--- a/lib/bridgedb/Storage.py
+++ b/lib/bridgedb/Storage.py
@@ -8,6 +8,7 @@ import logging
import binascii
import sqlite3
import time
+import sha
toHex = binascii.b2a_hex
fromHex = binascii.a2b_hex
@@ -144,6 +145,13 @@ SCHEMA2_SCRIPT = """
CREATE INDEX BlockedBridgesBlockingCountry on BlockedBridges(hex_key);
+ CREATE TABLE WarnedEmails (
+ email PRIMARY KEY NOT NULL,
+ when_warned
+ );
+
+ CREATE INDEX WarnedEmailsWasWarned on WarnedEmails ( email );
+
INSERT INTO Config VALUES ( 'schema-version', 2 );
"""
@@ -227,6 +235,7 @@ class Database:
cur.execute("DELETE FROM EmailedBridges WHERE when_mailed < ?", (t,))
def getEmailTime(self, addr):
+ addr = sha.new(addr).hexdigest()
cur = self._cur
cur.execute("SELECT when_mailed FROM EmailedBridges WHERE "
"email = ?", (addr,))
@@ -236,6 +245,7 @@ class Database:
return strToTime(v[0])
def setEmailTime(self, addr, whenMailed):
+ addr = sha.new(addr).hexdigest()
cur = self._cur
t = timeToStr(whenMailed)
cur.execute("INSERT OR REPLACE INTO EmailedBridges "
@@ -321,6 +331,33 @@ class Database:
return False
return True
+ def getWarnedEmail(self, addr):
+ addr = sha.new(addr).hexdigest()
+ cur = self._cur
+ cur.execute("SELECT * FROM WarnedEmails WHERE "
+ " email = ?", (addr,))
+ v = cur.fetchone()
+ if v is None:
+ return False
+ return True
+
+ def setWarnedEmail(self, addr, warned=True, whenWarned=time.time()):
+ addr = sha.new(addr).hexdigest()
+ t = timeToStr(whenWarned)
+ cur = self._cur
+ if warned == True:
+ cur.execute("INSERT INTO WarnedEmails"
+ "(email,when_warned) VALUES (?,?)", (addr, t,))
+ elif warned == False:
+ cur.execute("DELETE FROM WarnedEmails WHERE "
+ "email = ?", (addr,))
+
+ def cleanWarnedEmails(self, expireBefore):
+ cur = self._cur
+ t = timeToStr(expireBefore)
+
+ cur.execute("DELETE FROM WarnedEmails WHERE when_warned < ?", (t,))
+
def openDatabase(sqlite_file):
conn = sqlite3.Connection(sqlite_file)
cur = conn.cursor()
diff --git a/lib/bridgedb/Tests.py b/lib/bridgedb/Tests.py
index 558fedf..9648f86 100644
--- a/lib/bridgedb/Tests.py
+++ b/lib/bridgedb/Tests.py
@@ -45,6 +45,41 @@ class RhymesWith255Category:
def contains(self, ip):
return ip.endswith(".255")
+class EmailBridgeDistTests(unittest.TestCase):
+ def setUp(self):
+ self.fd, self.fname = tempfile.mkstemp()
+ self.db = bridgedb.Storage.Database(self.fname)
+ bridgedb.Storage.setGlobalDB(self.db)
+ self.cur = self.db._conn.cursor()
+
+ def tearDown(self):
+ self.db.close()
+ os.close(self.fd)
+ os.unlink(self.fname)
+
+ def testEmailRateLimit(self):
+ db = self.db
+ EMAIL_DOMAIN_MAP = {'example.com':'example.com'}
+ d = bridgedb.Dist.EmailBasedDistributor(
+ "Foo",
+ {'example.com': 'example.com',
+ 'dkim.example.com': 'dkim.example.com'},
+ {'example.com': [], 'dkim.example.com': ['dkim']})
+ for _ in xrange(256):
+ d.insert(fakeBridge())
+ d.getBridgesForEmail('abc at example.com', 1, 3)
+ self.assertRaises(bridgedb.Dist.TooSoonEmail,
+ d.getBridgesForEmail, 'abc at example.com', 1, 3)
+ self.assertRaises(bridgedb.Dist.IgnoreEmail,
+ d.getBridgesForEmail, 'abc at example.com', 1, 3)
+
+ def testUnsupportedDomain(self):
+ db = self.db
+ self.assertRaises(bridgedb.Dist.UnsupportedDomain,
+ bridgedb.Dist.normalizeEmail, 'bad at email.com',
+ {'example.com':'example.com'},
+ {'example.com':[]})
+
class IPBridgeDistTests(unittest.TestCase):
def dumbAreaMapper(self, ip):
return ip
@@ -250,12 +285,22 @@ class SQLStorageTests(unittest.TestCase):
self.assertEquals(set(db.getBlockingCountries(b2.fingerprint)),
set(['uk', 'cn', 'de', 'jp', 'se', 'kr']))
+ self.assertEquals(db.getWarnedEmail("def at example.com"), False)
+ db.setWarnedEmail("def at example.com")
+ self.assertEquals(db.getWarnedEmail("def at example.com"), True)
+ db.setWarnedEmail("def at example.com", False)
+ self.assertEquals(db.getWarnedEmail("def at example.com"), False)
+
+ db.setWarnedEmail("def at example.com")
+ self.assertEquals(db.getWarnedEmail("def at example.com"), True)
+ db.cleanWarnedEmails(t+200)
+ self.assertEquals(db.getWarnedEmail("def at example.com"), False)
def testSuite():
suite = unittest.TestSuite()
loader = unittest.TestLoader()
- for klass in [ IPBridgeDistTests, DictStorageTests, SQLStorageTests ]:
+ for klass in [ IPBridgeDistTests, DictStorageTests, SQLStorageTests, EmailBridgeDistTests ]:
suite.addTest(loader.loadTestsFromTestCase(klass))
for module in [ bridgedb.Bridges,
More information about the tor-commits
mailing list