September 2011 - tor-commits - lists.torproject.org

[arm/master] Notice that multi-line input isn't implemented
by atagar＠torproject.org 19 Sep '11

19 Sep '11

commit 0b68c433fd1b58bc628eb53234a656845ff18a57 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Sep 18 21:43:21 2011 -0700 Notice that multi-line input isn't implemented A couple controller options (LOADCONF and POSTDESCRIPTOR) use multi-line controller input. However, this isn't yet implemented and probably won't be for the next release (those controller options really aren't very useful for the interpretor). --- src/util/torInterpretor.py | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diff --git a/src/util/torInterpretor.py b/src/util/torInterpretor.py index 825e502..3726517 100644 --- a/src/util/torInterpretor.py +++ b/src/util/torInterpretor.py @@ -787,6 +787,9 @@ class ControlInterpretor: conn.setOptions(paramList, isReset) except Exception, exc: outputEntry.append((str(exc), ERROR_FORMAT)) + elif cmd.replace("+", "") in ("LOADCONF", "POSTDESCRIPTOR"): + # provides a notice that multi-line controller input isn't yet implemented + outputEntry.append((MULTILINE_UNIMPLEMENTED_NOTICE + "\n", ERROR_FORMAT)) else: try: response = conn.getTorCtl().sendAndRecv("%s\r\n" % input)

1 0

[bridgedb/master] update README
by arma＠torproject.org 19 Sep '11

19 Sep '11

commit 054560d33d74e7120f1de62eb6f637b2f2e1439f Author: aagbsn <aagbsn(a)extc.org> Date: Sun Sep 18 20:11:54 2011 -0700 update README --- README | 26 ++++++++++++++++++++------ 1 files changed, 20 insertions(+), 6 deletions(-) diff --git a/README b/README index a21e273..2c0b357 100644 --- a/README +++ b/README @@ -14,9 +14,16 @@ To set up: - To install Maxmind GeoIP - Debian: apt-get install python-geoip - Others: http://www.maxmind.com/app/python - - To enable recaptcha support, set the options RECAPTCHA_ENABLED, - RECAPTCHA_PUB_KEY, and RECAPTCHA_PRIV_KEY in bridgedb.conf. - A recaptcha.net account is required. + - To enable recaptcha support + - set the options RECAPTCHA_ENABLED, RECAPTCHA_PUB_KEY, + and RECAPTCHA_PRIV_KEY in bridgedb.conf. + - A recaptcha.net account is required. + - Install these required packages: + - Debian: apt-get install python-recaptcha python-beautifulsoup + - Python: pip install recaptcha-client BeautifulSoup + - Others: http://pypi.python.org/pypi/recaptcha-client + : http://pypi.python.org/pypi/BeautifulSoup + To re-generate and update the i18n files (in case translated strings have changed in BridgeDB): @@ -61,9 +68,16 @@ To indicate which bridges are blocked: - For GeoIP support make sure to install Maxmind GeoIP To update the SQL schema: - - CREATE TABLE BlockedBridges ( id INTEGER PRIMARY KEY NOT NULL, hex_key, blocking_country); - - CREATE INDEX BlockedBridgesBlockingCountry on BlockedBridges(hex_key); - - REPLACE INTO Config VALUES ( 'schema-version', 2 ); + - Install sqlite3: + - Debian: apt-get install sqlite3 + - Others: http://www.sqlite.org/download.html + - Run "sqlite3 path/to/bridgedist.db.sqlite" + - Enter the following commands at the prompt: sqlite> + CREATE TABLE BlockedBridges ( id INTEGER PRIMARY KEY NOT NULL, hex_key, blocking_country); + CREATE INDEX BlockedBridgesBlockingCountry on BlockedBridges(hex_key); + CREATE TABLE WarnedEmails ( email PRIMARY KEY NOT NULL, when_warned); + CREATE INDEX WarnedEmailsWasWarned on WarnedEmails ( email ); + REPLACE INTO Config VALUES ( 'schema-version', 2 ); Support

1 0

[gettor/master] mike doesn't get torbutton support in gettor
by arma＠torproject.org 19 Sep '11

19 Sep '11

commit 4ce8743d7bae95dbc8486e42ff1cc8acb3b74439 Author: Roger Dingledine <arma(a)torproject.org> Date: Sun Sep 18 22:15:54 2011 -0400 mike doesn't get torbutton support in gettor but it's because we don't want to hand out xpi's separately, not because he refuses to do bignum math on the xpi files. --- gettor.conf | 2 -- 1 files changed, 0 insertions(+), 2 deletions(-) diff --git a/gettor.conf b/gettor.conf index 56f8c35..27bafd1 100644 --- a/gettor.conf +++ b/gettor.conf @@ -206,6 +206,4 @@ PACKAGES = { "linux-x86_64_zh_CN": ("torbrowser/linux/tor-browser-gnu-linux-x86_64-*-zh-CN.tar.gz", "unavailable"), - # Mike won't sign Torbutton; He doesn't get gettor support - #"torbutton": "torbutton-current.xpi", }

1 0

[bridgedb/master] add email domains to captcha template
by arma＠torproject.org 19 Sep '11

19 Sep '11

commit 7a75aeede51179b657dc1d8521cbabdb24f631f5 Author: aagbsn <aagbsn(a)extc.org> Date: Sun Sep 18 17:07:45 2011 -0700 add email domains to captcha template --- lib/bridgedb/Server.py | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diff --git a/lib/bridgedb/Server.py b/lib/bridgedb/Server.py index 1089598..964506a 100644 --- a/lib/bridgedb/Server.py +++ b/lib/bridgedb/Server.py @@ -191,6 +191,11 @@ class WebResource(twisted.web.resource.Resource): def buildHTMLMessageTemplateWithCaptcha(self, t, challenge, img): """Builds a translated html response with recaptcha""" + if self.domains: + email_domain_list = "<ul>" \ + + "".join(("<li>%s</li>"%d for d in self.domains)) + "</ul>" + else: + email_domain_list = "<p>" + t.gettext(I18n.BRIDGEDB_TEXT[8]) + "</p>" recaptchaTemplate = textwrap.dedent("""\ <form action="" method="POST"> @@ -225,6 +230,7 @@ class WebResource(twisted.web.resource.Resource): + "<p>" + t.gettext(I18n.BRIDGEDB_TEXT[9]) + "</p>" \ + "<p>" + recaptchaTemplate + "</p>" \ + "<p>" + t.gettext(I18n.BRIDGEDB_TEXT[4]) + "</p>" \ + + email_domain_list \ + "</body></html>" return html_msg

1 0

[bridgedb/master] 1608 - don't give out known blocked bridges
by arma＠torproject.org 19 Sep '11

19 Sep '11

commit 0fc5f539b31f61b7206a9825ca589b62530e3d8f Author: aagbsn <aagbsn(a)extc.org> Date: Mon Jul 25 16:32:15 2011 -0700 1608 - don't give out known blocked bridges This set of changes allows bridgedb to filter responses by country; if a bridge is known to be blocked in a country it can be filtered from the set of returned bridges. --- README | 16 +++++++++++ bridgedb.conf | 3 ++ lib/bridgedb/Bridges.py | 66 +++++++++++++++++++++++++++++++++++++++++++++- lib/bridgedb/Dist.py | 8 +++--- lib/bridgedb/I18n.py | 4 ++- lib/bridgedb/Main.py | 11 ++++++++ lib/bridgedb/Server.py | 61 +++++++++++++++++++++++++++++++++---------- lib/bridgedb/Storage.py | 64 ++++++++++++++++++++++++++++++++++++++++++--- lib/bridgedb/Tests.py | 18 +++++++++++++ lib/bridgedb/Time.py | 6 ++-- 10 files changed, 228 insertions(+), 29 deletions(-) diff --git a/README b/README index 64b402a..c05317a 100644 --- a/README +++ b/README @@ -11,6 +11,9 @@ To set up: - You can make a self-signed certificate with openssl req -x509 -new -nodes > cert - Set up PYTHONPATH to include "~/lib/python2.4/site-packages/". + - To install Maxmind GeoIP + - Debian: apt-get install python-geoip + - Others: http://www.maxmind.com/app/python To re-generate and update the i18n files (in case translated strings have changed in BridgeDB): @@ -47,5 +50,18 @@ To use with email: - Any mail sent to the email port with a subject or a single line _exactly_ equal to "get bridges" will get answered, assuming the domain is okay. +To indicate which bridges are blocked: + - Uncomment or add COUNTRY_BLOCK_FILE to your bridgedb.conf + - The syntax of the COUNTRY_BLOCK_FILE, 1 entry per line: + fingerprint <bridge fingerprint> country-code <country code> + - If this file is present, bridgedb will filter blocked bridges from responses + - For GeoIP support make sure to install Maxmind GeoIP + +To update the SQL schema: + - CREATE TABLE BlockedBridges ( id INTEGER PRIMARY KEY NOT NULL, hex_key, blocking_country); + - CREATE INDEX BlockedBridgesBlockingCountry on BlockedBridges(hex_key); + - REPLACE INTO Config VALUES ( 'schema-version', 2 ); + + Support - Send your questions to nickm. diff --git a/bridgedb.conf b/bridgedb.conf index e486e0a..11afc02 100644 --- a/bridgedb.conf +++ b/bridgedb.conf @@ -21,6 +21,9 @@ BRIDGE_FILES = [ "./bridge-descriptors" ] # current bridges are Running. STATUS_FILE = "networkstatus-bridges" +# Either a file that contains blocked bridges list or None +#COUNTRY_BLOCK_FILE = "./blocked-bridges" + # Only consider routers whose purpose matches this string. BRIDGE_PURPOSE = "bridge" # File to store persistent info in. diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index 229cc5a..358e362 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -104,6 +104,7 @@ class Bridge: ## fingerprint -- The bridge's identity digest, in lowercase hex, with ## no spaces. ## running,stable -- DOCDOC + ## blockingCountries -- list of country codes blocking this bridge def __init__(self, nickname, ip, orport, fingerprint=None, id_digest=None): """Create a new Bridge. One of fingerprint and id_digest must be set.""" @@ -111,6 +112,7 @@ class Bridge: self.ip = ip self.orport = orport self.running = self.stable = None + self.blockingCountries = None if id_digest is not None: assert fingerprint is None if len(id_digest) != DIGEST_LEN: @@ -151,6 +153,15 @@ class Bridge: if stable is not None: self.stable = stable + def setBlockingCountries(self, blockingCountries): + if blockingCountries is not None: + self.blockingCountries = blockingCountries + + def isBlocked(self, countryCode): + if self.blockingCountries is not None and countryCode is not None: + if countryCode in self.blockingCountries: + return True + return False def parseDescFile(f, bridge_purpose='bridge'): """Generator. Parses a cached-descriptors file 'f', and yields a Bridge @@ -200,10 +211,24 @@ def parseStatusFile(f): flags = line.split() yield ID, ("Running" in flags), ("Stable" in flags) +def parseCountryBlockFile(f): + """Generator. Parses a blocked-bridges file 'f', and yields a + fingerprint, countryCode tuple for every entry""" + fingerprint = countryCode = None + for line in f: + line = line.strip() + m = re.match(r"fingerprint\s+(?P<fingerprint>\w+?)\s+country-code\s+(?P<countryCode>\w+)$", line) + try: + fingerprint = m.group('fingerprint').lower() + countryCode = m.group('countryCode').lower() + yield fingerprint, countryCode + except AttributeError, IndexError: + logging.warn("Unparseable line in blocked-bridges file: %s", line) + class BridgeHolder: """Abstract base class for all classes that hold bridges.""" def insert(self, bridge): - raise NotImplemented() + raise NotImplementedError def clear(self): pass @@ -326,7 +351,7 @@ class BridgeRing(BridgeHolder): assert len(r) == N return r - def getBridges(self, pos, N=1): + def getBridges(self, pos, N=1, countryCode=None): """Return the N bridges appearing in the ring after position pos""" forced = [] for _,_,count,subring in self.subrings: @@ -523,3 +548,40 @@ class BridgeSplitter(BridgeHolder): def dumpAssignments(self, f, description=""): for name,ring in self.ringsByName.iteritems(): ring.dumpAssignments(f, "%s %s" % (description, name)) + +class BridgeBlock: + """Base class that abstracts bridge blocking""" + def __init__(self): + pass + + def insert(self, fingerprint, blockingRule): + raise NotImplementedError + + def clear(self): + pass + + def assignmentsArePersistent(self): + return True + +class CountryBlock(BridgeBlock): + """Countrywide bridge blocking""" + def __init__(self): + self.db = bridgedb.Storage.getDB() + + def clear(self): + assert self.db + self.db.cleanBridgeBlocks() + self.db.commit() + + def insert(self, fingerprint, blockingRule): + """ insert a country based blocking rule """ + assert self.db + countryCode = blockingRule + self.db.addBridgeBlock(fingerprint, countryCode) + self.db.commit() + + def getBlockingCountries(self, fingerprint): + """ returns a list of country codes where this fingerprint is blocked""" + assert self.db + if fingerprint is not None: + return self.db.getBlockingCountries(fingerprint) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index f2274c7..5394cfa 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -75,7 +75,7 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder): """Assign a bridge to this distributor.""" self.splitter.insert(bridge) - def getBridgesForIP(self, ip, epoch, N=1): + def getBridgesForIP(self, ip, epoch, N=1, countryCode=None): """Return a list of bridges to give to a user. ip -- the user's IP address, as a dotted quad. epoch -- the time period when we got this request. This can @@ -94,7 +94,7 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder): if category.contains(ip): logging.info("category<%s>%s"%(epoch,area)) pos = self.areaOrderHmac("category<%s>%s"%(epoch,area)) - return ring.getBridges(pos, N) + return ring.getBridges(pos, N, countryCode) # Which bridge cluster should we look at? h = int( self.areaClusterHmac(area)[:8], 16) @@ -240,7 +240,7 @@ class EmailBasedDistributor(bridgedb.Bridges.BridgeHolder): """Assign a bridge to this distributor.""" self.ring.insert(bridge) - def getBridgesForEmail(self, emailaddress, epoch, N=1, parameters=None): + def getBridgesForEmail(self, emailaddress, epoch, N=1, parameters=None, countryCode=None): """Return a list of bridges to give to a user. emailaddress -- the user's email address, as given in a from line. epoch -- the time period when we got this request. This can @@ -266,7 +266,7 @@ class EmailBasedDistributor(bridgedb.Bridges.BridgeHolder): raise TooSoonEmail("Too many emails; wait till later", emailaddress) pos = self.emailHmac("<%s>%s" % (epoch, emailaddress)) - result = self.ring.getBridges(pos, N) + result = self.ring.getBridges(pos, N, countryCode) db.setEmailTime(emailaddress, now) db.commit() diff --git a/lib/bridgedb/I18n.py b/lib/bridgedb/I18n.py index fc023f7..44adb8c 100644 --- a/lib/bridgedb/I18n.py +++ b/lib/bridgedb/I18n.py @@ -49,5 +49,7 @@ bridge addresses."""), # BRIDGEDB_TEXT[7] _("""(no bridges currently available)"""), # BRIDGEDB_TEXT[8] - _("""(e-mail requests not currently supported)""") + _("""(e-mail requests not currently supported)"""), + # BRIDGEDB_TEXT[9] + _("""(Might be blocked)""") ] diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 99d43b6..42f78c3 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -146,15 +146,24 @@ def load(cfg, splitter, clear=False): """Read all the bridge files from cfg, and pass them into a splitter object. """ + countryblock = Bridges.CountryBlock() if clear: logging.info("Clearing old bridges") splitter.clear() + logging.info("Clearing old blocked bridges") + countryblock.clear() logging.info("Loading bridges") status = {} if hasattr(cfg, "STATUS_FILE"): f = open(cfg.STATUS_FILE, 'r') for ID, running, stable in Bridges.parseStatusFile(f): status[ID] = running, stable + f.close() + if hasattr(cfg, "COUNTRY_BLOCK_FILE"): + f = open(cfg.COUNTRY_BLOCK_FILE, 'r') + for fingerprint, countryCode in Bridges.parseCountryBlockFile(f): + countryblock.insert(fingerprint, countryCode) + f.close() for fname in cfg.BRIDGE_FILES: f = open(fname, 'r') for bridge in Bridges.parseDescFile(f, cfg.BRIDGE_PURPOSE): @@ -162,6 +171,8 @@ def load(cfg, splitter, clear=False): if s is not None: running, stable = s bridge.setStatus(running=running, stable=stable) + bridge.setBlockingCountries( + countryblock.getBlockingCountries(bridge.fingerprint)) splitter.insert(bridge) f.close() diff --git a/lib/bridgedb/Server.py b/lib/bridgedb/Server.py index e2e0222..8ec659e 100644 --- a/lib/bridgedb/Server.py +++ b/lib/bridgedb/Server.py @@ -26,6 +26,16 @@ import twisted.mail.smtp import bridgedb.Dist import bridgedb.I18n as I18n +try: + import GeoIP + # GeoIP data object: choose database here + # This is the same geoip implementation that pytorctl uses + geoip = GeoIP.new(GeoIP.GEOIP_STANDARD) + logging.info("GeoIP database loaded") +except: + geoip = None + logging.warn("GeoIP database not found") + class WebResource(twisted.web.resource.Resource): """This resource is used by Twisted Web to give a web page with some bridges in response to a request.""" @@ -51,6 +61,7 @@ class WebResource(twisted.web.resource.Resource): interval = self.schedule.getInterval(time.time()) bridges = ( ) ip = None + countryCode = None if self.useForwardedHeader: h = request.getHeader("X-Forwarded-For") if h: @@ -61,26 +72,29 @@ class WebResource(twisted.web.resource.Resource): else: ip = request.getClientIP() - # See if we did get a request for a certain locale, otherwise fall back - # to 'en': - # Try evaluating the path /foo first, then check if we got a ?lang=foo - default_lang = lang = "en" - if len(request.path) > 1: - lang = request.path[1:] - if lang == default_lang: - lang = request.args.get("lang", [default_lang]) - lang = lang[0] - t = I18n.getLang(lang) + if geoip: + countryCode = geoip.country_code_by_addr(ip) + + # allow client to specify a country + forcecc = getCCFromRequest(request) + if forcecc != None: + countryCode = forcecc + + # get locale + t = getLocaleFromRequest(request) format = request.args.get("format", None) if format and len(format): format = format[0] # choose the first arg if ip: bridges = self.distributor.getBridgesForIP(ip, interval, - self.nBridgesToGive) + self.nBridgesToGive, + countryCode) if bridges: - answer = "".join("%s\n" % b.getConfigLine(self.includeFingerprints) - for b in bridges) + answer = "".join("%s %s\n" % ( + b.getConfigLine(self.includeFingerprints), + (I18n.BRIDGEDB_TEXT[9] if b.isBlocked(countryCode) else "") + ) for b in bridges) else: answer = t.gettext(I18n.BRIDGEDB_TEXT[7]) @@ -253,7 +267,8 @@ def getMailResponse(lines, ctx): try: interval = ctx.schedule.getInterval(time.time()) bridges = ctx.distributor.getBridgesForEmail(clientAddr, - interval, ctx.N) + interval, ctx.N, + countryCode=None) except bridgedb.Dist.BadEmail, e: logging.info("Got a mail from a bad email address %r: %s.", clientAddr, e) @@ -329,6 +344,7 @@ def getLocaleFromPlusAddr(address): return replyLocale + class MailContext: """Helper object that holds information used by email subsystem.""" def __init__(self, cfg, dist, sched): @@ -451,3 +467,20 @@ def runServers(): """Start all the servers that we've configured. Exits when they do.""" reactor.run() +def getLocaleFromRequest(request): + # See if we did get a request for a certain locale, otherwise fall back + # to 'en': + # Try evaluating the path /foo first, then check if we got a ?lang=foo + default_lang = lang = "en" + if len(request.path) > 1: + lang = request.path[1:] + if lang == default_lang: + lang = request.args.get("lang", [default_lang]) + lang = lang[0] + return I18n.getLang(lang) + +def getCCFromRequest(request): + path = re.sub(r'[^a-zA-Z]', '', request.path) + if len(path) == 2: + return path.lower() + return None diff --git a/lib/bridgedb/Storage.py b/lib/bridgedb/Storage.py index 30b4c57..5d42930 100644 --- a/lib/bridgedb/Storage.py +++ b/lib/bridgedb/Storage.py @@ -111,7 +111,7 @@ class SqliteDict: # Here is the SQL schema. -SCHEMA1_SCRIPT = """ +SCHEMA2_SCRIPT = """ CREATE TABLE Config ( key PRIMARY KEY NOT NULL, value @@ -136,7 +136,15 @@ SCHEMA1_SCRIPT = """ CREATE INDEX EmailedBridgesWhenMailed on EmailedBridges ( email ); - INSERT INTO Config VALUES ( 'schema-version', 1 ); + CREATE TABLE BlockedBridges ( + id INTEGER PRIMARY KEY NOT NULL, + hex_key, + blocking_country + ); + + CREATE INDEX BlockedBridgesBlockingCountry on BlockedBridges(hex_key); + + INSERT INTO Config VALUES ( 'schema-version', 2 ); """ class BridgeData: @@ -267,6 +275,52 @@ class Database: cur.execute("UPDATE Bridges SET distributor = ? WHERE hex_key = ?", (distributor, hex_key)) + def addBridgeBlock(self, fingerprint, countryCode): + cur = self._cur + cur.execute("INSERT OR REPLACE INTO BlockedBridges " + "(hex_key,blocking_country) VALUES (?,?)", + (fingerprint, countryCode)) + + def delBridgeBlock(self, fingerprint, countryCode): + cur = self._cur + cur.execute("DELETE FROM BlockedBridges WHERE hex_key = ? " + "AND blocking_country = ?", (fingerprint, countryCode)) + + def cleanBridgeBlocks(self): + cur = self._cur + cur.execute("DELETE FROM BlockedBridges") + + def getBlockingCountries(self, fingerprint): + cur = self._cur + cur.execute("SELECT hex_key, blocking_country FROM BlockedBridges WHERE hex_key = ? ", + (fingerprint,)) + v = cur.fetchall() + if v is None: + return None + + # return list of country-codes + return [ str(result[1]) for (result) in v ] + + def getBlockedBridges(self, countryCode): + cur = self._cur + cur.execute("SELECT hex_key, blocking_country FROM BlockedBridges WHERE blocking_country = ? ", + (countryCode,)) + v = cur.fetchall() + if v is None: + return None + # return list of fingerprints + return [ str(result[0]) for (result) in v ] + + def isBlocked(self, fingerprint, countryCode): + cur = self._cur + cur.execute("SELECT hex_key, blocking_country FROM BlockedBridges WHERE " + "hex_key = ? AND blocking_country = ?", + (fingerprint, countryCode)) + v = cur.fetchone() + if v is None: + return False + return True + def openDatabase(sqlite_file): conn = sqlite3.Connection(sqlite_file) cur = conn.cursor() @@ -274,11 +328,11 @@ def openDatabase(sqlite_file): try: cur.execute("SELECT value FROM Config WHERE key = 'schema-version'") val, = cur.fetchone() - if val != 1: + if val != 2: logging.warn("Unknown schema version %s in database.", val) except sqlite3.OperationalError: logging.warn("No Config table found in DB; creating tables") - cur.executescript(SCHEMA1_SCRIPT) + cur.executescript(SCHEMA2_SCRIPT) conn.commit() finally: cur.close() @@ -292,7 +346,7 @@ def openOrConvertDatabase(sqlite_file, db_file): conn = sqlite3.Connection(sqlite_file) cur = conn.cursor() - cur.executescript(SCHEMA1_SCRIPT) + cur.executescript(SCHEMA2_SCRIPT) conn.commit() import anydbm diff --git a/lib/bridgedb/Tests.py b/lib/bridgedb/Tests.py index 9dea4b4..558fedf 100644 --- a/lib/bridgedb/Tests.py +++ b/lib/bridgedb/Tests.py @@ -233,6 +233,24 @@ class SQLStorageTests(unittest.TestCase): cur.execute("SELECT * FROM EmailedBridges") self.assertEquals(len(cur.fetchall()), 1) + db.addBridgeBlock(b2.fingerprint, 'us') + self.assertEquals(db.isBlocked(b2.fingerprint, 'us'), True) + db.delBridgeBlock(b2.fingerprint, 'us') + self.assertEquals(db.isBlocked(b2.fingerprint, 'us'), False) + db.addBridgeBlock(b2.fingerprint, 'uk') + db.addBridgeBlock(b3.fingerprint, 'uk') + self.assertEquals(set([b2.fingerprint, b3.fingerprint]), + set(db.getBlockedBridges('uk'))) + + db.addBridgeBlock(b2.fingerprint, 'cn') + db.addBridgeBlock(b2.fingerprint, 'de') + db.addBridgeBlock(b2.fingerprint, 'jp') + db.addBridgeBlock(b2.fingerprint, 'se') + db.addBridgeBlock(b2.fingerprint, 'kr') + + self.assertEquals(set(db.getBlockingCountries(b2.fingerprint)), + set(['uk', 'cn', 'de', 'jp', 'se', 'kr'])) + def testSuite(): suite = unittest.TestSuite() loader = unittest.TestLoader() diff --git a/lib/bridgedb/Time.py b/lib/bridgedb/Time.py index 63c50d5..8d7c65a 100644 --- a/lib/bridgedb/Time.py +++ b/lib/bridgedb/Time.py @@ -13,11 +13,11 @@ KNOWN_INTERVALS = [ "hour", "day", "week", "month" ] class Schedule: def intervalStart(self, when): - raise NotImplemented() + raise NotImplementedError def getInterval(self, when): - raise NotImplemented() + raise NotImplementedError def nextIntervalStarts(self, when): - raise NotImplemented() + raise NotImplementedError class IntervalSchedule(Schedule): """An IntervalSchedule splits time into somewhat natural periods,

1 0

[bridgedb/master] 2678 - filter bridges in same /16 from response
by arma＠torproject.org 19 Sep '11

19 Sep '11

commit 75b386c8d2df617359eb21d7c2e3e147be6df363 Author: aagbsn <aagbsn(a)extc.org> Date: Mon Jul 25 16:27:00 2011 -0700 2678 - filter bridges in same /16 from response --- lib/bridgedb/Bridges.py | 14 +++++++++++++- lib/bridgedb/Dist.py | 6 +++--- lib/bridgedb/Server.py | 2 +- lib/bridgedb/Storage.py | 4 ++-- lib/bridgedb/Tests.py | 25 +++++++++++++++++++++++++ 5 files changed, 44 insertions(+), 7 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index eca6db5..229cc5a 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -340,7 +340,19 @@ class BridgeRing(BridgeHolder): keys.append(k) keys = keys[:N] keys.sort() - return [ self.bridges[k] for k in keys ] + + #Do not return bridges from the same /16 + bridges = [ self.bridges[k] for k in keys ] + filteredbridges = [] + slash16s = dict() + + for bridge in bridges: + m = re.match(r'(\d+\.\d+)\.\d+\.\d+', bridge.ip) + upper16 = m.group(1) + if upper16 not in slash16s: + filteredbridges.append(bridge) + slash16s[upper16] = True + return filteredbridges def getBridgeByID(self, fp): """Return the bridge whose identity digest is fp, or None if no such diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index e11d21e..f2274c7 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -249,10 +249,10 @@ class EmailBasedDistributor(bridgedb.Bridges.BridgeHolder): """ now = time.time() try: - emailaddress = normalizeEmail(emailaddress, self.domainmap, - self.domainrules) + emailaddress = normalizeEmail(emailaddress, self.domainmap, + self.domainrules) except BadEmail: - return [] #XXXX log the exception + return [] #XXXX log the exception if emailaddress is None: return [] #XXXX raise an exception. diff --git a/lib/bridgedb/Server.py b/lib/bridgedb/Server.py index 20b349b..e2e0222 100644 --- a/lib/bridgedb/Server.py +++ b/lib/bridgedb/Server.py @@ -219,7 +219,7 @@ def getMailResponse(lines, ctx): try: _, addrdomain = bridgedb.Dist.extractAddrSpec(clientAddr.lower()) except bridgedb.Dist.BadEmail: - logging.info("Ignoring bad address on incoming email.") + logging.info("Ignoring bad address on incoming email.") return None,None if not addrdomain: logging.info("Couldn't parse domain from %r", clientAddr) diff --git a/lib/bridgedb/Storage.py b/lib/bridgedb/Storage.py index 62ed71c..30b4c57 100644 --- a/lib/bridgedb/Storage.py +++ b/lib/bridgedb/Storage.py @@ -76,7 +76,7 @@ class SqliteDict: self._cursor.execute(self._getStmt, (k,)) val = self._cursor.fetchone() if val == None: - return v; + return v else: return val[0] def setdefault(self, k, v): @@ -216,7 +216,7 @@ class Database: cur = self._cur t = timeToStr(expireBefore) - cur.execute("DELETE FROM EmailedBridges WHERE when_mailed < ?", (t,)); + cur.execute("DELETE FROM EmailedBridges WHERE when_mailed < ?", (t,)) def getEmailTime(self, addr): cur = self._cur diff --git a/lib/bridgedb/Tests.py b/lib/bridgedb/Tests.py index 1353658..9dea4b4 100644 --- a/lib/bridgedb/Tests.py +++ b/lib/bridgedb/Tests.py @@ -16,6 +16,7 @@ import bridgedb.Main import bridgedb.Dist import bridgedb.Time import bridgedb.Storage +import re def suppressWarnings(): warnings.filterwarnings('ignore', '.*tmpnam.*') @@ -23,12 +24,23 @@ def suppressWarnings(): def randomIP(): return ".".join([str(random.randrange(1,256)) for _ in xrange(4)]) +def random16IP(): + upper = "123.123." # same 16 + lower = ".".join([str(random.randrange(1,256)) for _ in xrange(2)]) + return upper+lower + def fakeBridge(orport=8080): nn = "bridge-%s"%random.randrange(0,1000000) ip = randomIP() fp = "".join([random.choice("0123456789ABCDEF") for _ in xrange(40)]) return bridgedb.Bridges.Bridge(nn,ip,orport,fingerprint=fp) +def fake16Bridge(orport=8080): + nn = "bridge-%s"%random.randrange(0,1000000) + ip = random16IP() + fp = "".join([random.choice("0123456789ABCDEF") for _ in xrange(40)]) + return bridgedb.Bridges.Bridge(nn,ip,orport,fingerprint=fp) + class RhymesWith255Category: def contains(self, ip): return ip.endswith(".255") @@ -83,6 +95,19 @@ class IPBridgeDistTests(unittest.TestCase): self.assertEquals(len(fps), 5) self.assertTrue(count >= 1) + def testDistWithFilter16(self): + d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo") + for _ in xrange(256): + d.insert(fake16Bridge()) + n = d.getBridgesForIP("1.2.3.4", "x", 10) + + slash16s = dict() + for bridge in n: + m = re.match(r'(\d+\.\d+)\.\d+\.\d+', bridge.ip) + upper16 = m.group(1) + self.assertTrue(upper16 not in slash16s) + slash16s[upper16] = True + class DictStorageTests(unittest.TestCase): def setUp(self): self.fd, self.fname = tempfile.mkstemp()

1 0

[bridgedb/master] 1860 - email rate limiting support
by arma＠torproject.org 19 Sep '11

19 Sep '11

commit 6081a27dfc38634eb1f47c462d2bf9c6aab599d8 Author: aagbsn <aagbsn(a)extc.org> Date: Mon Jul 25 17:04:14 2011 -0700 1860 - email rate limiting support This set of changes implements email rate-limiting for the EmailBasedDistributor. Abusers are warned, and then temporarily blacklisted. --- lib/bridgedb/Dist.py | 20 +++++++++++++++++++- lib/bridgedb/I18n.py | 9 ++++++++- lib/bridgedb/Server.py | 37 +++++++++++++++++++++++++++++++++++++ lib/bridgedb/Storage.py | 37 +++++++++++++++++++++++++++++++++++++ lib/bridgedb/Tests.py | 47 ++++++++++++++++++++++++++++++++++++++++++++++- 5 files changed, 147 insertions(+), 3 deletions(-) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index 5394cfa..030c7fa 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -146,6 +146,10 @@ class TooSoonEmail(BadEmail): """Raised when we got a request from this address too recently.""" pass +class IgnoreEmail(BadEmail): + """Raised when we get requests from this address after rate warning.""" + pass + def extractAddrSpec(addr): """Given an email From line, try to extract and parse the addrspec portion. Returns localpart,domain on success; raises BadEmail @@ -257,14 +261,27 @@ class EmailBasedDistributor(bridgedb.Bridges.BridgeHolder): return [] #XXXX raise an exception. db = bridgedb.Storage.getDB() + wasWarned = db.getWarnedEmail(emailaddress) lastSaw = db.getEmailTime(emailaddress) if lastSaw is not None and lastSaw + MAX_EMAIL_RATE >= now: + if wasWarned: + logging.warn("Got a request for bridges from %r; we already " + "sent a warning. Ignoring.", emailaddress) + raise IgnoreEmail("Client was warned", emailaddress) + else: + db.setWarnedEmail(emailaddress, True, now) + db.commit() + logging.warn("Got a request for bridges from %r; we already " - "answered one within the last %d seconds. Ignoring.", + "answered one within the last %d seconds. Warning.", emailaddress, MAX_EMAIL_RATE) raise TooSoonEmail("Too many emails; wait till later", emailaddress) + # warning period is over + elif wasWarned: + db.setWarnedEmail(emailaddress, False) + pos = self.emailHmac("<%s>%s" % (epoch, emailaddress)) result = self.ring.getBridges(pos, N, countryCode) @@ -279,6 +296,7 @@ class EmailBasedDistributor(bridgedb.Bridges.BridgeHolder): db = bridgedb.Storage.getDB() try: db.cleanEmailedBridges(time.time()-MAX_EMAIL_RATE) + db.cleanWarnedBridges(time.time()-MAX_EMAIL_RATE) except: db.rollback() raise diff --git a/lib/bridgedb/I18n.py b/lib/bridgedb/I18n.py index 44adb8c..750ca4a 100644 --- a/lib/bridgedb/I18n.py +++ b/lib/bridgedb/I18n.py @@ -51,5 +51,12 @@ bridge addresses."""), # BRIDGEDB_TEXT[8] _("""(e-mail requests not currently supported)"""), # BRIDGEDB_TEXT[9] - _("""(Might be blocked)""") + _("""To receive your bridge relays, please prove you are human"""), + # BRIDGEDB_TEXT[10] + _("""You have exceeded the rate limit. Please slow down, the minimum time between +emails is: """), + # BRIDGEDB_TEXT[11] + _("""hours"""), + # BRIDGEDB_TEXT[12] + _("""All further emails will be ignored.""") ] diff --git a/lib/bridgedb/Server.py b/lib/bridgedb/Server.py index 8ec659e..9cbeb15 100644 --- a/lib/bridgedb/Server.py +++ b/lib/bridgedb/Server.py @@ -274,6 +274,35 @@ def getMailResponse(lines, ctx): clientAddr, e) return None, None + # Handle rate limited email + except bridgedb.Dist.TooSoonEmail, e: + logging.info("Got a mail too frequently; warning %r: %s.", + clientAddr, e) + + # Compose a warning email + f = StringIO() + w = MimeWriter.MimeWriter(f) + w.addheader("From", ctx.fromAddr) + w.addheader("To", clientAddr) + w.addheader("Message-ID", twisted.mail.smtp.messageid()) + if not subject.startswith("Re:"): subject = "Re: %s"%subject + w.addheader("Subject", subject) + if msgID: + w.addheader("In-Reply-To", msgID) + w.addheader("Date", twisted.mail.smtp.rfc822date()) + body = w.startbody("text/plain") + + # MAX_EMAIL_RATE is in seconds, convert to hours + EMAIL_MESSAGE_RATELIMIT = buildSpamWarningTemplate(t) + body.write(EMAIL_MESSAGE_RATELIMIT % (bridgedb.Dist.MAX_EMAIL_RATE / 3600)) + f.seek(0) + return clientAddr, f + + except bridgedb.Dist.IgnoreEmail, e: + logging.info("Got a mail too frequently; ignoring %r: %s.", + clientAddr, e) + return None, None + # Generate the message. f = StringIO() w = MimeWriter.MimeWriter(f) @@ -311,6 +340,14 @@ def buildMessageTemplate(t): return msg_template +def buildSpamWarningTemplate(t): + msg_template = t.gettext(I18n.BRIDGEDB_TEXT[5]) + "\n\n" \ + + t.gettext(I18n.BRIDGEDB_TEXT[10]) + "\n\n" \ + + "%s " \ + + t.gettext(I18n.BRIDGEDB_TEXT[11]) + "\n\n" \ + + t.gettext(I18n.BRIDGEDB_TEXT[12]) + "\n\n" + return msg_template + def replyToMail(lines, ctx): """Given a list of lines from an incoming email message, and a MailContext object, possibly send a reply. diff --git a/lib/bridgedb/Storage.py b/lib/bridgedb/Storage.py index 5d42930..da64a30 100644 --- a/lib/bridgedb/Storage.py +++ b/lib/bridgedb/Storage.py @@ -8,6 +8,7 @@ import logging import binascii import sqlite3 import time +import sha toHex = binascii.b2a_hex fromHex = binascii.a2b_hex @@ -144,6 +145,13 @@ SCHEMA2_SCRIPT = """ CREATE INDEX BlockedBridgesBlockingCountry on BlockedBridges(hex_key); + CREATE TABLE WarnedEmails ( + email PRIMARY KEY NOT NULL, + when_warned + ); + + CREATE INDEX WarnedEmailsWasWarned on WarnedEmails ( email ); + INSERT INTO Config VALUES ( 'schema-version', 2 ); """ @@ -227,6 +235,7 @@ class Database: cur.execute("DELETE FROM EmailedBridges WHERE when_mailed < ?", (t,)) def getEmailTime(self, addr): + addr = sha.new(addr).hexdigest() cur = self._cur cur.execute("SELECT when_mailed FROM EmailedBridges WHERE " "email = ?", (addr,)) @@ -236,6 +245,7 @@ class Database: return strToTime(v[0]) def setEmailTime(self, addr, whenMailed): + addr = sha.new(addr).hexdigest() cur = self._cur t = timeToStr(whenMailed) cur.execute("INSERT OR REPLACE INTO EmailedBridges " @@ -321,6 +331,33 @@ class Database: return False return True + def getWarnedEmail(self, addr): + addr = sha.new(addr).hexdigest() + cur = self._cur + cur.execute("SELECT * FROM WarnedEmails WHERE " + " email = ?", (addr,)) + v = cur.fetchone() + if v is None: + return False + return True + + def setWarnedEmail(self, addr, warned=True, whenWarned=time.time()): + addr = sha.new(addr).hexdigest() + t = timeToStr(whenWarned) + cur = self._cur + if warned == True: + cur.execute("INSERT INTO WarnedEmails" + "(email,when_warned) VALUES (?,?)", (addr, t,)) + elif warned == False: + cur.execute("DELETE FROM WarnedEmails WHERE " + "email = ?", (addr,)) + + def cleanWarnedEmails(self, expireBefore): + cur = self._cur + t = timeToStr(expireBefore) + + cur.execute("DELETE FROM WarnedEmails WHERE when_warned < ?", (t,)) + def openDatabase(sqlite_file): conn = sqlite3.Connection(sqlite_file) cur = conn.cursor() diff --git a/lib/bridgedb/Tests.py b/lib/bridgedb/Tests.py index 558fedf..9648f86 100644 --- a/lib/bridgedb/Tests.py +++ b/lib/bridgedb/Tests.py @@ -45,6 +45,41 @@ class RhymesWith255Category: def contains(self, ip): return ip.endswith(".255") +class EmailBridgeDistTests(unittest.TestCase): + def setUp(self): + self.fd, self.fname = tempfile.mkstemp() + self.db = bridgedb.Storage.Database(self.fname) + bridgedb.Storage.setGlobalDB(self.db) + self.cur = self.db._conn.cursor() + + def tearDown(self): + self.db.close() + os.close(self.fd) + os.unlink(self.fname) + + def testEmailRateLimit(self): + db = self.db + EMAIL_DOMAIN_MAP = {'example.com':'example.com'} + d = bridgedb.Dist.EmailBasedDistributor( + "Foo", + {'example.com': 'example.com', + 'dkim.example.com': 'dkim.example.com'}, + {'example.com': [], 'dkim.example.com': ['dkim']}) + for _ in xrange(256): + d.insert(fakeBridge()) + d.getBridgesForEmail('abc(a)example.com', 1, 3) + self.assertRaises(bridgedb.Dist.TooSoonEmail, + d.getBridgesForEmail, 'abc(a)example.com', 1, 3) + self.assertRaises(bridgedb.Dist.IgnoreEmail, + d.getBridgesForEmail, 'abc(a)example.com', 1, 3) + + def testUnsupportedDomain(self): + db = self.db + self.assertRaises(bridgedb.Dist.UnsupportedDomain, + bridgedb.Dist.normalizeEmail, 'bad(a)email.com', + {'example.com':'example.com'}, + {'example.com':[]}) + class IPBridgeDistTests(unittest.TestCase): def dumbAreaMapper(self, ip): return ip @@ -250,12 +285,22 @@ class SQLStorageTests(unittest.TestCase): self.assertEquals(set(db.getBlockingCountries(b2.fingerprint)), set(['uk', 'cn', 'de', 'jp', 'se', 'kr'])) + self.assertEquals(db.getWarnedEmail("def(a)example.com"), False) + db.setWarnedEmail("def(a)example.com") + self.assertEquals(db.getWarnedEmail("def(a)example.com"), True) + db.setWarnedEmail("def(a)example.com", False) + self.assertEquals(db.getWarnedEmail("def(a)example.com"), False) + + db.setWarnedEmail("def(a)example.com") + self.assertEquals(db.getWarnedEmail("def(a)example.com"), True) + db.cleanWarnedEmails(t+200) + self.assertEquals(db.getWarnedEmail("def(a)example.com"), False) def testSuite(): suite = unittest.TestSuite() loader = unittest.TestLoader() - for klass in [ IPBridgeDistTests, DictStorageTests, SQLStorageTests ]: + for klass in [ IPBridgeDistTests, DictStorageTests, SQLStorageTests, EmailBridgeDistTests ]: suite.addTest(loader.loadTestsFromTestCase(klass)) for module in [ bridgedb.Bridges,

1 0

[bridgedb/master] 1836 - recaptcha support
by arma＠torproject.org 19 Sep '11

19 Sep '11

commit 2d670fe64c04b632c9bbc8f0d2ccddd6e2b0baaa Author: aagbsn <aagbsn(a)extc.org> Date: Mon Jul 25 17:18:20 2011 -0700 1836 - recaptcha support This set of changes adds recaptcha support to bridgedb. The implementation proxies responses from clients to preserve their privacy. --- README | 3 + bridgedb.conf | 7 +++ lib/bridgedb/I18n.py | 10 ++++- lib/bridgedb/Main.py | 5 ++- lib/bridgedb/Raptcha.py | 40 +++++++++++++++ lib/bridgedb/Server.py | 122 +++++++++++++++++++++++++++++++++++++++++++++- 6 files changed, 182 insertions(+), 5 deletions(-) diff --git a/README b/README index c05317a..a21e273 100644 --- a/README +++ b/README @@ -14,6 +14,9 @@ To set up: - To install Maxmind GeoIP - Debian: apt-get install python-geoip - Others: http://www.maxmind.com/app/python + - To enable recaptcha support, set the options RECAPTCHA_ENABLED, + RECAPTCHA_PUB_KEY, and RECAPTCHA_PRIV_KEY in bridgedb.conf. + A recaptcha.net account is required. To re-generate and update the i18n files (in case translated strings have changed in BridgeDB): diff --git a/bridgedb.conf b/bridgedb.conf index 11afc02..e5d7314 100644 --- a/bridgedb.conf +++ b/bridgedb.conf @@ -147,3 +147,10 @@ EMAIL_INCLUDE_FINGERPRINTS=False RESERVED_SHARE=2 FILE_BUCKETS = {} + +# Options related to recaptcha support. +# Enable/Disable recaptcha +RECAPTCHA_ENABLED = False +# Recaptcha API keys +RECAPTCHA_PUB_KEY = '' +RECAPTCHA_PRIV_KEY = '' diff --git a/lib/bridgedb/I18n.py b/lib/bridgedb/I18n.py index 750ca4a..24629f1 100644 --- a/lib/bridgedb/I18n.py +++ b/lib/bridgedb/I18n.py @@ -58,5 +58,13 @@ emails is: """), # BRIDGEDB_TEXT[11] _("""hours"""), # BRIDGEDB_TEXT[12] - _("""All further emails will be ignored.""") + _("""All further emails will be ignored."""), + # BRIDGEDB_TEXT[13] + _("""Type the two words"""), + # BRIDGEDB_TEXT[14] + _("""I am human"""), + # BRIDGEDB_TEXT[15] + _("""Upgrade your browser to Firefox"""), + # BRIDGEDB_TEXT[16] + _("""(Might be blocked)""") ] diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 42f78c3..2ef56bf 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -95,7 +95,10 @@ CONFIG = Conf( RESERVED_SHARE=2, - FILE_BUCKETS = {} + FILE_BUCKETS = {}, + RECAPTCHA_ENABLED = False, + RECAPTCHA_PUB_KEY = '', + RECAPTCHA_PRIV_KEY = '', ) def configureLogging(cfg): diff --git a/lib/bridgedb/Raptcha.py b/lib/bridgedb/Raptcha.py new file mode 100644 index 0000000..1af8dd6 --- /dev/null +++ b/lib/bridgedb/Raptcha.py @@ -0,0 +1,40 @@ +# Raptcha +# Copyright (c) 2010, The Tor Project, Inc. +# See LICENSE for licensing information + +""" +This module wraps the recaptcha api and proxies requests to protect privacy. +""" +import recaptcha.client.captcha as captcha +from BeautifulSoup import BeautifulSoup +import urllib2 + +class Raptcha(): + """ A recaptcha captcha and method to request them """ + + def __init__(self, pubkey=None, privkey=None): + self.pubkey = pubkey + self.privkey = privkey + self.image = None + self.challenge = None + + def get(self): + """ gets a fresh captcha """ + + if (self.pubkey == '') or (self.privkey == ''): + raise RaptchaKeyError + urlbase = captcha.API_SERVER + form = "/noscript?k=%s" % self.pubkey + + # extract and store image from captcha + html = urllib2.urlopen(urlbase+form).read() + soup = BeautifulSoup(html) + imgurl = urlbase+"/"+ soup.find('img')['src'] + self.challenge = str(soup.find('input', {'name' : 'recaptcha_challenge_field'})['value']) + self.image = urllib2.urlopen(imgurl).read() + +class RaptchaKeyError(Exception): + """ Exception raised when recaptcha API keys are not supplied""" + def __init__(self): + msg = 'You must supply recaptcha API keys' + Exception.__init__(self, msg) diff --git a/lib/bridgedb/Server.py b/lib/bridgedb/Server.py index 9cbeb15..1089598 100644 --- a/lib/bridgedb/Server.py +++ b/lib/bridgedb/Server.py @@ -26,6 +26,12 @@ import twisted.mail.smtp import bridgedb.Dist import bridgedb.I18n as I18n +import recaptcha.client.captcha as captcha +from random import randint +from bridgedb.Raptcha import Raptcha +import base64 +import textwrap + try: import GeoIP # GeoIP data object: choose database here @@ -42,7 +48,9 @@ class WebResource(twisted.web.resource.Resource): isLeaf = True def __init__(self, distributor, schedule, N=1, useForwardedHeader=False, - includeFingerprints=True, domains=[]): + includeFingerprints=True, + useRecaptcha=False,recaptchaPrivKey='', recaptchaPubKey='', + domains=[]): """Create a new WebResource. distributor -- an IPBasedDistributor object schedule -- an IntervalSchedule object @@ -57,7 +65,57 @@ class WebResource(twisted.web.resource.Resource): self.includeFingerprints = includeFingerprints self.domains = domains + # recaptcha options + self.useRecaptcha = useRecaptcha + self.recaptchaPrivKey = recaptchaPrivKey + self.recaptchaPubKey = recaptchaPubKey + def render_GET(self, request): + if self.useRecaptcha: + # get a captcha + c = Raptcha(self.recaptchaPubKey, self.recaptchaPrivKey) + c.get() + + # TODO: this does not work for versions of IE < 8.0 + imgstr = 'data:image/jpeg;base64,%s' % base64.b64encode(c.image) + HTML_CAPTCHA_TEMPLATE = self.buildHTMLMessageTemplateWithCaptcha( + getLocaleFromRequest(request), c.challenge, imgstr) + return HTML_CAPTCHA_TEMPLATE + else: + return self.getBridgeRequestAnswer(request) + + + def render_POST(self, request): + + # check captcha if recaptcha support is enabled + if self.useRecaptcha: + try: + challenge = request.args['recaptcha_challenge_field'][0] + response = request.args['recaptcha_response_field'][0] + + except: + return self.render_GET(request) + + # generate a random IP for the captcha submission + remote_ip = '%d.%d.%d.%d' % (randint(1,255),randint(1,255), + randint(1,255),randint(1,255)) + + recaptcha_response = captcha.submit(challenge, response, + self.recaptchaPrivKey, remote_ip) + if recaptcha_response.is_valid: + logging.info("Valid recaptcha from %s. Parameters were %r", + remote_ip, request.args) + else: + logging.info("Invalid recaptcha from %s. Parameters were %r", + remote_ip, request.args) + logging.info("Recaptcha error code: %s", recaptcha_response.error_code) + return self.render_GET(request) # redirect back to captcha + + return self.getBridgeRequestAnswer(request) + + def getBridgeRequestAnswer(self, request): + """ returns a response to a bridge request """ + interval = self.schedule.getInterval(time.time()) bridges = ( ) ip = None @@ -131,6 +189,44 @@ class WebResource(twisted.web.resource.Resource): return html_msg + def buildHTMLMessageTemplateWithCaptcha(self, t, challenge, img): + """Builds a translated html response with recaptcha""" + + recaptchaTemplate = textwrap.dedent("""\ + <form action="" method="POST"> + <input type="hidden" name="recaptcha_challenge_field" + id="recaptcha_challenge_field"\ + value="{recaptchaChallengeField}"> + <img width="300" height="57" alt="{bridgeDBText14}"\ + src="{recaptchaImgSrc}"> + <div class="recaptcha_input_area"> + <label for="recaptcha_response_field">{bridgeDBText12}</label> + </div> + <div> + <input name="recaptcha_response_field"\ + id="recaptcha_response_field" + type="text" autocomplete="off"> + </div> + <div> + <input type="submit" name="submit" value="{bridgeDBText13}"> + </div> + </form> + """).strip() + + recaptchaTemplate = recaptchaTemplate.format( + recaptchaChallengeField=challenge, + recaptchaImgSrc=img, + bridgeDBText12=t.gettext(I18n.BRIDGEDB_TEXT[13]), + bridgeDBText13=t.gettext(I18n.BRIDGEDB_TEXT[14]), + bridgeDBText14=t.gettext(I18n.BRIDGEDB_TEXT[15])) + + html_msg = "<html><body>" \ + + "<p>" + t.gettext(I18n.BRIDGEDB_TEXT[1]) + "</p>" \ + + "<p>" + t.gettext(I18n.BRIDGEDB_TEXT[9]) + "</p>" \ + + "<p>" + recaptchaTemplate + "</p>" \ + + "<p>" + t.gettext(I18n.BRIDGEDB_TEXT[4]) + "</p>" \ + + "</body></html>" + return html_msg def addWebServer(cfg, dist, sched): """Set up a web server. @@ -142,6 +238,9 @@ def addWebServer(cfg, dist, sched): HTTPS_PORT HTTPS_BIND_IP HTTPS_USE_IP_FROM_FORWARDED_HEADER + RECAPTCHA_ENABLED + RECAPTCHA_PUB_KEY + RECAPTCHA_PRIV_KEY dist -- an IPBasedDistributor object. sched -- an IntervalSchedule object. """ @@ -152,7 +251,10 @@ def addWebServer(cfg, dist, sched): resource = WebResource(dist, sched, cfg.HTTPS_N_BRIDGES_PER_ANSWER, cfg.HTTP_USE_IP_FROM_FORWARDED_HEADER, includeFingerprints=cfg.HTTPS_INCLUDE_FINGERPRINTS, - domains=cfg.EMAIL_DOMAINS) + useRecaptcha=cfg.RECAPTCHA_ENABLED, + domains=cfg.EMAIL_DOMAINS, + recaptchaPrivKey=cfg.RECAPTCHA_PRIV_KEY, + recaptchaPubKey=cfg.RECAPTCHA_PUB_KEY) site = Site(resource) reactor.listenTCP(cfg.HTTP_UNENCRYPTED_PORT, site, interface=ip) if cfg.HTTPS_PORT: @@ -164,7 +266,10 @@ def addWebServer(cfg, dist, sched): resource = WebResource(dist, sched, cfg.HTTPS_N_BRIDGES_PER_ANSWER, cfg.HTTPS_USE_IP_FROM_FORWARDED_HEADER, includeFingerprints=cfg.HTTPS_INCLUDE_FINGERPRINTS, - domains=cfg.EMAIL_DOMAINS) + domains=cfg.EMAIL_DOMAINS, + useRecaptcha=cfg.RECAPTCHA_ENABLED, + recaptchaPrivKey=cfg.RECAPTCHA_PRIV_KEY, + recaptchaPubKey=cfg.RECAPTCHA_PUB_KEY) site = Site(resource) reactor.listenSSL(cfg.HTTPS_PORT, site, factory, interface=ip) return site @@ -381,6 +486,17 @@ def getLocaleFromPlusAddr(address): return replyLocale +def getLocaleFromRequest(request): + # See if we did get a request for a certain locale, otherwise fall back + # to 'en': + # Try evaluating the path /foo first, then check if we got a ?lang=foo + default_lang = lang = "en" + if len(request.path) > 1: + lang = request.path[1:] + if lang == default_lang: + lang = request.args.get("lang", [default_lang]) + lang = lang[0] + return I18n.getLang(lang) class MailContext: """Helper object that holds information used by email subsystem."""

1 0

[gettor/master] Insert a dirty hack into MakeStat.py as long as we must keep
by kaner＠torproject.org 19 Sep '11

19 Sep '11

commit 56176f0babfb3623279b3dc0732e78e23a3e6e41 Author: Christian Fromme <kaner(a)strace.org> Date: Mon Sep 12 20:07:17 2011 +0200 Insert a dirty hack into MakeStat.py as long as we must keep log files with old packages names around.. --- MakeStat.py | 16 ++++++++++++++++ 1 files changed, 16 insertions(+), 0 deletions(-) diff --git a/MakeStat.py b/MakeStat.py index f124370..24ceeaf 100644 --- a/MakeStat.py +++ b/MakeStat.py @@ -12,6 +12,7 @@ def makestats(filename, configPackages): # Initialize package counter packageCounter = { 'None': 0} + dateInfo = "" for k in configPackages.keys(): packageCounter[k] = 0 @@ -30,6 +31,21 @@ def makestats(filename, configPackages): reqInfo = ast.literal_eval(match.group(2)) package = reqInfo['package'] if package is not None: + match = re.match("tor-browser-bundle(_.*)", package) + if match: + package = "windows" + match.group(1) + match = re.match("linux-browser-bundle(-.*)", package) + if match: + package = "linux" + match.group(1) + match = re.match("(macosx-i386)(.*)", package) + if match: + package = "macosx-i386" + match = re.match("(macosx-ppc)(.*)", package) + if match: + package = "macosx-ppc" + match = re.match("windows-bundle", package) + if match: + continue packageCounter[package] += 1 else: packageCounter['None'] += 1

1 0

[gettor/master] Configuration file correction: s/macosx/macos/
by kaner＠torproject.org 19 Sep '11

19 Sep '11

commit ea20c37577818f64f00e37f056bcdb936a058457 Author: Christian Fromme <kaner(a)strace.org> Date: Sun Sep 18 21:27:41 2011 +0200 Configuration file correction: s/macosx/macos/ --- gettor.conf | 28 ++++++++++++++-------------- 1 files changed, 14 insertions(+), 14 deletions(-) diff --git a/gettor.conf b/gettor.conf index 29a2494..56f8c35 100644 --- a/gettor.conf +++ b/gettor.conf @@ -95,46 +95,46 @@ PACKAGES = { "source": ("tor-*.tar.gz", "unavailable"), - "macosx-ppc": + "macos-ppc": ("vidalia-bundles/vidalia-bundle-*-ppc.dmg", "unavailable"), - "macosx-i386": + "macos-i386": ("torbrowser/osx/TorBrowser-*-dev-osx-i386-en-US.zip", "unavailable"), - "macosx-i386_en": + "macos-i386_en": ("torbrowser/osx/TorBrowser-*-dev-osx-i386-en-US.zip", "unavailable"), - "macosx-i386_ar": + "macos-i386_ar": ("torbrowser/osx/TorBrowser-*-dev-osx-i386-ar.zip", "unavailable"), - "macosx-i386_de": + "macos-i386_de": ("torbrowser/osx/TorBrowser-*-dev-osx-i386-de.zip", "unavailable"), - "macosx-i386_es-ES": + "macos-i386_es-ES": ("torbrowser/osx/TorBrowser-*-dev-osx-i386-es-ES.zip", "unavailable"), - "macosx-i386_fa": + "macos-i386_fa": ("torbrowser/osx/TorBrowser-*-dev-osx-i386-fa.zip", "unavailable"), - "macosx-i386_fr": + "macos-i386_fr": ("torbrowser/osx/TorBrowser-*-dev-osx-i386-fr.zip", "unavailable"), - "macosx-i386_it": + "macos-i386_it": ("torbrowser/osx/TorBrowser-*-dev-osx-i386-it.zip", "unavailable"), - "macosx-i386_nl": + "macos-i386_nl": ("torbrowser/osx/TorBrowser-*-dev-osx-i386-nl.zip", "unavailable"), - "macosx-i386_pl": + "macos-i386_pl": ("torbrowser/osx/TorBrowser-*-dev-osx-i386-pl.zip", "unavailable"), - "macosx-i386_pt": + "macos-i386_pt": ("torbrowser/osx/TorBrowser-*-dev-osx-i386-pt-PT.zip", "unavailable"), - "macosx-i386_ru": + "macos-i386_ru": ("torbrowser/osx/TorBrowser-*-dev-osx-i386-ru.zip", "unavailable"), - "macosx-i386_zh_CN": + "macos-i386_zh_CN": ("torbrowser/osx/TorBrowser-*-dev-osx-i386-zh-CN.zip", "unavailable"), "linux-i386":

1 0