October 2015 - tor-commits - lists.torproject.org

[doctor/master] Deduplicate endpoints we alarm on
by atagar＠torproject.org 04 Oct '15

04 Oct '15

commit 214fc5d1408d424b5152aa2ebcd79e8bd43a6669 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Oct 4 14:21:28 2015 -0700 Deduplicate endpoints we alarm on Seems an address/port can appear in a consensus more than once. Deduplicating those. --- fingerprint_change_checker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fingerprint_change_checker.py b/fingerprint_change_checker.py index 72cca9b..a0ae061 100755 --- a/fingerprint_change_checker.py +++ b/fingerprint_change_checker.py @@ -31,8 +31,8 @@ log = util.get_logger('fingerprint_change_checker') def main(): fingerprint_changes = load_fingerprint_changes() - downloader = DescriptorDownloader(timeout = 60) - alarm_for = [] + downloader = DescriptorDownloader(timeout = 15) + alarm_for = set() for relay in downloader.get_consensus(): prior_fingerprints = fingerprint_changes.setdefault((relay.address, relay.or_port), {}) @@ -52,7 +52,7 @@ def main(): # if we've changed more than three times in the last thirty days then alarm if len(prior_fingerprints) >= 3: - alarm_for.append((relay.address, relay.or_port)) + alarm_for.add((relay.address, relay.or_port)) if alarm_for: log.debug("Sending a notification for %i relays..." % len(alarm_for))

1 0

[doctor/master] Can't modify during iteration
by atagar＠torproject.org 04 Oct '15

04 Oct '15

commit db8fc25e902c913f3ed683960681cef8dd7444a5 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Oct 4 13:41:40 2015 -0700 Can't modify during iteration Oops, we'd hit 'RuntimeError: dictionary changed size during iteration' during the bit where we clear out old entries. --- fingerprint_change_checker.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/fingerprint_change_checker.py b/fingerprint_change_checker.py index 289f3c0..72cca9b 100755 --- a/fingerprint_change_checker.py +++ b/fingerprint_change_checker.py @@ -43,10 +43,11 @@ def main(): # drop fingerprint changes that are over thirty days old - for fp in prior_fingerprints: - if time.time() - prior_fingerprints[fp] > THIRTY_DAYS: - log.debug("Removing fingerprint for %s:%s (%s) which was published %i days ago" % (relay.address, relay.or_port, fp, prior_fingerprints[fp] / 60 / 60 / 24)) - del prior_fingerprints[fp] + old_fingerprints = [fp for fp in prior_fingerprints if (time.time() - prior_fingerprints[fp] > THIRTY_DAYS)] + + for fp in old_fingerprints: + log.debug("Removing fingerprint for %s:%s (%s) which was published %i days ago" % (relay.address, relay.or_port, fp, prior_fingerprints[fp] / 60 / 60 / 24)) + del prior_fingerprints[fp] # if we've changed more than three times in the last thirty days then alarm

1 0

[doctor/master] Script to check for fingerprint changes
by atagar＠torproject.org 04 Oct '15

04 Oct '15

commit 6ad1ce1e3d9f1d088fab8fcc55b7ea9712c9cc42 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Oct 4 13:25:29 2015 -0700 Script to check for fingerprint changes During our recent dev meeting David Goulet asked me for a DocTor check for relays rapidly changing their fingerprint. Doing so likely means they're trying to get into a privilaged position to snoop on hidden services. Plan is to keep an eye on this script for a bit (just whipped it up, no doubt it has issues), then send the results to David when we have some confidence in it. Later the results will be sent to bad-relays@. --- .gitignore | 1 + consensus_health_checker.py | 2 +- descriptor_checker.py | 2 +- fingerprint_change_checker.py | 128 +++++++++++++++++++++++++++++++++++++++++ sybil_checker.py | 6 +- 5 files changed, 134 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 1a47934..c320e41 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ logs/ data/contact_information.cfg data/fingerprints +data/fingerprint_changes data/last_notified.cfg stem gmail_pw diff --git a/consensus_health_checker.py b/consensus_health_checker.py index 8733cb4..82c21c7 100755 --- a/consensus_health_checker.py +++ b/consensus_health_checker.py @@ -742,7 +742,7 @@ def _get_documents(label, resource): for authority, query in queries.items(): try: documents[authority] = query.run()[0] - except Exception, exc: + except Exception as exc: if label == 'vote': # try to download the vote via the other authorities diff --git a/descriptor_checker.py b/descriptor_checker.py index 8050ca8..434b4d6 100755 --- a/descriptor_checker.py +++ b/descriptor_checker.py @@ -95,7 +95,7 @@ def send_email(subject, descriptor_type, query): try: timestamp = datetime.datetime.now().strftime("%m/%d/%Y %H:%M") util.send(subject, body = EMAIL_BODY % (descriptor_type, query.download_url, timestamp, query.error), to = [util.ERROR_ADDRESS]) - except Exception, exc: + except Exception as exc: log.warn("Unable to send email: %s" % exc) diff --git a/fingerprint_change_checker.py b/fingerprint_change_checker.py new file mode 100755 index 0000000..289f3c0 --- /dev/null +++ b/fingerprint_change_checker.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python +# Copyright 2015, Damian Johnson and The Tor Project +# See LICENSE for licensing information + +""" +Simple script that checks to see if relays rapidly change their finterprint. +This can indicate malicious intent toward hidden services. +""" + +import datetime +import time +import traceback + +import util + +from stem.descriptor.remote import DescriptorDownloader +from stem.util import datetime_to_unix, conf + +EMAIL_SUBJECT = 'Relays Changing Fingerprint' + +EMAIL_BODY = """\ +The following relays are frequently changing their fingerprints... + +""" + +FINGERPRINT_CHANGES_FILE = util.get_path('data', 'fingerprint_changes') +THIRTY_DAYS = 30 * 24 * 60 * 60 + +log = util.get_logger('fingerprint_change_checker') + + +def main(): + fingerprint_changes = load_fingerprint_changes() + downloader = DescriptorDownloader(timeout = 60) + alarm_for = [] + + for relay in downloader.get_consensus(): + prior_fingerprints = fingerprint_changes.setdefault((relay.address, relay.or_port), {}) + + if relay.fingerprint not in prior_fingerprints: + log.debug("Registering a new fingerprint for %s:%s (%s)" % (relay.address, relay.or_port, relay.fingerprint)) + prior_fingerprints[relay.fingerprint] = datetime_to_unix(relay.published) + + # drop fingerprint changes that are over thirty days old + + for fp in prior_fingerprints: + if time.time() - prior_fingerprints[fp] > THIRTY_DAYS: + log.debug("Removing fingerprint for %s:%s (%s) which was published %i days ago" % (relay.address, relay.or_port, fp, prior_fingerprints[fp] / 60 / 60 / 24)) + del prior_fingerprints[fp] + + # if we've changed more than three times in the last thirty days then alarm + + if len(prior_fingerprints) >= 3: + alarm_for.append((relay.address, relay.or_port)) + + if alarm_for: + log.debug("Sending a notification for %i relays..." % len(alarm_for)) + body = EMAIL_BODY + + for address, or_port in alarm_for: + fp_changes = fingerprint_changes[(address, or_port)] + log.debug("* %s:%s has had %i fingerprints: %s" % (address, or_port, len(fp_changes), ', '.join(fp_changes.keys()))) + body += "* %s:%s\n" % (address, or_port) + + for fingerprint, published in fp_changes.items(): + body += " %s at %s\n" % (fingerprint, datetime.datetime.fromtimestamp(published).strftime('%Y-%m-%d %H:%M:%S')) + + body += "\n" + + try: + util.send(EMAIL_SUBJECT, body = body, to = ['atagar(a)torproject.org']) + except Exception as exc: + log.warn("Unable to send email: %s" % exc) + + save_fingerprint_changes(fingerprint_changes) + + +def load_fingerprint_changes(): + """ + Loads information about prior fingerprint changes we've persisted. This + provides a dictionary of the form... + + (address, or_port) => {fingerprint: published_timestamp...} + """ + + log.debug("Loading fingerprint changes...") + config = conf.get_config('fingerprint_changes') + + try: + config.load(FINGERPRINT_CHANGES_FILE) + fingerprint_changes = {} + + for key in config.keys(): + address, or_port = key.split(':', 1) + + for value in config.get(key, []): + fingerprint, published = value.split(':', 1) + fingerprint_changes.setdefault((address, int(or_port)), {})[fingerprint] = float(published) + + log.debug(" information for %i relays found" % len(fingerprint_changes)) + return fingerprint_changes + except IOError as exc: + log.debug(" unable to read '%s': %s" % (FINGERPRINT_CHANGES_FILE, exc)) + return {} + + +def save_fingerprint_changes(fingerprint_changes): + log.debug("Saving fingerprint changes for %i relays" % len(fingerprint_changes)) + config = conf.get_config('fingerprint_changes') + config.clear() + + for address, or_port in fingerprint_changes: + for fingerprint, published in fingerprint_changes[(address, or_port)].items(): + config.set('%s:%s' % (address, or_port), '%s:%s' % (fingerprint, published), overwrite = False) + + try: + config.save(FINGERPRINT_CHANGES_FILE) + except IOError as exc: + log.debug(" unable to save '%s': %s" % (FINGERPRINT_CHANGES_FILE, exc)) + + +if __name__ == '__main__': + try: + main() + except: + msg = "fingerprint_change_checker.py failed with:\n\n%s" % traceback.format_exc() + log.error(msg) + util.send("Script Error", body = msg, to = [util.ERROR_ADDRESS]) diff --git a/sybil_checker.py b/sybil_checker.py index c51d753..52fba0c 100755 --- a/sybil_checker.py +++ b/sybil_checker.py @@ -94,7 +94,7 @@ def send_email(new_relays): body += "\n".join(relay_entries) util.send(EMAIL_SUBJECT, body = body) - except Exception, exc: + except Exception as exc: log.warn("Unable to send email: %s" % exc) @@ -117,7 +117,7 @@ def load_fingerprints(): log.debug(" %i fingerprints found" % len(fingerprints)) return set(fingerprints) - except Exception, exc: + except Exception as exc: log.debug(" unable to read '%s': %s" % (FINGERPRINTS_FILE, exc)) return set() @@ -131,7 +131,7 @@ def save_fingerprints(fingerprints): with open(FINGERPRINTS_FILE, 'w') as fingerprint_file: fingerprint_file.write('\n'.join(fingerprints)) - except Exception, exc: + except Exception as exc: log.debug("Unable to save fingerprints to '%s': %s" % (FINGERPRINTS_FILE, exc))

1 0

[stem/master] Note that Config's save() method can raise an IOError
by atagar＠torproject.org 04 Oct '15

04 Oct '15

commit b629115033b77bbffcd81240a48774c860dd1eea Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Oct 4 12:03:07 2015 -0700 Note that Config's save() method can raise an IOError Oops, seems we forgot to note this. Of course this method can raise IOErrors. --- stem/util/conf.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/stem/util/conf.py b/stem/util/conf.py index 6b0efd7..92be685 100644 --- a/stem/util/conf.py +++ b/stem/util/conf.py @@ -535,7 +535,9 @@ class Config(object): :param str path: location to be saved to - :raises: **ValueError** if no path was provided and we've never been provided one + :raises: + * **IOError** if we fail to save the file (insufficient permissions, etc) + * **ValueError** if no path was provided and we've never been provided one """ if path:

1 0

[stem/master] Adding datetime_to_unix() util function
by atagar＠torproject.org 04 Oct '15

04 Oct '15

commit fe03a01c97cb9fd41a07f7f1bb6facb633fdbbd2 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Oct 4 11:46:16 2015 -0700 Adding datetime_to_unix() util function Our descriptors provide datetime objects. This is great, but often we want a unix timestamp instead. Python's datetime builtin truly sucks so need a util for this. --- docs/change_log.rst | 4 ++++ stem/util/__init__.py | 16 ++++++++++++++++ test/settings.cfg | 1 + test/unit/util/__init__.py | 12 ++++++++++++ 4 files changed, 33 insertions(+) diff --git a/docs/change_log.rst b/docs/change_log.rst index 727b0cb..cfd26d1 100644 --- a/docs/change_log.rst +++ b/docs/change_log.rst @@ -61,6 +61,10 @@ The following are only available within Stem's `git repository * Unable to read descriptors from data directories on Windows due to their CRLF newlines (:trac:`17051`) * TypeError under python3 when using 'use_mirrors = True' (:trac:`17083`) + * **Utilities** + + * Added :func:`stem.util.datetime_to_unix` + * **Interpreter** * Added a '--tor [path]' argument to specify the tor binary to run. diff --git a/stem/util/__init__.py b/stem/util/__init__.py index 1c18df3..b7f0a3c 100644 --- a/stem/util/__init__.py +++ b/stem/util/__init__.py @@ -18,3 +18,19 @@ __all__ = [ 'test_tools', 'tor_tools', ] + +import datetime + + +def datetime_to_unix(timestamp): + """ + Converts a utc datetime object to a unix timestamp. + + .. versionadded:: 1.5.0 + + :param datetime timestamp: timestamp to be converted + + :returns: **float** for the unix timestamp of the given datetime object + """ + + return (timestamp - datetime.datetime(1970, 1, 1)).total_seconds() diff --git a/test/settings.cfg b/test/settings.cfg index 0b050d5..fffbdcf 100644 --- a/test/settings.cfg +++ b/test/settings.cfg @@ -158,6 +158,7 @@ test.unit_tests |test.unit.util.str_tools.TestStrTools |test.unit.util.system.TestSystem |test.unit.util.tor_tools.TestTorTools +|test.unit.util.__init__.TestBaseUtil |test.unit.descriptor.export.TestExport |test.unit.descriptor.reader.TestDescriptorReader |test.unit.descriptor.remote.TestDescriptorDownloader diff --git a/test/unit/util/__init__.py b/test/unit/util/__init__.py index 04ab051..e755625 100644 --- a/test/unit/util/__init__.py +++ b/test/unit/util/__init__.py @@ -11,3 +11,15 @@ __all__ = [ 'system', 'tor_tools', ] + +import datetime +import time +import unittest + +from stem.util import datetime_to_unix + + +class TestBaseUtil(unittest.TestCase): + def test_datetime_to_unix(self): + self.assertEqual(1344251971.0, datetime_to_unix(datetime.datetime(2012, 8, 6, 11, 19, 31))) + self.assertTrue((time.time() - datetime_to_unix(datetime.datetime.utcnow())) < 2)

1 0

[tor/master] fix easy typo
by arma＠torproject.org 04 Oct '15

04 Oct '15

commit c9cb5516ab3380f266c7f5db68899cb4cff49202 Author: Roger Dingledine <arma(a)torproject.org> Date: Sun Oct 4 12:28:25 2015 -0400 fix easy typo --- src/or/or.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/or/or.h b/src/or/or.h index a80cd55..651d8be 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -4015,7 +4015,7 @@ typedef struct { char *ConsensusParams; /** Authority only: minimum number of measured bandwidths we must see - * before we only beliee measured bandwidths to assign flags. */ + * before we only believe measured bandwidths to assign flags. */ int MinMeasuredBWsForAuthToIgnoreAdvertised; /** The length of time that we think an initial consensus should be fresh.

1 0

[translation/https_everywhere_completed] Update translations for https_everywhere_completed
by translation＠torproject.org 04 Oct '15

04 Oct '15

commit 31bb4ae55fd683351151a53b3d5e58ad418b2717 Author: Translation commit bot <translation(a)torproject.org> Date: Sun Oct 4 13:15:22 2015 +0000 Update translations for https_everywhere_completed --- ta/https-everywhere.dtd | 1 + 1 file changed, 1 insertion(+) diff --git a/ta/https-everywhere.dtd b/ta/https-everywhere.dtd index ac570c2..d9da355 100644 --- a/ta/https-everywhere.dtd +++ b/ta/https-everywhere.dtd @@ -17,6 +17,7 @@ <!ENTITY https-everywhere.menu.globalDisable "எல்லா இடங்களிலும் HTTPS முடக்கு"> <!ENTITY https-everywhere.menu.blockHttpRequests "அனைத்து HTTP கோரிக்கைகள் தடு"> <!ENTITY https-everywhere.menu.showCounter "காட்டு கருமபீடம்"> +<!ENTITY https-everywhere.menu.viewAllRules "அனைத்து விதிகள் காண்கவும்"> <!ENTITY https-everywhere.prefs.title "HTTPS எல்லா இடங்களிலும் விருப்பங்கள்"> <!ENTITY https-everywhere.prefs.enable_all "அனைத்தும் செயலாக்கு">

1 0

[translation/https_everywhere] Update translations for https_everywhere
by translation＠torproject.org 04 Oct '15

04 Oct '15

commit 7d02f280889151e99019b3caa2f76a36ee95c8bb Author: Translation commit bot <translation(a)torproject.org> Date: Sun Oct 4 13:15:18 2015 +0000 Update translations for https_everywhere --- ta/https-everywhere.dtd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ta/https-everywhere.dtd b/ta/https-everywhere.dtd index 29961eb..d9da355 100644 --- a/ta/https-everywhere.dtd +++ b/ta/https-everywhere.dtd @@ -17,7 +17,7 @@ <!ENTITY https-everywhere.menu.globalDisable "எல்லா இடங்களிலும் HTTPS முடக்கு"> <!ENTITY https-everywhere.menu.blockHttpRequests "அனைத்து HTTP கோரிக்கைகள் தடு"> <!ENTITY https-everywhere.menu.showCounter "காட்டு கருமபீடம்"> -<!ENTITY https-everywhere.menu.viewAllRules "View All Rules"> +<!ENTITY https-everywhere.menu.viewAllRules "அனைத்து விதிகள் காண்கவும்"> <!ENTITY https-everywhere.prefs.title "HTTPS எல்லா இடங்களிலும் விருப்பங்கள்"> <!ENTITY https-everywhere.prefs.enable_all "அனைத்தும் செயலாக்கு">

1 0

[translation/whisperback] Update translations for whisperback
by translation＠torproject.org 04 Oct '15

04 Oct '15

commit eb574438bb220752c52cd7ee09050344631d43e4 Author: Translation commit bot <translation(a)torproject.org> Date: Sun Oct 4 13:15:12 2015 +0000 Update translations for whisperback --- ta/ta.po | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ta/ta.po b/ta/ta.po index be41c07..d17c2cf 100644 --- a/ta/ta.po +++ b/ta/ta.po @@ -4,6 +4,7 @@ # # Translators: # annes badusha <badusha000(a)gmail.com>, 2014 +# Arun M S <arunistheking(a)gmail.com>, 2015 # git12a <git12(a)openmailbox.org>, 2015 # Khaleel Jageer <jskcse4(a)gmail.com>, 2014 msgid "" @@ -11,9 +12,9 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2014-03-17 17:40+0100\n" -"PO-Revision-Date: 2015-02-13 10:18+0000\n" -"Last-Translator: git12a <git12(a)openmailbox.org>\n" -"Language-Team: Tamil (http://www.transifex.com/projects/p/torproject/language/ta/)\n" +"PO-Revision-Date: 2015-10-04 12:50+0000\n" +"Last-Translator: Arun M S <arunistheking(a)gmail.com>\n" +"Language-Team: Tamil (http://www.transifex.com/otf/torproject/language/ta/)\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" @@ -40,7 +41,7 @@ msgstr "செல்லா தொடர்பு OpenPGP ஆணை பொ msgid "" "The %s variable was not found in any of the configuration files " "/etc/whisperback/config.py, ~/.whisperback/config.py, ./config.py" -msgstr "" +msgstr "%s வெரியபில்ஸ் எந்த கட்டமைப்பு கோப்புகளிலும் காணவில்லை\n/etc/whisperback/config.py, ~/.whisperback/config.py, ./config.py" #: ../whisperBack/gui.py:154 msgid "Unable to load a valid configuration."

1 0

[translation/abouttor-homepage_completed] Update translations for abouttor-homepage_completed
by translation＠torproject.org 04 Oct '15

04 Oct '15

commit 1a5e7bd2774ab9014c290c54a7886ebc180dc701 Author: Translation commit bot <translation(a)torproject.org> Date: Sun Oct 4 12:45:39 2015 +0000 Update translations for abouttor-homepage_completed --- ta/aboutTor.dtd | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ta/aboutTor.dtd b/ta/aboutTor.dtd index 5e270c6..4f227fa 100644 --- a/ta/aboutTor.dtd +++ b/ta/aboutTor.dtd @@ -1,5 +1,5 @@  @@ -8,7 +8,8 @@ <!ENTITY aboutTor.outOfDateTorOn.label "எனினும், இந்த இணைய உலாவி காலாவதியானது"> <!ENTITY aboutTor.outOfDateTorOff.label "மேலும், இந்த இணைய உலாவி காலாவதியானது"> -<!ENTITY aboutTor.outOfDate2.label "வெங்காயத்தில் கிளிக் செய்த, பின்னர் Tor உலாவி மூட்டையை புதுப்பிக்க பதிவிறக்கத்தை தேர்வு செய்யவும்."> +<!ENTITY aboutTor.outOfDate2.label "வெங்காயத்தின் மீது சொடுக்கவும். பின்னர் +Tor உலாவியை புதுப்பிக்க பதிவிறக்கத்தை தேர்வு செய்யவும்."> <!ENTITY aboutTor.check.label "Tor பிணைய அமைவுகளை சோதனை செய்"> @@ -25,6 +26,7 @@ <!ENTITY aboutTor.searchSPPost.link "https://startpage.com/rth/search"> <!ENTITY aboutTor.searchDDGPost.link "https://duckduckgo.com/html/"> +<!ENTITY aboutTor.searchDCPost.link "https://search.disconnect.me/searchTerms/search?ses=Google&location_opt…"> <!ENTITY aboutTor.torInfo1.label "கூடுதல் தகவல்:"> <!ENTITY aboutTor.torInfo2.label "நாடு மற்றும் IP முகவரி:">

1 0