July 2016 - tor-commits - lists.torproject.org

[bridgedb/master] Merge branch 'master' into develop
by isis＠torproject.org 28 Jul '16

28 Jul '16

commit c7d9cc4978f008fe657dd88653975248066dd1f6 Merge: 7931067 7d6defe Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Dec 1 02:45:11 2015 +0000 Merge branch 'master' into develop

1 0

[bridgedb/master] Clean up *.unparseable descriptor files more than 24 hours old.
by isis＠torproject.org 28 Jul '16

28 Jul '16

commit b6c740f79a4b9263e49dfe9f14314425b496aa40 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon May 2 14:03:21 2016 +0000 Clean up *.unparseable descriptor files more than 24 hours old. * ADD a config options for a new DELETE_UNPARSEABLE_DESCRIPTORS task, which, by default, runs once every 24 hours and removes *.unparseable descriptor files more than 24 hours old. * ADD file deletion utility `bridgedb.util.deleteFilesOlderThan` and a scheduled function, `bridgedb.runner.cleanupUnparseableDescriptors`, which calls the former. * ADD unittests for `bridgedb.util.deleteFilesOlderThan`. * FIXES #18237: https://bugs.torproject.org/18237 --- bridgedb.conf | 2 ++ bridgedb/Main.py | 12 ++++++++++++ bridgedb/runner.py | 29 +++++++++++++++++++++++++++++ bridgedb/util.py | 21 +++++++++++++++++++++ test/test_util.py | 31 +++++++++++++++++++++++++++++++ 5 files changed, 95 insertions(+) diff --git a/bridgedb.conf b/bridgedb.conf index 7805e15..52a8ca7 100644 --- a/bridgedb.conf +++ b/bridgedb.conf @@ -260,6 +260,8 @@ TASKS = { # scripts/get-exit-list) and add those exit relays to the list of proxies # loaded from the PROXY_LIST_FILES: 'GET_TOR_EXIT_LIST': 3 * 60 * 60, + # Delete *.unparseable descriptor files which are more than 24 hours old: + 'DELETE_UNPARSEABLE_DESCRIPTORS': 24 * 60 * 60, } # SUPPORTED_TRANSPORTS is a dictionary mapping Pluggable Transport methodnames diff --git a/bridgedb/Main.py b/bridgedb/Main.py index b281d21..bf4c213 100644 --- a/bridgedb/Main.py +++ b/bridgedb/Main.py @@ -23,6 +23,7 @@ from twisted.internet import task from bridgedb import crypto from bridgedb import persistent from bridgedb import proxy +from bridgedb import runner from bridgedb import util from bridgedb.bridges import MalformedBridgeInfo from bridgedb.bridges import MissingServerDescriptorDigest @@ -453,6 +454,17 @@ def run(options, reactor=reactor): state.proxies, config.SERVER_PUBLIC_EXTERNAL_IP) + if config.TASKS.get('DELETE_UNPARSEABLE_DESCRIPTORS'): + delUnparseableSecs = config.TASKS['DELETE_UNPARSEABLE_DESCRIPTORS'] + else: + delUnparseableSecs = 24 * 60 * 60 # Default to 24 hours + + # We use the directory name of STATUS_FILE, since that directory + # is where the *.unparseable descriptor files will be written to. + tasks['DELETE_UNPARSEABLE_DESCRIPTORS'] = task.LoopingCall( + runner.cleanupUnparseableDescriptors, + os.path.dirname(config.STATUS_FILE), delUnparseableSecs) + # Schedule all configured repeating tasks: for name, seconds in config.TASKS.items(): if seconds: diff --git a/bridgedb/runner.py b/bridgedb/runner.py index 6ac069f..597b1b2 100644 --- a/bridgedb/runner.py +++ b/bridgedb/runner.py @@ -17,12 +17,41 @@ from __future__ import print_function +import glob import logging import sys import os from twisted.python import procutils +from bridgedb import util + + +def cleanupUnparseableDescriptors(directory, seconds): + """Delete any ``*.unparseable`` descriptor files in ``directory`` with + mtimes more than ``seconds`` ago. + + The :func:`bridgedb.parsers._copyUnparseableDescriptors` function + will make copies of any files we attempt to parse which contain + unparseable descriptors. This function should run on a timer to + clean them up. + + :param str directory: The directory in which to search for unparseable + descriptors. + :param int olderThan: If a file's mtime is more than this number + (in seconds), it will be deleted. + """ + files = [] + + for pattern in ["*.unparseable", "*.unparseable.xz"]: + files.extend(glob.glob(os.sep.join([directory, pattern]))) + + if files: + logging.info("Deleting old unparseable descriptor files...") + logging.debug("Considered for deletion: %s" % "\n".join(files)) + + deleted = util.deleteFilesOlderThan(files, seconds) + logging.info("Deleted %d unparseable descriptor files." % len(deleted)) def find(filename): """Find the executable ``filename``. diff --git a/bridgedb/util.py b/bridgedb/util.py index 4c558c4..42e4664 100644 --- a/bridgedb/util.py +++ b/bridgedb/util.py @@ -18,6 +18,7 @@ import logging import logging.config import logging.handlers import os +import time from twisted.python import components @@ -144,6 +145,26 @@ def configureLogging(cfg): logging.info("Level: %s", logLevel) logging.info("Safe Logging: %sabled" % ("En" if safelogging else "Dis")) +def deleteFilesOlderThan(files, seconds): + """Delete any file in ``files`` with an mtime more than ``seconds`` ago. + + :param list files: A list of paths to files which should be + considered for deletion. + :param int seconds: If a file's mtime is more than this number (in + seconds), it will be deleted. + :rtype: list + :returns: A list of the deleted files. + """ + deleted = [] + now = int(time.time()) + + for fn in files: + if (now - os.stat(fn).st_mtime) > seconds: + os.unlink(fn) + deleted.append(fn) + + return deleted + def levenshteinDistance(s1, s2, len1=None, len2=None, offset1=0, offset2=0, memo=None): """Compute the Levenstein Distance between two strings. diff --git a/test/test_util.py b/test/test_util.py index da4ddf4..848ce12 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -17,6 +17,7 @@ from __future__ import unicode_literals import logging import os +import time from twisted.mail.smtp import Address from twisted.trial import unittest @@ -71,6 +72,36 @@ class MiscLoggingUtilTests(unittest.TestCase): util.logging.info("BridgeDB's email address: bridges(a)torproject.org") +class FileUtilityTests(unittest.TestCase): + """Unittests for `bridgedb.util.deleteFilesOlderThan`.""" + + def setUp(self): + self._directory = self.id() + self.newfile = os.sep.join([self._directory, "newfile"]) + self.oldfile = os.sep.join([self._directory, "oldfile"]) + self.testfiles = [self.newfile, self.oldfile] + os.mkdir(self._directory) + + now = time.time() + + for fn in self.testfiles: + with open(fn, "w") as fd: + fd.flush() + + # Change the mtime of the "oldfile" to be two days old: + os.utime(self.oldfile, (now, now - (48 * 60 * 60))) + + def test_deleteFilesOlderThan_deletes_old_files(self): + """The function should delete appropriate files.""" + deleted = util.deleteFilesOlderThan(self.testfiles ,24 * 60 * 60) + self.assertIn(self.oldfile, deleted) + + def test_deleteFilesOlderThan_keeps_new_files(self): + """The function should delete appropriate files.""" + deleted = util.deleteFilesOlderThan(self.testfiles ,24 * 60 * 60) + self.assertNotIn(self.newfile, deleted) + + class LevenshteinDistanceTests(unittest.TestCase): """Unittests for `bridgedb.util.levenshteinDistance."""

1 0

[bridgedb/develop] Bump the Twisted version.
by isis＠torproject.org 28 Jul '16

28 Jul '16

commit 2f3c79c9fad27cdb04ee50573a462f4f9530588e Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon May 30 15:34:56 2016 +0000 Bump the Twisted version. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9e4aac1..4591511 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ Babel==0.9.6 BeautifulSoup==3.2.1 Mako==0.8.1 -Twisted==13.2.0 +Twisted==14.0.2 https://ipaddr-py.googlecode.com/files/ipaddr-2.1.10.tar.gz#sha1=c608450b07… pycrypto==2.6.1 pyOpenSSL==0.14

1 0

[bridgedb/develop] Merge branch 'master' into develop
by isis＠torproject.org 28 Jul '16

28 Jul '16

commit fb309152cc2db7ce49e2e104f3a8e20c54a47a8b Merge: 936958a 908294f Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Jul 28 16:38:15 2016 +0000 Merge branch 'master' into develop

1 0

[bridgedb/develop] Clean up *.unparseable descriptor files more than 24 hours old.
by isis＠torproject.org 28 Jul '16

28 Jul '16

commit b6c740f79a4b9263e49dfe9f14314425b496aa40 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon May 2 14:03:21 2016 +0000 Clean up *.unparseable descriptor files more than 24 hours old. * ADD a config options for a new DELETE_UNPARSEABLE_DESCRIPTORS task, which, by default, runs once every 24 hours and removes *.unparseable descriptor files more than 24 hours old. * ADD file deletion utility `bridgedb.util.deleteFilesOlderThan` and a scheduled function, `bridgedb.runner.cleanupUnparseableDescriptors`, which calls the former. * ADD unittests for `bridgedb.util.deleteFilesOlderThan`. * FIXES #18237: https://bugs.torproject.org/18237 --- bridgedb.conf | 2 ++ bridgedb/Main.py | 12 ++++++++++++ bridgedb/runner.py | 29 +++++++++++++++++++++++++++++ bridgedb/util.py | 21 +++++++++++++++++++++ test/test_util.py | 31 +++++++++++++++++++++++++++++++ 5 files changed, 95 insertions(+) diff --git a/bridgedb.conf b/bridgedb.conf index 7805e15..52a8ca7 100644 --- a/bridgedb.conf +++ b/bridgedb.conf @@ -260,6 +260,8 @@ TASKS = { # scripts/get-exit-list) and add those exit relays to the list of proxies # loaded from the PROXY_LIST_FILES: 'GET_TOR_EXIT_LIST': 3 * 60 * 60, + # Delete *.unparseable descriptor files which are more than 24 hours old: + 'DELETE_UNPARSEABLE_DESCRIPTORS': 24 * 60 * 60, } # SUPPORTED_TRANSPORTS is a dictionary mapping Pluggable Transport methodnames diff --git a/bridgedb/Main.py b/bridgedb/Main.py index b281d21..bf4c213 100644 --- a/bridgedb/Main.py +++ b/bridgedb/Main.py @@ -23,6 +23,7 @@ from twisted.internet import task from bridgedb import crypto from bridgedb import persistent from bridgedb import proxy +from bridgedb import runner from bridgedb import util from bridgedb.bridges import MalformedBridgeInfo from bridgedb.bridges import MissingServerDescriptorDigest @@ -453,6 +454,17 @@ def run(options, reactor=reactor): state.proxies, config.SERVER_PUBLIC_EXTERNAL_IP) + if config.TASKS.get('DELETE_UNPARSEABLE_DESCRIPTORS'): + delUnparseableSecs = config.TASKS['DELETE_UNPARSEABLE_DESCRIPTORS'] + else: + delUnparseableSecs = 24 * 60 * 60 # Default to 24 hours + + # We use the directory name of STATUS_FILE, since that directory + # is where the *.unparseable descriptor files will be written to. + tasks['DELETE_UNPARSEABLE_DESCRIPTORS'] = task.LoopingCall( + runner.cleanupUnparseableDescriptors, + os.path.dirname(config.STATUS_FILE), delUnparseableSecs) + # Schedule all configured repeating tasks: for name, seconds in config.TASKS.items(): if seconds: diff --git a/bridgedb/runner.py b/bridgedb/runner.py index 6ac069f..597b1b2 100644 --- a/bridgedb/runner.py +++ b/bridgedb/runner.py @@ -17,12 +17,41 @@ from __future__ import print_function +import glob import logging import sys import os from twisted.python import procutils +from bridgedb import util + + +def cleanupUnparseableDescriptors(directory, seconds): + """Delete any ``*.unparseable`` descriptor files in ``directory`` with + mtimes more than ``seconds`` ago. + + The :func:`bridgedb.parsers._copyUnparseableDescriptors` function + will make copies of any files we attempt to parse which contain + unparseable descriptors. This function should run on a timer to + clean them up. + + :param str directory: The directory in which to search for unparseable + descriptors. + :param int olderThan: If a file's mtime is more than this number + (in seconds), it will be deleted. + """ + files = [] + + for pattern in ["*.unparseable", "*.unparseable.xz"]: + files.extend(glob.glob(os.sep.join([directory, pattern]))) + + if files: + logging.info("Deleting old unparseable descriptor files...") + logging.debug("Considered for deletion: %s" % "\n".join(files)) + + deleted = util.deleteFilesOlderThan(files, seconds) + logging.info("Deleted %d unparseable descriptor files." % len(deleted)) def find(filename): """Find the executable ``filename``. diff --git a/bridgedb/util.py b/bridgedb/util.py index 4c558c4..42e4664 100644 --- a/bridgedb/util.py +++ b/bridgedb/util.py @@ -18,6 +18,7 @@ import logging import logging.config import logging.handlers import os +import time from twisted.python import components @@ -144,6 +145,26 @@ def configureLogging(cfg): logging.info("Level: %s", logLevel) logging.info("Safe Logging: %sabled" % ("En" if safelogging else "Dis")) +def deleteFilesOlderThan(files, seconds): + """Delete any file in ``files`` with an mtime more than ``seconds`` ago. + + :param list files: A list of paths to files which should be + considered for deletion. + :param int seconds: If a file's mtime is more than this number (in + seconds), it will be deleted. + :rtype: list + :returns: A list of the deleted files. + """ + deleted = [] + now = int(time.time()) + + for fn in files: + if (now - os.stat(fn).st_mtime) > seconds: + os.unlink(fn) + deleted.append(fn) + + return deleted + def levenshteinDistance(s1, s2, len1=None, len2=None, offset1=0, offset2=0, memo=None): """Compute the Levenstein Distance between two strings. diff --git a/test/test_util.py b/test/test_util.py index da4ddf4..848ce12 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -17,6 +17,7 @@ from __future__ import unicode_literals import logging import os +import time from twisted.mail.smtp import Address from twisted.trial import unittest @@ -71,6 +72,36 @@ class MiscLoggingUtilTests(unittest.TestCase): util.logging.info("BridgeDB's email address: bridges(a)torproject.org") +class FileUtilityTests(unittest.TestCase): + """Unittests for `bridgedb.util.deleteFilesOlderThan`.""" + + def setUp(self): + self._directory = self.id() + self.newfile = os.sep.join([self._directory, "newfile"]) + self.oldfile = os.sep.join([self._directory, "oldfile"]) + self.testfiles = [self.newfile, self.oldfile] + os.mkdir(self._directory) + + now = time.time() + + for fn in self.testfiles: + with open(fn, "w") as fd: + fd.flush() + + # Change the mtime of the "oldfile" to be two days old: + os.utime(self.oldfile, (now, now - (48 * 60 * 60))) + + def test_deleteFilesOlderThan_deletes_old_files(self): + """The function should delete appropriate files.""" + deleted = util.deleteFilesOlderThan(self.testfiles ,24 * 60 * 60) + self.assertIn(self.oldfile, deleted) + + def test_deleteFilesOlderThan_keeps_new_files(self): + """The function should delete appropriate files.""" + deleted = util.deleteFilesOlderThan(self.testfiles ,24 * 60 * 60) + self.assertNotIn(self.newfile, deleted) + + class LevenshteinDistanceTests(unittest.TestCase): """Unittests for `bridgedb.util.levenshteinDistance."""

1 0

[bridgedb/master] Add 0.3.5 to the CHANGELOG.
by isis＠torproject.org 28 Jul '16

28 Jul '16

commit 0f7c26f062a1a5229d024422e16349b6940da256 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Dec 1 02:53:30 2015 +0000 Add 0.3.5 to the CHANGELOG. --- CHANGELOG | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index ba128f9..ce2e47f 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,4 +1,4 @@ -Changes in version 0.3.4 - 2015-11-30 +Changes in version 0.3.4 - 0.3.5 - 2015-11-30 * FIXES #14685 https://bugs.torproject.org/14685 This disables distribution of obfs2 bridges. This pluggable

1 0

[bridgedb/master] Only run txrecaptcha.submit() unittest on CI infrastructure.
by isis＠torproject.org 28 Jul '16

28 Jul '16

commit ad647e38847cf88f6b4f99951d2f79cbe4919c16 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon May 2 16:28:00 2016 +0000 Only run txrecaptcha.submit() unittest on CI infrastructure. --- test/test_txrecaptcha.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/test_txrecaptcha.py b/test/test_txrecaptcha.py index 4704d49..9e4f5f6 100644 --- a/test/test_txrecaptcha.py +++ b/test/test_txrecaptcha.py @@ -10,6 +10,7 @@ """Unittests for the bridgedb.txrecaptcha module.""" import logging +import os from twisted.internet import defer from twisted.internet import reactor @@ -19,6 +20,7 @@ from twisted.internet.error import ConnectionLost from twisted.internet.error import ConnectionRefusedError from twisted.test import proto_helpers from twisted.trial import unittest +from twisted.trial.unittest import SkipTest from twisted.python import failure from twisted.web.client import ResponseDone from twisted.web.http_headers import Headers @@ -208,6 +210,10 @@ class SubmitTests(unittest.TestCase): :func:`txrecaptcha.submit` should be a :class:`txcaptcha.RecaptchaResponse`. """ + if not os.environ.get("CI"): + raise SkipTest(("This test requires network so it is only run " + "on CI servers.")) + def checkResponse(response): """Check that the response is a :class:`txcaptcha.RecaptchaResponse`.

1 0

[bridgedb/develop] Merge branch 'fix/18949' into develop
by isis＠torproject.org 28 Jul '16

28 Jul '16

commit 6164c627a2653a81bcfa4197288b23a504f979e9 Merge: 0bb6014 2f3c79c Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon May 30 16:06:46 2016 +0000 Merge branch 'fix/18949' into develop .travis.yml | 12 +-- requirements.txt | 2 +- test/https_helpers.py | 228 +++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 233 insertions(+), 9 deletions(-)

1 0

[bridgedb/master] Add a CHANGELOG entry for #18237.
by isis＠torproject.org 28 Jul '16

28 Jul '16

commit 743917a843be970847fdd3933af026f7c3e27e8e Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon May 2 15:05:24 2016 +0000 Add a CHANGELOG entry for #18237. --- CHANGELOG | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index ce2e47f..036c057 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,11 @@ +Changes in version 0.3.6 - 2016-XXX + + * FIXES #18237 https://bugs.torproject.org/18237 + During descriptor parsing, BridgeDB saves copies of descriptor files + which couldn't be parsed, for later debugging purposes. To avoid + filing up the runtime directory with these files, we now delete files + older than 24 hours, every 24 hours. + Changes in version 0.3.4 - 0.3.5 - 2015-11-30 * FIXES #14685 https://bugs.torproject.org/14685

1 0

[bridgedb/master] Fix a typo in a unittest docstring.
by isis＠torproject.org 28 Jul '16

28 Jul '16

commit 0bb6014377e23c29bc774bc5f080c6534440c9bc Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon May 2 16:28:17 2016 +0000 Fix a typo in a unittest docstring. --- test/test_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_util.py b/test/test_util.py index da4ddf4..f7a2245 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -72,7 +72,7 @@ class MiscLoggingUtilTests(unittest.TestCase): class LevenshteinDistanceTests(unittest.TestCase): - """Unittests for `bridgedb.util.levenshteinDistance.""" + """Unittests for `bridgedb.util.levenshteinDistance`.""" def test_levenshteinDistance_blank_blank(self): """The Levenshtein Distance between '' and '' should be 0."""

1 0