March 2019 - tor-commits - lists.torproject.org

[sbws/master] gitignore: Ignore build/
by juga＠torproject.org 03 Mar '19

03 Mar '19

commit cbe68173dc4c0dfea0c764948bb2e7367aa9e576 Author: juga0 <juga(a)riseup.net> Date: Thu Feb 28 12:12:26 2019 +0000 gitignore: Ignore build/ --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ece16b2..da6b723 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ docs/build htmlcov .pytest_cache dist +build

1 0

[sbws/master] MANIFEST: Include .ini, exclude __pycache__ files
by juga＠torproject.org 03 Mar '19

03 Mar '19

commit 56e3e92a1d5df752aa9dcc8717d690077f578541 Author: juga0 <juga(a)riseup.net> Date: Thu Feb 28 12:11:33 2019 +0000 MANIFEST: Include .ini, exclude __pycache__ files --- MANIFEST.in | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 608581d..1418ce1 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,8 +1,7 @@ include *.md include *.rst -include sbws/*.ini -include examples * +include *.ini recursive-include docs * prune docs/build recursive-include tests * -prune **/__pycache__ +recursive-exclude **/__pycache__ *

1 0

[sbws/master] fixup! scanner: check that ResultDump queue is not full
by juga＠torproject.org 03 Mar '19

03 Mar '19

commit cf5b324d4557b5b7dc8891f64a7f1d12dff6154e Author: juga0 <juga(a)riseup.net> Date: Sat Feb 23 15:51:15 2019 +0000 fixup! scanner: check that ResultDump queue is not full --- sbws/core/scanner.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/sbws/core/scanner.py b/sbws/core/scanner.py index 4c85ee4..7ab1e6e 100644 --- a/sbws/core/scanner.py +++ b/sbws/core/scanner.py @@ -1,4 +1,5 @@ ''' Measure the relays. ''' +import queue from ..lib.circuitbuilder import GapsCircuitBuilder as CB from ..lib.resultdump import ResultDump @@ -317,16 +318,19 @@ def result_putter(result_dump): result -- and return that function so it can be used by someone else ''' def closure(measurement_result): - # in case the queue is full, wait until is not. # Since result_dump thread is calling queue.get() every second, # the queue should be full for only 1 second. - while result_dump.queue.full(): - log.info('The results queue is full, after 1 second it should ' - 'not be full.') - time.sleep(1) - # Non blocking, wait a maximum of 1 second if the queue is full. - # Because of the timeout, the previous while should not be needed. - result_dump.queue.put(measurement_result, timeout=1) + # This call blocks at maximum timeout seconds. + try: + result_dump.queue.put(measurement_result, timeout=3) + except queue.Full: + # The result would be lost, the scanner will continue working. + log.warning( + "The queue with measurements is full, when adding %s.\n" + "It is possible that the thread that get them to " + "write them to the disk (ResultDump.enter) is stalled.", + measurement_result + ) return closure

1 0

[sbws/master] resultdump: Document how the results are gotten
by juga＠torproject.org 03 Mar '19

03 Mar '19

commit 43add2481a4084412798f6a62db2c5c9e9511639 Author: juga0 <juga(a)riseup.net> Date: Sat Feb 16 14:47:12 2019 +0000 resultdump: Document how the results are gotten from the thread. --- sbws/lib/resultdump.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/sbws/lib/resultdump.py b/sbws/lib/resultdump.py index 52a0b7c..384bd42 100644 --- a/sbws/lib/resultdump.py +++ b/sbws/lib/resultdump.py @@ -582,7 +582,22 @@ class ResultDump: log.info(msg) def enter(self): - ''' Main loop for the ResultDump thread ''' + """Main loop for the ResultDump thread. + + When there are results in the queue, queue.get will get them until + there are not anymore or timeout happen. + + For every result it gets, it process it and store in the filesystem, + which takes ~1 millisecond and will not trigger the timeout. + It can then store in the filesystem ~1000 results per second. + + I does not accept any other data type than Results or list of Results, + therefore is not possible to put big data types in the queue. + + If there are not any results in the queue, it waits 1 second and checks + again. + + """ with self.data_lock: self.data = load_recent_results_in_datadir( self.fresh_days, self.datadir)

1 0

[sbws/master] scanner: check that ResultDump queue is not full
by juga＠torproject.org 03 Mar '19

03 Mar '19

commit ada70d4f85ac7ce6f9847eeae832ec7620106470 Author: juga0 <juga(a)riseup.net> Date: Tue Jan 8 16:27:44 2019 +0000 scanner: check that ResultDump queue is not full and put the result in the queue with a timeout. Fixes bug #28866. Bugfix v0.1.0. --- sbws/core/scanner.py | 12 +++++++++++- tests/unit/conftest.py | 20 ++++++++++++++++++++ tests/unit/core/test_scanner.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 1 deletion(-) diff --git a/sbws/core/scanner.py b/sbws/core/scanner.py index 7246f12..4c85ee4 100644 --- a/sbws/core/scanner.py +++ b/sbws/core/scanner.py @@ -315,8 +315,18 @@ def _next_expected_amount(expected_amount, result_time, download_times, def result_putter(result_dump): ''' Create a function that takes a single argument -- the measurement result -- and return that function so it can be used by someone else ''' + def closure(measurement_result): - return result_dump.queue.put(measurement_result) + # in case the queue is full, wait until is not. + # Since result_dump thread is calling queue.get() every second, + # the queue should be full for only 1 second. + while result_dump.queue.full(): + log.info('The results queue is full, after 1 second it should ' + 'not be full.') + time.sleep(1) + # Non blocking, wait a maximum of 1 second if the queue is full. + # Because of the timeout, the previous while should not be needed. + result_dump.queue.put(measurement_result, timeout=1) return closure diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 4dd442b..ada1b9b 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -267,3 +267,23 @@ def sbwshome_success_result_two_relays(sbwshome_only_datadir, conf): write_result_to_datadir(RESULT_SUCCESS2, dd) write_result_to_datadir(RESULT_SUCCESS2, dd) return sbwshome_only_datadir + + +(a)pytest.fixture(scope='function') +def end_event(): + import threading + return threading.Event() + + +(a)pytest.fixture(scope='function') +def rd(args, conf, end_event): + from sbws.lib.resultdump import ResultDump + # in Travis the next line gives the error: + # TypeError: __init__() takes 3 positional arguments but 4 were given + # No idea why. + # Returning None to disable the test in case ResultDump can not be + # initialized. + try: + return ResultDump(args, conf, end_event) + except TypeError: + return None diff --git a/tests/unit/core/test_scanner.py b/tests/unit/core/test_scanner.py new file mode 100644 index 0000000..3f84472 --- /dev/null +++ b/tests/unit/core/test_scanner.py @@ -0,0 +1,28 @@ +"""Unit tests for scanner.py.""" +import pytest + +from sbws.core.scanner import result_putter + + +def test_result_putter(sbwshome_only_datadir, result_success, rd, end_event): + if rd is None: + pytest.skip("ResultDump is None") + # Put one item in the queue + callback = result_putter(rd) + callback(result_success) + assert rd.queue.qsize() == 1 + + # Make queue maxsize 1, so that it'll be full after the first callback. + # The second callback will wait 1 second, then the queue will be empty + # again. + rd.queue.maxsize = 1 + callback(result_success) + # after putting 1 result, the queue will be full + assert rd.queue.qsize() == 1 + assert rd.queue.full() + # it's still possible to put another results, because the callback will + # wait 1 second and the queue will be empty again. + callback(result_success) + assert rd.queue.qsize() == 1 + assert rd.queue.full() + end_event.set()

1 0

[sbws/master] Merge branch 'bug28866'
by juga＠torproject.org 03 Mar '19

03 Mar '19

commit 901cf9216c8df054d05fd51315b51ec14128ac49 Merge: cbe6817 cf5b324 Author: juga0 <juga(a)riseup.net> Date: Sun Mar 3 12:44:14 2019 +0000 Merge branch 'bug28866' sbws/core/scanner.py | 16 +++++++++++++++- sbws/lib/resultdump.py | 17 ++++++++++++++++- tests/unit/conftest.py | 20 ++++++++++++++++++++ tests/unit/core/test_scanner.py | 28 ++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 2 deletions(-) diff --cc sbws/core/scanner.py index 3fbd378,7ab1e6e..4ad2cb5 --- a/sbws/core/scanner.py +++ b/sbws/core/scanner.py @@@ -1,11 -1,6 +1,12 @@@ ''' Measure the relays. ''' + import queue +import signal +import sys +import threading +import traceback +import uuid + from ..lib.circuitbuilder import GapsCircuitBuilder as CB from ..lib.resultdump import ResultDump from ..lib.resultdump import ResultSuccess, ResultErrorCircuit

1 0

[translation/support-portal] Update translations for support-portal
by translation＠torproject.org 03 Mar '19

03 Mar '19

commit f12cdb251fad0d1b8825cec8969c221bf28851a4 Author: Translation commit bot <translation(a)torproject.org> Date: Sun Mar 3 09:20:04 2019 +0000 Update translations for support-portal --- contents+fa.po | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/contents+fa.po b/contents+fa.po index cdb688a0d..53a7a453a 100644 --- a/contents+fa.po +++ b/contents+fa.po @@ -5497,6 +5497,13 @@ msgid "" "href=\"https://www.facebook.com/notes/protect-the-graph/making-connections-" "to-facebook-more-secure/1526085754298237/\">Facebook</a>." msgstr "" +"به سرویس‌های پیازی برای چت بدون فراداده (مِتادیتا) یا به اشتراک گذاشتن فایل،" +" ارتباط بین خبرنگاران یا منابع آنها مانند <a " +"href=\"https://securedrop.org/\">SecureDrop</a> یا <a " +"href=\"https://onionshare.org/\">OnionShare</a>، به روزرسانی امن‌تر " +"نرم افزار یا راههای ایمن‌تر دسترسی به سایتهای محبوب مانند <a " +"href=\"https://www.facebook.com/notes/protect-the-graph/making-connections-" +"to-facebook-more-secure/1526085754298237/\">فیسبوک</a> تکیه می‌شود." #: https//support.torproject.org/onionservices/what-is-a-dot-onion/ #: (content/onionservices/onionservices-2/contents+en.lrquestion.description)

1 0

[translation/support-portal] Update translations for support-portal
by translation＠torproject.org 03 Mar '19

03 Mar '19

commit 0adb31be72b6405c0868d20b43215097213ecac6 Author: Translation commit bot <translation(a)torproject.org> Date: Sun Mar 3 08:50:02 2019 +0000 Update translations for support-portal --- contents+fa.po | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/contents+fa.po b/contents+fa.po index 48c8c9bbb..cdb688a0d 100644 --- a/contents+fa.po +++ b/contents+fa.po @@ -2863,6 +2863,9 @@ msgid "" " on your Desktop, though be aware that you may have accidentally unticked " "the option to create a shortcut." msgstr "" +"تنظیمات پیشفرض نصاب ویندوز یک میانبر برای شما بر روی میز کاری ایجاد خواهد " +"کرد، اگرچه ممکن است که شما به شکل اتفاقی گزینه ایجاد میانبر را غیر فعال کرده" +" باشید." #: https//support.torproject.org/tbb/cannot-find-tor-on-windows/ #: (content/tbb/tbb-11/contents+en.lrquestion.description) @@ -3124,6 +3127,7 @@ msgid "" "Tor Browser won't connect, but it doesn’t seem to be an issue with " "censorship." msgstr "" +"مرورگر تور متصل نمی‌شود، ولی به نظر می‌رسد که مشکل سانسور وجود نداشته باشد." #: https//support.torproject.org/tbb/tor-browser-will-not-connect-no-censorship/ #: (content/tbb/tbb-20/contents+en.lrquestion.description) @@ -3307,6 +3311,8 @@ msgid "" "The Torbutton icon (the little onion in the top left corner of the browser) " "will display a yellow triangle." msgstr "" +"آیکون Torbutton (پیاز کوچک بالا سمت چپ /راست مرورگر) یک مثلث زرد نشان خواهد " +"داد." #: https//support.torproject.org/tbb/updating-tor-browser/ #: (content/tbb/tbb-27/contents+en.lrquestion.description) @@ -3335,7 +3341,7 @@ msgstr "مرورگر تور به روز رسانی ها را نصب خواهد #: https//support.torproject.org/tbb/updating-tor-browser/ #: (content/tbb/tbb-27/contents+en.lrquestion.description) msgid "<img class=\"\" src=\"/static/images/image3.png\" alt=\"New release alert\">" -msgstr "" +msgstr "<img class=\"\" src=\"/static/images/image3.png\" alt=\"نسخه جدید موجود است\"/>" #: https//support.torproject.org/tbb/uninstall-tor-browser/ #: (content/tbb/tbb-28/contents+en.lrquestion.title) @@ -4294,7 +4300,7 @@ msgstr "" #: https//support.torproject.org/gettor/to-use-gettor-via-twitter/ #: (content/gettor/gettor-3/contents+en.lrquestion.title) msgid "To use GetTor via Twitter." -msgstr "" +msgstr "برای استفاده از GetTor از طریق توییتر." #: https//support.torproject.org/gettor/to-use-gettor-via-twitter/ #: (content/gettor/gettor-3/contents+en.lrquestion.description) @@ -4303,6 +4309,9 @@ msgid "" "href=\"https://twitter.com/get_tor\">@get_tor</a> with one of the " "following codes in it (you don't need to follow the account):" msgstr "" +"برای دریافت لینک برای دانلود کردن مرورگر تور، یک پیغام مستقیم به <a " +"href=\"https://twitter.com/get_tor\">@get_tor</a> با یکی از کدهای زیر" +" بفرستید (شما نیازی به دنبال کردن این حساب کاربری ندارید):" #: https//support.torproject.org/gettor/to-use-gettor-via-twitter/ #: (content/gettor/gettor-3/contents+en.lrquestion.description) @@ -4900,6 +4909,9 @@ msgid "" "href=\"https://trac.torproject.org/projects/tor/wiki/TorRelayGuide\">Tor " "Relay Guide</a>." msgstr "" +"برای عمیق‌ترین منبع برای اجرای یک پل، <a " +"href=\"https://trac.torproject.org/projects/tor/wiki/TorRelayGuide\">راهنمای" +" پل تور</a> را ببینید." #: https//support.torproject.org/operators/how-do-i-run-a-middle-or-guard-relay/ #: (content/operators/operators-1/contents+en.lrquestion.description) @@ -5206,6 +5218,7 @@ msgstr "" #: (content/operators/operators-3/contents+en.lrquestion.title) msgid "How do I run a middle or guard relay on FreeBSD or HardenedBSD?" msgstr "" +"چگونه یک پل میانی یا پل ورودی روی FreeBSD یا HardenedBSD راه اندازی کنم؟" #: https//support.torproject.org/operators/how-do-i-run-a-middle-or-guard-on-freebsd/ #: (content/operators/operators-3/contents+en.lrquestion.description) @@ -5503,7 +5516,7 @@ msgstr "" #: https//support.torproject.org/onionservices/what-is-a-dot-onion/ #: (content/onionservices/onionservices-2/contents+en.lrquestion.description) msgid "<img class=\"\" src=\"/static/images/onion-website.png\" alt=\"Onion icon\">" -msgstr "" +msgstr "<img class=\"\" src=\"/static/images/onion-website.png\" alt=\"آیکون پیاز\"/>" #: https//support.torproject.org/onionservices/what-is-a-dot-onion/ #: (content/onionservices/onionservices-2/contents+en.lrquestion.description)

1 0

[translation/support-portal] Update translations for support-portal
by translation＠torproject.org 03 Mar '19

03 Mar '19

commit f1fb42a48f768c1efcb4fdfcf5c3dd1fe8c2dd0f Author: Translation commit bot <translation(a)torproject.org> Date: Sun Mar 3 08:20:09 2019 +0000 Update translations for support-portal --- contents+fa.po | 2 +- contents+tr.po | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/contents+fa.po b/contents+fa.po index 391495ef3..48c8c9bbb 100644 --- a/contents+fa.po +++ b/contents+fa.po @@ -4328,7 +4328,7 @@ msgstr "" #: https//support.torproject.org/gettor/to-use-gettor-via-xmpp/ #: (content/gettor/gettor-4/contents+en.lrquestion.title) msgid "To use GetTor via XMPP (Jitsi, CoyIM)." -msgstr "" +msgstr "برای استفاده GetTor از طریق XMPP (مانند Jitsi یا CoyIM)" #: https//support.torproject.org/gettor/to-use-gettor-via-xmpp/ #: (content/gettor/gettor-4/contents+en.lrquestion.description) diff --git a/contents+tr.po b/contents+tr.po index 7b88b2017..b055c7747 100644 --- a/contents+tr.po +++ b/contents+tr.po @@ -4,6 +4,7 @@ # Emma Peel, 2019 # Alperen Kitapçı <alperenmirac(a)gmail.com>, 2019 # ilkeryus <ilkeryus(a)gmail.com>, 2019 +# cenk y. <goncagul(a)national.shitposting.agency>, 2019 # msgid "" msgstr "" @@ -11,7 +12,7 @@ msgstr "" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2019-01-28 19:24+CET\n" "PO-Revision-Date: 2018-10-02 22:41+0000\n" -"Last-Translator: ilkeryus <ilkeryus(a)gmail.com>, 2019\n" +"Last-Translator: cenk y. <goncagul(a)national.shitposting.agency>, 2019\n" "Language-Team: Turkish (https://www.transifex.com/otf/teams/1519/tr/)\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -93,7 +94,7 @@ msgstr "" #: https//support.torproject.org/misc/tor-glossary/ #: (content/misc/glossary/contents+en.lrquestion.description) msgid "### antivirus software" -msgstr "" +msgstr "### antivirus yazılımı" #: https//support.torproject.org/glossary/ #: (content/glossary/contents+en.lrquestion.description) @@ -103,6 +104,8 @@ msgid "" "An antivirus software is used to prevent, detect and remove malicious " "software." msgstr "" +"Bir antivirus yazılımı zararlı yazılımı önlemek, bulmak ve kaldırmak için " +"kullanılır." #: https//support.torproject.org/glossary/ #: (content/glossary/contents+en.lrquestion.description)

1 0

[stem/master] Bandwidth file version must be second header
by atagar＠torproject.org 02 Mar '19

02 Mar '19

commit 9c4d22713705ffc46adfb413e8a4fd10d3366883 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Mar 2 14:44:47 2019 -0800 Bandwidth file version must be second header Good point from juga that the bandwidth file spec requires the 'version' header to be in the second position (unless the document is version 1.0.0, in which case the version attribute is not present at all). https://trac.torproject.org/projects/tor/ticket/29539 Ensuring the descriptors we create conform with this, and if the user parses a bandwidth file with 'validate = True' then check this. --- stem/descriptor/bandwidth_file.py | 16 ++++++++++++++++ test/unit/descriptor/bandwidth_file.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/stem/descriptor/bandwidth_file.py b/stem/descriptor/bandwidth_file.py index f73dc007..75fe2639 100644 --- a/stem/descriptor/bandwidth_file.py +++ b/stem/descriptor/bandwidth_file.py @@ -106,6 +106,9 @@ def _parse_header(descriptor, entries): content.readline() # skip the first line, which should be the timestamp + index = 1 + version_index = None + while True: line = content.readline().strip() @@ -119,14 +122,22 @@ def _parse_header(descriptor, entries): if b'=' in line: key, value = stem.util.str_tools._to_unicode(line).split('=', 1) header[key] = value + + if key == 'version': + version_index = index else: raise ValueError("Header expected to be key=value pairs, but had '%s'" % line) + index += 1 + descriptor.header = header for attr, (keyword, cls) in HEADER_ATTR.items(): setattr(descriptor, attr, cls(header.get(keyword, HEADER_DEFAULT.get(attr)))) + if version_index is not None and version_index != 1: + raise ValueError("The 'version' header must be in the second position") + def _parse_timestamp(descriptor, entries): first_line = io.BytesIO(descriptor.get_bytes()).readline().strip() @@ -245,6 +256,11 @@ class BandwidthFile(Descriptor): if version == '1.0.0' and header: raise ValueError('Headers require BandwidthFile version 1.1 or later') elif version != '1.0.0': + # ensure 'version' is the second header + + if 'version' not in exclude: + lines.append(stem.util.str_tools._to_bytes('version=%s' % header.pop('version'))) + for k, v in header.items(): lines.append(stem.util.str_tools._to_bytes('%s=%s' % (k, v))) diff --git a/test/unit/descriptor/bandwidth_file.py b/test/unit/descriptor/bandwidth_file.py index e8684cf4..7b1cd198 100644 --- a/test/unit/descriptor/bandwidth_file.py +++ b/test/unit/descriptor/bandwidth_file.py @@ -59,6 +59,20 @@ new_header=neat stuff ===== """.strip() +WRONG_VERSION_POSITION = """ +1410723598 +file_created=2019-01-14T05:35:06 +version=1.1.0 +===== +""".strip() + +RIGHT_VERSION_POSITION = """ +1410723598 +version=1.1.0 +file_created=2019-01-14T05:35:06 +===== +""".strip() + class TestBandwidthFile(unittest.TestCase): def test_from_str(self): @@ -180,6 +194,26 @@ class TestBandwidthFile(unittest.TestCase): self.assertRaisesWith(ValueError, 'Headers require BandwidthFile version 1.1 or later', BandwidthFile.create, {'new_header': 'neat stuff'}) + def test_version_position(self): + """ + Our 'version' header must be in the second position if validated, but + otherwise doesn't matter. (:trac:`29539`) + """ + + desc = BandwidthFile.from_str(WRONG_VERSION_POSITION) + self.assertEqual('1.1.0', desc.version) + + self.assertRaisesWith(ValueError, "The 'version' header must be in the second position", BandwidthFile.from_str, WRONG_VERSION_POSITION, validate = True) + + content = BandwidthFile.content(OrderedDict([ + ('timestamp', '1410723598'), + ('file_created', '2019-01-14T05:35:06'), + ('version', '1.1.0'), + ('content', []), + ])) + + self.assertEqual(RIGHT_VERSION_POSITION, content) + def test_header_alternate_div(self): """ To support backward compatability four character dividers are allowed.

1 0