May 2015 - tor-commits - lists.torproject.org

[ooni-probe/master] Use the read the docs theme for sphinx
by art＠torproject.org 02 May '15

02 May '15

commit 63628478205ab861cdcadb2d38a932aa0b1c7973 Author: Arturo Filastò <art(a)fuffa.org> Date: Sat May 2 13:06:16 2015 +0200 Use the read the docs theme for sphinx --- docs/source/conf.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 778192a..f12deba 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -11,6 +11,7 @@ # All configuration values have a default; values that are commented out # serve to show the default. +import sphinx_rtd_theme import sys, os # If extensions (or modules to document with autodoc) are in another directory, @@ -43,17 +44,17 @@ source_suffix = '.rst' master_doc = 'index' # General information about the project. -project = u'OONI' -copyright = u'2012, The Tor Project' +project = u'OONI: Open Observatory of Network Interference' +copyright = u'2015, The Tor Project' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '0.1' +version = '1.3.1' # The full version, including alpha/beta/rc tags. -release = '0.1-alpha' +release = '1.3.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -94,7 +95,7 @@ pygments_style = 'sphinx' # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'default' +html_theme = 'sphinx_rtd_theme' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -102,7 +103,7 @@ html_theme = 'default' #html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # The name for this set of Sphinx documents. If None, it defaults to # "<project> v<release> documentation". @@ -123,7 +124,7 @@ html_theme = 'default' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +# html_static_path = ['_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format.

1 0

[ooni-probe/master] flesh out tcp scan tutorial
by art＠torproject.org 02 May '15

02 May '15

commit acbf9de939b9a477fb7faea39e9206accc13abea Author: poly <poly(a)darkdepths.net> Date: Thu Apr 30 20:05:02 2015 +0400 flesh out tcp scan tutorial --- docs/source/tutorial.rst | 119 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 117 insertions(+), 2 deletions(-) diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index 15afad7..4c0f4ce 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -5,7 +5,122 @@ Writing a TCP port scanner -------------------------- Our goal is to write an OONI test that takes as input a list of ip:port -combinations and attempts to detect which ones are reachable and open and -which ones are not. +combinations from a file and attempts to detect which ones are reachable +and open and which ones are not. +Following this tutorial requires basic knowledge of event-driven programming +(specifically 'Twisted') +Creating test +-------------- + +We base our test on the TCP subclass :class:`ooni.templates.tcpt.TCPTest`, +since it the most suitable for our requirements. We then create the file +'tcp_port_scanner.py' inside 'ooni/nettests/experimental'. + +All tests start in the 'experimental' directory, until they are tested +and may then be moved to the stable test folders. + +We start by adding our base class into 'tcp_port_scanner.py':: + + class TCPPortScan(tcpt.TCPTest): + +Setup +------ + +OONI requires some basic information from all tests before running them. +To provide this information we add the following to our test:: + + class TCPPortScan(tcpt.TCPTest): + name = 'TCP_port_scan_test' + author = 'OONI Contributor <ooni(a)torproject.org>' + version = 0.0.1 + + # Some tests may require root access + requiresRoot = False + requiresTor = False + +Inputs +------ + +We need to provide a file to read the IP addresses from. We can do +this by adding:: + + class TCPPortScan(tcpt.TCPTest): + ... + inputFile = ['file', 'f', None, "Short file description"] + requiredOptions = ['file'] + ... + +When the test is run, the file will be opened by +:class:`ooni.nettest.NetTestCase.inputProcessor`. By default the +'inputProcessor' will read line-by-line and feed them to the test. +However, in this test case we may want to skip over the comment lines +that start with a '#'. We override 'inputProcessor':: + + class TCPPortScan(tcpt.TCPTest): + ... + def inputProcessor(self, filename): + fp = open(filename) + for x in fp.xreadlines(): + if x.startswith("#"): + continue + yield x.strip() + fp.close() + ... + +Once the test is run, the input may be accessed via 'self.input'. + +Writing Tests +------------- + +Any functioned prefixed with 'test' will be run by OONI for each +input. We implement the TCP connection test by adding:: + + class TCPPortScan(tcpt.TCPTest): + ... + def test_tcp_port(self): + def got_response(response): + self.report['connection_success'] = True + print "Connection Successful" + + def connection_failed(failure): + self.report['connection_success'] = False + print "Connection Failed" + + self.address = self.input.split(":")[0] + self.port = self.input.split(":")[1] + payload = "" + + d = self.sendPayload(payload) + d.addErrback(connection_failed) + d.addCallback(got_response) + + return d + +Note that all test functions **must** return a Deferred object. +As shown above, information can add to the OONI report using:: + + self.report['something'] = value + +Runninng Tests +-------------- + +If OONI is installed on your computer, you may run the above test +by:: + + ooniprobe nettests/experimental/tcp_port_scanner.py -f /path/to/ip_list.txt + +By default, this will cause OONI to connect to tor and upload the results +of the test. To save time during development, consider using the '-n' flag, +temporarily disabling upload. Additionally, the '-v' is also useful, +as OONI by default doesn't print the line if a runtime error occurs:: + + ooniprobe -n -v nettests/experimental/tcp_port_scanner.py -f /path/to/ip_list.txt + +Reports +------- + +Once 'ooniprobe' is invoked, a report will be created in the current working +directory. This report is structed in the YAMLOO format - a format based on +YAML.

1 0

[ooni-probe/master] Merge pull request #391 from 0xPoly/docs
by art＠torproject.org 02 May '15

02 May '15

commit 4bad32a545d798997ae54d70c37eb084d7a7fc38 Merge: 80463cd 93ec144 Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Apr 30 17:52:20 2015 +0200 Merge pull request #391 from 0xPoly/docs minor tweaks to writing tests docs docs/source/writing_tests.rst | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-)

1 0

[ooni-probe/master] tweaked writing tests docs
by art＠torproject.org 02 May '15

02 May '15

commit 93ec144df983b76fae68cb0343f36cafaf567e9e Author: poly <poly(a)darkdepths.net> Date: Thu Apr 30 18:29:45 2015 +0400 tweaked writing tests docs --- docs/source/writing_tests.rst | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/source/writing_tests.rst b/docs/source/writing_tests.rst index 0932f3a..2132010 100644 --- a/docs/source/writing_tests.rst +++ b/docs/source/writing_tests.rst @@ -18,8 +18,8 @@ multiple Test Methods. If the test you plan to write is not listed on the `Tor OONI trac page <https://trac.torproject.org/projects/tor/wiki/doc/OONI/Tests>`_, you should -add it to the list and then add a description about it following the `Test -Template <https://gitweb.torproject.org/ooni-probe.git/blob/HEAD:/docs/source/tests/t…>`_ +add it to the list and then add a description about it following the `test +template <https://gitweb.torproject.org/ooni-probe.git/plain/docs/source/tests/templa…>`_. Tests are driven by inputs. For every input a new test instance is created, internally the _setUp method is called that is defined inside of test @@ -33,7 +33,7 @@ Inputs ------ Inputs are what is given as input to every iteration of the Test Case. -Iflyou have 100 inputs, then every test case will be run 100 times. +If you have 100 inputs, then every test case will be run 100 times. To configure a static set of inputs you should define the :class:`ooni.nettest.NetTestCase` attribute ``inputs``. The test will be @@ -57,6 +57,13 @@ this:: yield x.strip() fp.close() +For example, if you wanted to modify inputProcessor to read enteries from a CSV file, you could use:: + + def inputProcessor(self, filename): + with open(filename) as csvFile: + reader = DictReader(csvFile) + for entry in reader: + yield entry Setup and command line passing ------------------------------

1 0

[ooni-probe/master] Merge pull request #390 from anadahz/fix/requirements
by art＠torproject.org 02 May '15

02 May '15

commit 80463cd172c90bf0cf7d1455aecb9bbe5d9fd113 Merge: 4f8f1b1 ab557c1 Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Apr 30 14:40:47 2015 +0200 Merge pull request #390 from anadahz/fix/requirements Add supported pip Twisted versions explicitly requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

1 0

[meek/master] Don't close subprocess stdin in terminateprocess-buffer.
by dcf＠torproject.org 01 May '15

01 May '15

commit 098ac93e1158ce87de209ba317b171a78a248d61 Author: David Fifield <david(a)bamsoftware.com> Date: Thu Apr 30 20:54:31 2015 -0700 Don't close subprocess stdin in terminateprocess-buffer. It started being closed in 1fa2da7c6c03500ee899061a773fb7cfb0c1bf5f, which added compliance with the TOR_PT_EXIT_ON_STDIN_CLOSE environment variable. Closing stdin of the subprocess doesn't make sense, because the purpose of terminateprocess-buffer is to keep the chile stdin open after its own stdin has closed. This fixes a regression in Tor Browser 4.5 on Windows 7: https://trac.torproject.org/projects/tor/ticket/15872#comment:5 However, the same code, even when it closes stdin, works on Windows 8, and I don't know why. --- terminateprocess-buffer/terminateprocess-buffer.go | 1 - 1 file changed, 1 deletion(-) diff --git a/terminateprocess-buffer/terminateprocess-buffer.go b/terminateprocess-buffer/terminateprocess-buffer.go index ef6f353..6f77e98 100644 --- a/terminateprocess-buffer/terminateprocess-buffer.go +++ b/terminateprocess-buffer/terminateprocess-buffer.go @@ -43,7 +43,6 @@ func main() { log.Fatal(err) } io.Copy(stdin, os.Stdin) - stdin.Close() err = cmd.Wait() if err != nil { log.Fatal(err)

1 0

[webwml/master] Removing extra space before '##translation metadata'
by sebastian＠torproject.org 01 May '15

01 May '15

commit b53ed7a9b755367874f51fde2e50fe47ee2d9e66 Author: Nima Fatemi <mrphs(a)riseup.net> Date: Fri May 1 15:42:58 2015 -0400 Removing extra space before '##translation metadata' Extra space before '##translation metadata' has made it render in html. Introduced in d3b0ad1b62ca511213af17f44835976c8872216a --- about/en/corepeople.wml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/about/en/corepeople.wml b/about/en/corepeople.wml index 6409372..e7c9219 100644 --- a/about/en/corepeople.wml +++ b/about/en/corepeople.wml @@ -1,4 +1,4 @@ -## translation metadata +## translation metadata # Revision: $Revision$ # Translation-Priority: 3-low #include "head.wmi" TITLE="Tor Project: Core People" CHARSET="UTF-8"

1 0

[translation/abouttor-homepage] Update translations for abouttor-homepage
by translation＠torproject.org 01 May '15

01 May '15

commit ab8ddebafc694bef4c50107d967480ef285982aa Author: Translation commit bot <translation(a)torproject.org> Date: Fri May 1 17:15:38 2015 +0000 Update translations for abouttor-homepage --- sl_SI/aboutTor.dtd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sl_SI/aboutTor.dtd b/sl_SI/aboutTor.dtd index fac275b..6f3dd27 100644 --- a/sl_SI/aboutTor.dtd +++ b/sl_SI/aboutTor.dtd @@ -6,8 +6,8 @@ <!ENTITY aboutTor.title "O Toru"> -<!ENTITY aboutTor.outOfDateTorOn.label "VENDAR, pa je ta brskalnik zastarel."> -<!ENTITY aboutTor.outOfDateTorOff.label "TUDI, ta brskalnik je zastarel."> +<!ENTITY aboutTor.outOfDateTorOn.label "Vendar, je ta brskalnik zastarel."> +<!ENTITY aboutTor.outOfDateTorOff.label "Tudi, ta brskalnik je zastarel."> <!ENTITY aboutTor.outOfDate2.label "Klikni na čebulo in nato izberi Preglej za Tor posodobitve."> <!ENTITY aboutTor.check.label "Preizkusi Tor omrežne nastavitve">

1 0

[translation/abouttor-homepage] Update translations for abouttor-homepage
by translation＠torproject.org 01 May '15

01 May '15

commit 1d578b9fc2f71ca95505db92702945df2bb947e8 Author: Translation commit bot <translation(a)torproject.org> Date: Fri May 1 16:45:37 2015 +0000 Update translations for abouttor-homepage --- sl_SI/aboutTor.dtd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sl_SI/aboutTor.dtd b/sl_SI/aboutTor.dtd index 29d5ea7..fac275b 100644 --- a/sl_SI/aboutTor.dtd +++ b/sl_SI/aboutTor.dtd @@ -4,11 +4,11 @@ - vim: set sw=2 sts=2 ts=8 et syntax=xml: --> -<!ENTITY aboutTor.title "O Tor"> +<!ENTITY aboutTor.title "O Toru"> <!ENTITY aboutTor.outOfDateTorOn.label "VENDAR, pa je ta brskalnik zastarel."> <!ENTITY aboutTor.outOfDateTorOff.label "TUDI, ta brskalnik je zastarel."> -<!ENTITY aboutTor.outOfDate2.label "Click on the onion and then choose Check for Tor Browser Update."> +<!ENTITY aboutTor.outOfDate2.label "Klikni na čebulo in nato izberi Preglej za Tor posodobitve."> <!ENTITY aboutTor.check.label "Preizkusi Tor omrežne nastavitve">

1 0

[bridgedb/master] Rewrite `python setyp.py test` code to run Trial tests only.
by isis＠torproject.org 01 May '15

01 May '15

commit 382a4f5e54a26dc7a2179d17cb4129f68e3e650e Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Apr 16 04:23:24 2015 +0000 Rewrite `python setyp.py test` code to run Trial tests only. --- lib/bridgedb/parse/options.py | 27 ---------- lib/bridgedb/runner.py | 47 ---------------- setup.py | 119 +++++++++++++++++++++++++++++++++++++---- 3 files changed, 109 insertions(+), 84 deletions(-) diff --git a/lib/bridgedb/parse/options.py b/lib/bridgedb/parse/options.py index 2ed93d0..3979615 100644 --- a/lib/bridgedb/parse/options.py +++ b/lib/bridgedb/parse/options.py @@ -251,32 +251,6 @@ class BaseOptions(usage.Options): print(" self['rundir']=%s" % self['rundir']) -class TestOptions(BaseOptions): - """Suboptions for running twisted.trial and unittest based tests.""" - - longdesc = textwrap.dedent("""BridgeDB testing commands. - See the `bridgedb mock` command for generating testing environments.""") - - optFlags = [['coverage', 'c', 'Generate coverage statistics']] - optParameters = [ - ['file', 'f', None, 'Run tests in specific file(s) (trial only)'], - ['unittests', 'u', False, 'Run unittests in bridgedb.Tests'], - ['trial', 't', True, 'Run twisted.trial tests in bridgedb.test']] - - completionData = usage.Completions( - mutuallyExclusive=[('unittests', 'coverage'), - ('unittests', 'file')], - optActions={'file': usage.CompleteFiles('lib/bridgedb/test/test_*.py', - repeat=True, - descr="test filename")}, - extraActions=[ - usage.Completer(descr="extra arguments to pass to trial")]) - - def parseArgs(self, *args): - """Parse any additional arguments after the options and flags.""" - self['test_args'] = args - - class MockOptions(BaseOptions): """Suboptions for creating necessary conditions for testing purposes.""" @@ -315,7 +289,6 @@ class MainOptions(BaseOptions): ['dump-bridges', 'd', 'Dump bridges by hashring assignment into files'], ['reload', 'R', 'Reload bridge descriptors into running servers']] subCommands = [ - ['test', None, TestOptions, "Run twisted.trial tests or unittests"], ['mock', None, MockOptions, "Generate a testing environment"], ['SIGHUP', None, SIGHUPOptions, "Reload bridge descriptors into running servers"], diff --git a/lib/bridgedb/runner.py b/lib/bridgedb/runner.py index 7be7700..6ac069f 100644 --- a/lib/bridgedb/runner.py +++ b/lib/bridgedb/runner.py @@ -94,53 +94,6 @@ def generateDescriptors(count=None, rundir=None): del subprocess return statuscode -def runTrial(options): - """Run Twisted trial based unittests, optionally with coverage. - - :type options: :class:`~bridgedb.opt.TestOptions` - :param options: Parsed options for controlling the twisted.trial test - run. All unrecognised arguments after the known options will be passed - along to trial. - """ - from twisted.scripts import trial - - # Insert 'trial' as the first system cmdline argument: - sys.argv = ['trial'] - - if options['coverage']: - try: - from coverage import coverage - except ImportError as ie: - print(ie.message) - else: - cov = coverage() - cov.start() - sys.argv.append('--coverage') - sys.argv.append('--reporter=bwverbose') - - # Pass all arguments along to its options parser: - if 'test_args' in options: - for arg in options['test_args']: - sys.argv.append(arg) - # Tell trial to test the bridgedb package: - sys.argv.append('bridgedb.test') - trial.run() - - if options['coverage']: - cov.stop() - cov.html_report('_trial_temp/coverage/') - -def runTests(options): - """Run unittest based tests. - - :type options: :class:`~bridgedb.opt.TestOptions` - :param options: Parsed options for controlling the twisted.trial test - run. All unrecognised arguments after the known options will be passed - along to trial. - """ - testModule = __import__('bridgedb.Tests', globals(), '', []) - testModule.Tests.main() - def doDumpBridges(config): """Dump bridges by assignment to a file. diff --git a/setup.py b/setup.py index 9402071..032aa10 100644 --- a/setup.py +++ b/setup.py @@ -74,6 +74,7 @@ install_i18n = os.path.join('bridgedb', 'i18n') # Directory to install docs, license, and other text resources into: install_docs = os.path.join('share', 'doc', 'bridgedb') + def get_cmdclass(): """Get our cmdclass dictionary for use in setuptool.setup(). @@ -81,7 +82,7 @@ def get_cmdclass(): to add our own classes to the cmdclass dictionary, and then update that dictionary with the one returned from versioneer.get_cmdclass(). """ - cmdclass = {'test': runTests, + cmdclass = {'test': Trial, 'compile_catalog': compile_catalog, 'extract_messages': extract_messages, 'init_catalog': init_catalog, @@ -238,15 +239,69 @@ def get_data_files(filesonly=False): return data_files -class runTests(setuptools.Command): - # Based on setup.py from mixminion, which is based on setup.py - # from Zooko's pyutil package, which is in turn based on - # http://mail.python.org/pipermail/distutils-sig/2002-January/002714.html - description = "Run unit tests" - user_options = [] +class Trial(setuptools.Command): + """Twisted Trial setuptools command. + + Based on the setuptools Trial command in Zooko's Tahoe-LAFS, as well as + https://github.com/simplegeo/setuptools-trial/ (which is also based on the + Tahoe-LAFS code). + + Pieces of the original implementation of this 'test' command (that is, for + the original pyunit-based BridgeDB tests which, a long time ago, in a + galaxy far far away, lived in bridgedb.Tests) were based on setup.py from + Nick Mathewson's mixminion, which was based on the setup.py from Zooko's + pyutil package, which was in turn based on + http://mail.python.org/pipermail/distutils-sig/2002-January/002714.html. + + Crusty, old-ass Python, like hella wut. + """ + description = "Run Twisted Trial-based tests." + user_options = [ + ('debug', 'b', ("Run tests in a debugger. If that debugger is pdb, will " + "load '.pdbrc' from current directory if it exists.")), + ('debug-stacktraces', 'B', "Report Deferred creation and callback stack traces"), + ('debugger=', None, ("The fully qualified name of a debugger to use if " + "--debug is passed (default: pdb)")), + ('disablegc', None, "Disable the garbage collector"), + ('force-gc', None, "Have Trial run gc.collect() before and after each test case"), + ('jobs=', 'j', "Number of local workers to run, a strictly positive integer"), + ('profile', None, "Run tests under the Python profiler"), + ('random=', 'Z', "Run tests in random order using the specified seed"), + ('reactor=', 'r', "Which reactor to use"), + ('reporter=', None, "Customize Trial's output with a reporter plugin"), + ('rterrors', 'e', "Realtime errors: print out tracebacks as soon as they occur"), + ('spew', None, "Print an insanely verbose log of everything that happens"), + ('testmodule=', None, "Filename to grep for test cases (-*- test-case-name)"), + ('tbformat=', None, ("Specify the format to display tracebacks with. Valid " + "formats are 'plain', 'emacs', and 'cgitb' which uses " + "the nicely verbose stdlib cgitb.text function")), + ('unclean-warnings', None, "Turn dirty reactor errors into warnings"), + ('until-failure', 'u', "Repeat a test (specified by -s) until it fails."), + ('without-module=', None, ("Fake the lack of the specified modules, separated " + "with commas")), + ] + boolean_options = ['debug', 'debug-stacktraces', 'disablegc', 'force-gc', + 'profile', 'rterrors', 'spew', 'unclean-warnings', + 'until-failure'] def initialize_options(self): - pass + self.debug = None + self.debug_stacktraces = None + self.debugger = None + self.disablegc = None + self.force_gc = None + self.jobs = None + self.profile = None + self.random = None + self.reactor = None + self.reporter = None + self.rterrors = None + self.spew = None + self.testmodule = None + self.tbformat = None + self.unclean_warnings = None + self.until_failure = None + self.without_module = None def finalize_options(self): build = self.get_finalized_command('build') @@ -257,11 +312,55 @@ class runTests(setuptools.Command): self.run_command('build') old_path = sys.path[:] sys.path[0:0] = [self.build_purelib, self.build_platlib] + + result = 1 try: - testmod = __import__("bridgedb.Tests", globals(), "", []) - testmod.Tests.main() + result = self.run_tests() finally: sys.path = old_path + raise SystemExit(result) + + def run_tests(self): + # We do the import from Twisted inside the function instead of the top + # of the file because since Twisted is a setup_requires, we can't + # assume that Twisted will be installed on the user's system prior, so + # if we don't do the import here, then importing from this plugin will + # fail. + from twisted.scripts import trial + + if not self.testmodule: + self.testmodule = "bridgedb.test" + + # Handle parsing the trial options passed through the setuptools + # trial command. + cmd_options = [] + for opt in self.boolean_options: + if getattr(self, opt.replace('-', '_'), None): + cmd_options.append('--%s' % opt) + + for opt in ('debugger', 'jobs', 'random', 'reactor', 'reporter', + 'testmodule', 'tbformat', 'without-module'): + value = getattr(self, opt.replace('-', '_'), None) + if value is not None: + cmd_options.extend(['--%s' % opt, value]) + + config = trial.Options() + config.parseOptions(cmd_options) + config['tests'] = [self.testmodule,] + + trial._initialDebugSetup(config) + trialRunner = trial._makeRunner(config) + suite = trial._getSuite(config) + + # run the tests + if self.until_failure: + test_result = trialRunner.runUntilFailure(suite) + else: + test_result = trialRunner.run(suite) + + if test_result.wasSuccessful(): + return 0 # success + return 1 # failure # If there is an environment variable BRIDGEDB_INSTALL_DEPENDENCIES=0, it will

1 0