November 2012 - tor-commits - lists.torproject.org

r25908: {website} update the jobs page with the support-translator-position (website/trunk/about/en)
by Runa Sandvik 26 Nov '12

26 Nov '12

Author: runa Date: 2012-11-26 16:13:49 +0000 (Mon, 26 Nov 2012) New Revision: 25908 Modified: website/trunk/about/en/jobs.wml Log: update the jobs page with the support-translator-position Modified: website/trunk/about/en/jobs.wml =================================================================== --- website/trunk/about/en/jobs.wml 2012-11-26 15:04:21 UTC (rev 25907) +++ website/trunk/about/en/jobs.wml 2012-11-26 16:13:49 UTC (rev 25908) @@ -19,6 +19,7 @@ <li><a href="<page about/jobs-projectcoordinator>">Project coordinator</a></li> <li><a href="<page about/jobs-pluggabletransport>">Pluggable transport developer</a></li> <li><a href="<page about/jobs-browserhacker>">Browser hacker</a></li> +<li><a href="<page about/jobs-translatorsupport>">Support assistants and translators</a></li> </ul> <p>In the past (and hopefully again in the future, when we have more

1 0

[metrics-tasks/master] Add graphing code for new linf data (#1854).
by karsten＠torproject.org 26 Nov '12

26 Nov '12

commit eaae6a2bb25b122866e84549faa7c54cba7d53c8 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Nov 26 11:09:29 2012 -0500 Add graphing code for new linf data (#1854). --- task-1854/plot-entropy.R | 133 ++++++++++++++++++++++++++++++---------------- task-1854/pyextract.py | 29 ++++++++++ 2 files changed, 116 insertions(+), 46 deletions(-) diff --git a/task-1854/plot-entropy.R b/task-1854/plot-entropy.R index e28193f..735e65f 100644 --- a/task-1854/plot-entropy.R +++ b/task-1854/plot-entropy.R @@ -2,52 +2,93 @@ library(ggplot2) library(reshape) library(scales) -d <- read.csv("entropy.csv", header = FALSE, - col.names = c("validafter", "min_cw", "relays", "all", "max_all", - "exit", "max_exit", "guard", "max_guard", "country", "max_country", - "as", "max_as"), stringsAsFactor = FALSE) -max_validafter <- max(d$validafter) -d <- d[d$validafter == max_validafter, ] - -e <- d[d$min_cw <= 10000, ] -e <- data.frame(min_cw = e$min_cw, all = e$all / e$max_all, - exit = e$exit / e$max_exit, guard = e$guard / e$max_guard, - country = e$country / e$max_country, as = e$as / e$max_as) -e <- melt(e, "min_cw") -e <- data.frame(min_cw = e$min_cw, variable = - ifelse(e$variable == "all", "All relays", - ifelse(e$variable == "exit", "All exits", - ifelse(e$variable == "guard", "All guards", - ifelse(e$variable == "country", "All countries", - "All ASes")))), value = e$value) -ggplot(e, aes(x = min_cw, y = value)) + +e <- read.csv("extracted.csv", header = FALSE, + col.names = c("validafter", "min_adv_bw", "relays", "linf", "graph"), + stringsAsFactor = FALSE) + +l <- e[e$graph == 'last', ] +l <- data.frame(x = l$min_adv_bw, relays = l$relays, linf = l$linf) +l <- melt(l, "x") +ggplot(l, aes(x = x, y = value)) + +geom_line() + +facet_grid(variable ~ ., scales = "free_y") + +scale_x_log10(name = "\nAdvertised bandwidth cutoff in B/s (log scale)") + +scale_y_continuous(name = "") + +opts(title = paste("Consensus with valid-after time ", max(e$validafter), + "\n", sep = "")) + +l <- e[e$graph == 'last' & e$min_adv_bw >= 10000 & e$min_adv_bw <= 100000, ] +l <- data.frame(x = l$min_adv_bw, relays = l$relays, linf = l$linf) +l <- melt(l, "x") +ggplot(l, aes(x = x, y = value)) + +geom_line() + +facet_grid(variable ~ ., scales = "free_y") + +scale_x_log10(name = "\nAdvertised bandwidth cutoff in B/s (log scale)") + +scale_y_continuous(name = "") + +opts(title = paste("Consensus with valid-after time ", max(e$validafter), + "\n", sep = "")) + +h <- e[e$graph == 'history' & e$min_adv_bw == 10000, ] +h <- data.frame(validafter = h$validafter, relays = h$relays, linf = h$linf) +h <- aggregate(h[, 2:length(h)], by = list(x = as.Date(h$validafter)), FUN = mean) +h <- melt(h, "x") +ggplot(h, aes(x = as.POSIXct(x), y = value)) + +geom_line() + +facet_grid(variable ~ ., scales = "free_y") + +scale_x_datetime(name = "") + +scale_y_continuous(name = "") + +opts(title = "Advertised bandwidth cutoff 10000 B/s\n") + +h <- e[e$graph == 'history' & e$min_adv_bw == 50000, ] +h <- data.frame(validafter = h$validafter, relays = h$relays, linf = h$linf) +h <- aggregate(h[, 2:length(h)], by = list(x = as.Date(h$validafter)), FUN = mean) +h <- melt(h, "x") +ggplot(h, aes(x = as.POSIXct(x), y = value)) + +geom_line() + +facet_grid(variable ~ ., scales = "free_y") + +scale_x_datetime(name = "") + +scale_y_continuous(name = "") + +opts(title = "Advertised bandwidth cutoff 50000 B/s\n") + +h <- e[e$graph == 'history' & e$min_adv_bw == 100000, ] +h <- data.frame(validafter = h$validafter, relays = h$relays, linf = h$linf) +h <- aggregate(h[, 2:length(h)], by = list(x = as.Date(h$validafter)), FUN = mean) +h <- melt(h, "x") +ggplot(h, aes(x = as.POSIXct(x), y = value)) + geom_line() + -facet_wrap(~ variable) + -scale_x_continuous(name = "\nMinimum consensus weight") + -scale_y_continuous(name = "", limits = c(0, 1), labels = percent) + -opts(title = paste("Degree of anonymity based on consensus from ", - max_validafter, "\n", sep = "")) -ggsave("degree-of-anonymity-min-cw.png", width = 8, height = 6, dpi = 100) - -f <- d[d$min_cw <= 10000, ] -f <- rbind( - data.frame(min_cw = f$min_cw, entropy = f$all, max = f$max_all, - type = "All relays"), - data.frame(min_cw = f$min_cw, entropy = f$exit, max = f$max_exit, - type = "All exits"), - data.frame(min_cw = f$min_cw, entropy = f$guard, max = f$max_guard, - type = "All guards"), - data.frame(min_cw = f$min_cw, entropy = f$country, max = f$max_country, - type = "All countries"), - data.frame(min_cw = f$min_cw, entropy = f$as, max = f$max_as, - type = "All ASes")) -f <- melt(f, c("min_cw", "type")) -ggplot(f, aes(x = min_cw, y = value, colour = variable)) + +facet_grid(variable ~ ., scales = "free_y") + +scale_x_datetime(name = "") + +scale_y_continuous(name = "") + +opts(title = "Advertised bandwidth cutoff 100000 B/s\n") + +h <- e[e$graph == 'history' & e$min_adv_bw %in% c(10000, 50000, 100000), ] +h <- data.frame(validafter = h$validafter, min_adv_bw = h$min_adv_bw, relays = h$relays, linf = h$linf) +h <- aggregate(h[, 2:length(h)], + by = list(x = as.Date(h$validafter), min_adv_bw = h$min_adv_bw), FUN = mean) +h <- melt(h, c("x", "min_adv_bw")) +ggplot(h, aes(x = as.POSIXct(x), y = value, colour = as.factor(min_adv_bw))) + +geom_line() + +facet_grid(variable ~ ., scales = "free_y") + +scale_x_datetime(name = "") + +scale_y_continuous(name = "") + +scale_colour_hue(name = "Advertised bandwidth cutoff in B/s") + +opts(legend.position = "bottom") + +h <- e[e$graph == 'history' & e$min_adv_bw == 10000, ] +m10000 <- data.frame(linf = sort(h$linf), + frac_cons = (1:length(h$linf))/length(h$linf), min_adv_bw = "10000") +h <- e[e$graph == 'history' & e$min_adv_bw == 50000, ] +m50000 <- data.frame(linf = sort(h$linf), + frac_cons = (1:length(h$linf))/length(h$linf), min_adv_bw = "50000") +h <- e[e$graph == 'history' & e$min_adv_bw == 100000, ] +m100000 <- data.frame(linf = sort(h$linf), + frac_cons = (1:length(h$linf))/length(h$linf), min_adv_bw = "100000") +h <- rbind(m10000, m50000, m100000) +ggplot(h, aes(x = linf, y = frac_cons, colour = as.factor(min_adv_bw))) + geom_line() + -facet_wrap(~ type) + -scale_x_continuous(name = "\nMinimum consensus weight") + -scale_y_continuous(name = "", limits = c(0, max(f$value))) + -opts(title = paste("Entropy and maximum entropy based on consensus from ", - max_validafter, "\n", sep = ""), legend.position = "none") -ggsave("entropy-min-cw.png", width = 8, height = 6, dpi = 100) +scale_x_continuous(name = "") + +scale_y_continuous( + name = "Fraction of consensuses from 2011-11-01 to 2012-10-31\n") + +scale_colour_hue(name = "Advertised bandwidth cutoff in B/s") + +opts(legend.position = "bottom") diff --git a/task-1854/pyextract.py b/task-1854/pyextract.py new file mode 100644 index 0000000..4220a2a --- /dev/null +++ b/task-1854/pyextract.py @@ -0,0 +1,29 @@ +import os +import sys + +def main(): + out_file = open('extracted.csv', 'w') + last_validafter = '' + last_lines = [] + for line in open('entropy.csv'): + parts = line.strip().split(',') + validafter = parts[0] + min_adv_bw = int(parts[1]) + relays = parts[2] + linf = parts[3] + if last_validafter != validafter: + last_lines = [] + next_cutoffs = [0, 10000, 20000, 30000, 40000, 50000, 75000, + 100000, 100000000000000000000] + while min_adv_bw >= next_cutoffs[0]: + out_file.write("%s,%d,%s,%s,history\n" % (validafter, + next_cutoffs[0], relays, linf, )) + next_cutoffs.pop(0) + last_lines.append(line.strip()) + last_validafter = validafter + for line in last_lines: + out_file.write(line + ",last\n") + +if __name__ == '__main__': + main() +

1 0

r25907: {website} update the instructions for claiming t-shirts. (website/trunk/getinvolved/en)
by Andrew Lewman 26 Nov '12

26 Nov '12

Author: phobos Date: 2012-11-26 15:04:21 +0000 (Mon, 26 Nov 2012) New Revision: 25907 Modified: website/trunk/getinvolved/en/tshirt.wml Log: update the instructions for claiming t-shirts. Modified: website/trunk/getinvolved/en/tshirt.wml =================================================================== --- website/trunk/getinvolved/en/tshirt.wml 2012-11-25 23:22:26 UTC (rev 25906) +++ website/trunk/getinvolved/en/tshirt.wml 2012-11-26 15:04:21 UTC (rev 25907) @@ -9,16 +9,16 @@ <a href="<page getinvolved/volunteer>">Get Involved » </a> <a href="<page getinvolved/tshirt>">Tshirt</a> </div> - <div id="maincol"> + <div id="maincol"> <h1>Tor: T-shirt for Contributing</h1> <hr> - + <p>You can get one of these fine Tor T-shirts for contributing to the Tor project. There are three primary ways of contributing:</p> - + <ol> - <li>A large enough ($65+) <a href="<page donate/donate>">donation</a> to the Tor - Project.</li> + <li>A large enough ($65+) <a href="<page + donate/donate>">donation</a> to the Tor Project.</li> <li>Operate a fast <a href="<page docs/tor-doc-relay>">Tor relay</a> that's been running for the past two months: you are eligible if you allow exits to port 80 and you average 100 KB/s traffic, or if you're @@ -32,21 +32,20 @@ or establish yourself as a Tor advocate. </li> </ol> - + <p> - If you qualify, send mail to donations at torproject dot org with - a brief explanation. Be sure to specify a color preference, a size - (S/M/L/XL/XXL), a backup size if your first choice isn't available, - and a shipping address. + If you qualify, send mail to donations(a)torproject.org with + a brief explanation. You will receive a unique, one-time use code + to claim your t-shirt. </p> - + <p> You can choose between the traditional black and our conversation-starting bright green. You can also see the shirts <a href="<wiki>doc/TorShirt">in action</a> — add your own photos there too. </p> - + <div style="text-align: center"> <a href="$(IMGROOT)/black-tor-tshirt.png"><img src="$(IMGROOT)/black-tor-tshirt.png"></a> @@ -62,4 +61,4 @@  </div>  -#include <foot.wmi> +#include <foot.wmi>

1 0

[ooni-probe/master] Add logPrefixes to txagentwithsocks
by art＠torproject.org 26 Nov '12

26 Nov '12

commit 4b1c96d43520b2fd6df72e7e4845c879677d4c63 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Nov 26 13:15:02 2012 +0100 Add logPrefixes to txagentwithsocks * Improve http test template --- ooni/templates/httpt.py | 9 +++++---- ooni/utils/txagentwithsocks.py | 18 ++++++++++++++++-- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 2a368a0..480d011 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -66,6 +66,7 @@ class HTTPTest(NetTestCase): self.control_agent = Agent(reactor, sockshost="127.0.0.1", socksport=config.advanced.tor_socksport) + self.report['socksproxy'] = None sockshost, socksport = (None, None) if self.localOptions['socksproxy']: try: @@ -100,7 +101,7 @@ class HTTPTest(NetTestCase): def processInputs(self): pass - def addToReport(self, request, response=None): + def addToReport(self, request, response=None, response_body=None): """ Adds to the report the specified request and response. @@ -125,12 +126,12 @@ class HTTPTest(NetTestCase): 'headers': list(response.headers.getAllRawHeaders()), 'body': response_body, 'code': response.code - } + } self.report['requests'].append(request_response) def _processResponseBody(self, response_body, request, response, body_processor): log.debug("Processing response body") - self.addToReport(request, response) + self.addToReport(request, response, response_body) if body_processor: body_processor(response_body) else: @@ -200,10 +201,10 @@ class HTTPTest(NetTestCase): """ if not response: log.err("Got no response for request %s" % request) + self.addToReport(request, response) return else: log.debug("Got response %s" % response) - return if str(response.code).startswith('3'): self.processRedirect(response.headers.getRawHeaders('Location')[0]) diff --git a/ooni/utils/txagentwithsocks.py b/ooni/utils/txagentwithsocks.py index 49e5a3c..ffaa733 100644 --- a/ooni/utils/txagentwithsocks.py +++ b/ooni/utils/txagentwithsocks.py @@ -32,6 +32,9 @@ class SOCKSv5ClientProtocol(_WrappingProtocol): self._port = port self.ready = False + def logPrefix(self): + return 'SOCKSv5ClientProtocol' + def socks_state_0(self, data): # error state self._connectedDeferred.errback(SOCKSError(0x00)) @@ -90,6 +93,9 @@ class SOCKSv5ClientFactory(_WrappingFactory): _WrappingFactory.__init__(self, wrappedFactory) self._host, self._port = host, port + def logPrefix(self): + return 'SOCKSv5ClientFactory' + def buildProtocol(self, addr): try: proto = self._wrappedFactory.buildProtocol(addr) @@ -113,6 +119,9 @@ class SOCKS5ClientEndpoint(object): self._timeout = timeout self._bindAddress = bindAddress + def logPrefix(self): + return 'SOCKSv5ClientEndpoint' + def connect(self, protocolFactory): try: wf = SOCKSv5ClientFactory(protocolFactory, self._host, self._port) @@ -123,6 +132,7 @@ class SOCKS5ClientEndpoint(object): except: return defer.fail() + class TrueHeaders(http_headers.Headers): def __init__(self, rawHeaders=None): self._rawHeaders = dict() @@ -183,6 +193,9 @@ class TrueHeaders(http_headers.Headers): return default class HTTPClientParser(_newclient.HTTPClientParser): + def logPrefix(self): + return 'HTTPClientParser' + def connectionMade(self): self.headers = TrueHeaders() self.connHeaders = TrueHeaders() @@ -196,7 +209,6 @@ class HTTPClientParser(_newclient.HTTPClientParser): headers = self.headers headers.addRawHeader(name, value) - class HTTP11ClientProtocol(_newclient.HTTP11ClientProtocol): def request(self, request): if self._state != 'QUIESCENT': @@ -261,6 +273,9 @@ class Agent(client.Agent): self._sockshost = sockshost self._socksport = socksport + def logPrefix(self): + return 'SOCKSAgent' + def request(self, method, uri, headers=None, bodyProducer=None): if (uri.startswith('shttp') or uri.startswith('httpo')) and not HTTPConnectionPool: log.err("Requests over SOCKS are supported only with versions of Twisted >= 12.1.0") @@ -297,7 +312,6 @@ class Agent(client.Agent): parsedURI.host, parsedURI.port)) d = self._pool.getConnection(key, endpoint) - print headers._rawHeaders def cbConnected(proto): return proto.request( Request(method, requestPath, headers, bodyProducer,

1 0

[ooni-probe/master] Handle responseNeverReceived failure
by art＠torproject.org 26 Nov '12

26 Nov '12

commit 62d50ea8a826c2da3fbd17d7d184b5540a001239 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Nov 26 12:25:42 2012 +0100 Handle responseNeverReceived failure --- ooni/templates/httpt.py | 8 ++++++-- 1 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 0d53ebe..2a368a0 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -14,7 +14,7 @@ from twisted.internet.ssl import ClientContextFactory from twisted.internet import reactor from twisted.internet.error import ConnectionRefusedError, DNSLookupError, TCPTimedOutError -from twisted.web._newclient import Request, Response +from twisted.web._newclient import Request, Response, ResponseNeverReceived from ooni.nettest import NetTestCase from ooni.utils import log @@ -309,8 +309,12 @@ class HTTPTest(NetTestCase): self.report['failure'] = 'dns_lookup_error' elif isinstance(failure.value, TCPTimedOutError): - log.err("DNS lookup failure") + log.err("TCP Timed Out Error") self.report['failure'] = 'tcp_timed_out_error' + + elif isinstance(failure.value, ResponseNeverReceived): + log.err("Response Never Received") + self.report['failure'] = 'response_never_received' return d = agent.request(request['method'], request['url'], headers,

1 0

[ooni-probe/master] Rename setupProgressMeters to updateProgressMeters and add debugging.
by art＠torproject.org 26 Nov '12

26 Nov '12

commit bb2744ff8010883d1c78fa112b868161cfa62c12 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Nov 26 03:57:15 2012 +0100 Rename setupProgressMeters to updateProgressMeters and add debugging. * Call such function every time we iterate over a test --- ooni/runner.py | 13 +++++++++---- 1 files changed, 9 insertions(+), 4 deletions(-) diff --git a/ooni/runner.py b/ooni/runner.py index 942e3b5..6ea4a86 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -329,11 +329,10 @@ def increaseInputUnitIdx(test_filename): config.stateDict[test_filename] += 1 yield updateResumeFile(test_filename) -def setupProgressMeters(test_filename, input_unit_factory, +def updateProgressMeters(test_filename, input_unit_factory, test_case_number): """ - Sets up the meters required for keeping track of the current progress of - certain tests. + Update the progress meters for keeping track of test state. """ log.msg("Setting up progress meters") if not config.state.test_filename: @@ -347,6 +346,10 @@ def setupProgressMeters(test_filename, input_unit_factory, total_iterations = input_unit_items * test_case_number current_iteration = input_unit_idx * test_case_number + log.debug("Test case number: %s" % test_case_number) + log.debug("Total iterations: %s" % total_iterations) + log.debug("Current iteration: %s" % current_iteration) + def progress(): return (current_iteration / total_iterations) * 100.0 @@ -403,7 +406,7 @@ def runTestCases(test_cases, options, cmd_line_options): else: config.stateDict[test_filename] = 0 - setupProgressMeters(test_filename, input_unit_factory, len(test_cases)) + updateProgressMeters(test_filename, input_unit_factory, len(test_cases)) try: for input_unit in input_unit_factory: @@ -413,6 +416,8 @@ def runTestCases(test_cases, options, cmd_line_options): oreporter) yield increaseInputUnitIdx(test_filename) + updateProgressMeters(test_filename, input_unit_factory, len(test_cases)) + except Exception: log.exception("Problem in running test")

1 0

[ooni-probe/master] Store the state of the currently running tests, their progress and estimated time to completion
by art＠torproject.org 26 Nov '12

26 Nov '12

commit a3829e7a9700bfa748088c18e3afa44a086ed6e2 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Nov 26 03:38:47 2012 +0100 Store the state of the currently running tests, their progress and estimated time to completion * Display every 5 seconds a summary of all the running tests and their progress * Refactor input unit and nettest --- ooni/config.py | 1 + ooni/inputunit.py | 27 +++++++++++++----- ooni/nettest.py | 22 ++++++++++----- ooni/oonicli.py | 15 ++++++++++- ooni/reporter.py | 2 +- ooni/runner.py | 66 ++++++++++++++++++++++++++++++++-------------- tests/test_inputunit.py | 15 +++++++++- 7 files changed, 108 insertions(+), 40 deletions(-) diff --git a/ooni/config.py b/ooni/config.py index d86f4d7..69842b6 100644 --- a/ooni/config.py +++ b/ooni/config.py @@ -14,6 +14,7 @@ from ooni.utils import Storage reports = Storage() scapyFactory = None stateDict = None +state = Storage() # XXX refactor this to use a database resume_lock = defer.DeferredLock() diff --git a/ooni/inputunit.py b/ooni/inputunit.py index 65d578c..2ef89d8 100644 --- a/ooni/inputunit.py +++ b/ooni/inputunit.py @@ -9,7 +9,6 @@ # :authors: Arturo Filastò # :license: see included LICENSE file - class InputUnitFactory(object): """ This is a factory that takes the size of input units to be generated a set @@ -20,27 +19,39 @@ class InputUnitFactory(object): all the elements in memory to be able to produce InputUnits. """ inputUnitSize = 10 + length = None def __init__(self, inputs=[]): + """ + Args: + inputs (iterable): inputs *must* be an iterable. + """ self._inputs = iter(inputs) - self._idx = 0 + self.inputs = iter(inputs) self._ended = False def __iter__(self): return self + def __len__(self): + """ + Returns the number of input units in the input unit factory. + """ + if not self.length: + self.length = sum(1 for _ in self._inputs)/self.inputUnitSize + return self.length + def next(self): input_unit_elements = [] if self._ended: raise StopIteration - for i in xrange(self._idx, self._idx + self.inputUnitSize): + for i in xrange(self.inputUnitSize): try: - input_unit_elements.append(self._inputs.next()) + input_unit_elements.append(self.inputs.next()) except StopIteration: self._ended = True break - self._idx += self.inputUnitSize if not input_unit_elements: raise StopIteration @@ -55,12 +66,12 @@ class InputUnit(object): def __init__(self, inputs=[]): self._inputs = iter(inputs) - def __repr__(self): + def __str__(self): return "<%s inputs=%s>" % (self.__class__, self._inputs) def __add__(self, inputs): - for input in inputs: - self._inputs.append(input) + for i in inputs: + self._inputs.append(i) def __iter__(self): return self diff --git a/ooni/nettest.py b/ooni/nettest.py index e96fba1..e0393e7 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -22,6 +22,7 @@ from ooni.utils import log class NoPostProcessor(Exception): pass + class NetTestCase(object): """ This is the base of the OONI nettest universe. When you write a nettest @@ -150,17 +151,22 @@ class NetTestCase(object): if not self.localOptions[required_option]: raise usage.UsageError("%s not specified!" % required_option) - def _processOptions(self, options=None): + def _processOptions(self): if self.inputFilename: - self.inputs = self.inputProcessor(self.inputFilename) - - self._checkRequiredOptions() + inputProcessor = self.inputProcessor + inputFilename = self.inputFilename + class inputProcessorIterator(object): + """ + Here we convert the input processor generator into an iterator + so that we can run it twice. + """ + def __iter__(self): + return inputProcessor(inputFilename) + self.inputs = inputProcessorIterator() - # XXX perhaps we may want to name and version to be inside of a - # different method that is not called options. return {'inputs': self.inputs, - 'name': self.name, - 'version': self.version} + 'name': self.name, 'version': self.version + } def __repr__(self): return "<%s inputs=%s>" % (self.__class__, self.inputs) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 3a8b3df..2b0991b 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -15,7 +15,7 @@ import random import time import yaml -from twisted.internet import defer, reactor +from twisted.internet import defer, reactor, task from twisted.application import app from twisted.python import usage, failure from twisted.python.util import spewer @@ -78,6 +78,15 @@ class Options(usage.Options): except: raise usage.UsageError("No test filename specified!") +def updateStatusBar(): + for test_filename in config.state.keys(): + # The ETA is not updated so we we will not print it out for the + # moment. + eta = config.state[test_filename].eta() + progress = config.state[test_filename].progress() + progress_bar_frmt = "[%s] %s%%" % (test_filename, progress) + print progress_bar_frmt + def testsEnded(*arg, **kw): """ You can place here all the post shutdown tasks. @@ -123,6 +132,10 @@ def run(): d2 = defer.DeferredList(deck_dl) d2.addBoth(testsEnded) + # Print every 5 second the list of current tests running + l = task.LoopingCall(updateStatusBar) + l.start(5.0) + if config.start_reactor: log.debug("Starting reactor") reactor.run() diff --git a/ooni/reporter.py b/ooni/reporter.py index 3cc77d7..dac9aba 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -284,7 +284,7 @@ class OONIBReporter(OReporter): bodyProducer = StringProducer(json.dumps(request)) try: - response = yield self.agent.request("PUT", url, + response = yield self.agent.request("PUT", url, bodyProducer=bodyProducer) except: # XXX we must trap this in the runner and make sure to report the data later. diff --git a/ooni/runner.py b/ooni/runner.py index e7a40fd..942e3b5 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -26,7 +26,7 @@ from ooni.nettest import NetTestCase, NoPostProcessor from ooni import reporter, config -from ooni.utils import log, checkForRoot, NotRootError +from ooni.utils import log, checkForRoot, NotRootError, Storage def processTest(obj): """ @@ -67,7 +67,7 @@ def processTest(obj): try: log.debug("processing options") tmp_test_case_object = obj() - tmp_test_case_object._processOptions(options) + tmp_test_case_object._checkRequiredOptions() except usage.UsageError, e: test_name = tmp_test_case_object.name @@ -120,6 +120,9 @@ def makeTestCases(klass, tests, method_prefix): cases.append((klass, method_prefix+test)) return cases +class NoTestCasesFound(Exception): + pass + def loadTestsAndOptions(classes, cmd_line_options): """ Takes a list of test classes and returns their testcases and options. @@ -134,7 +137,10 @@ def loadTestsAndOptions(classes, cmd_line_options): test_cases = makeTestCases(klass, tests, method_prefix) test_klass = klass() - options = test_klass._processOptions(cmd_line_options) + options = test_klass._processOptions() + + if not test_cases: + raise NoTestCasesFound return test_cases, options @@ -220,7 +226,8 @@ def runTestCasesWithInputUnit(test_cases, input_unit, oreporter): dl = [] for test_input in input_unit: log.debug("Running test with this input %s" % test_input) - d = runTestCasesWithInput(test_cases, test_input, oreporter) + d = runTestCasesWithInput(test_cases, + test_input, oreporter) dl.append(d) return defer.DeferredList(dl) @@ -322,28 +329,45 @@ def increaseInputUnitIdx(test_filename): config.stateDict[test_filename] += 1 yield updateResumeFile(test_filename) +def setupProgressMeters(test_filename, input_unit_factory, + test_case_number): + """ + Sets up the meters required for keeping track of the current progress of + certain tests. + """ + log.msg("Setting up progress meters") + if not config.state.test_filename: + config.state[test_filename] = Storage() + + config.state[test_filename].per_item_average = 2.0 + + input_unit_idx = float(config.stateDict[test_filename]) + input_unit_items = float(len(input_unit_factory) + 1) + test_case_number = float(test_case_number) + total_iterations = input_unit_items * test_case_number + current_iteration = input_unit_idx * test_case_number + + def progress(): + return (current_iteration / total_iterations) * 100.0 + + config.state[test_filename].progress = progress + + def eta(): + return (total_iterations - current_iteration) \ + * config.state[test_filename].per_item_average + config.state[test_filename].eta = eta + + config.state[test_filename].input_unit_idx = input_unit_idx + config.state[test_filename].input_unit_items = input_unit_items + + @defer.inlineCallbacks def runTestCases(test_cases, options, cmd_line_options): log.debug("Running %s" % test_cases) log.debug("Options %s" % options) log.debug("cmd_line_options %s" % dict(cmd_line_options)) - try: - assert len(options) != 0, "Length of options is zero!" - except AssertionError, ae: - test_inputs = [] - log.err(ae) - else: - try: - first = options.pop(0) - except: - first = options - if 'inputs' in first: - test_inputs = options['inputs'] - else: - log.msg("Could not find inputs!") - log.msg("options[0] = %s" % first) - test_inputs = [None] + test_inputs = options['inputs'] if cmd_line_options['collector']: log.msg("Using remote collector, please be patient while we create the report.") @@ -379,6 +403,8 @@ def runTestCases(test_cases, options, cmd_line_options): else: config.stateDict[test_filename] = 0 + setupProgressMeters(test_filename, input_unit_factory, len(test_cases)) + try: for input_unit in input_unit_factory: log.debug("Running this input unit %s" % input_unit) diff --git a/tests/test_inputunit.py b/tests/test_inputunit.py index f591e23..1f9043c 100644 --- a/tests/test_inputunit.py +++ b/tests/test_inputunit.py @@ -1,10 +1,13 @@ import unittest from ooni.inputunit import InputUnit, InputUnitFactory +def dummyGenerator(): + for x in range(100): + yield x + class TestInputUnit(unittest.TestCase): def test_input_unit_factory(self): - inputs = range(100) - inputUnit = InputUnitFactory(inputs) + inputUnit = InputUnitFactory(range(100)) for i in inputUnit: self.assertEqual(len(list(i)), inputUnit.inputUnitSize) @@ -16,3 +19,11 @@ class TestInputUnit(unittest.TestCase): idx += 1 self.assertEqual(idx, 100) + + def test_input_unit_factory_length(self): + inputUnitFactory = InputUnitFactory(range(100)) + l1 = len(inputUnitFactory) + l2 = sum(1 for _ in inputUnitFactory) + self.assertEqual(l1, 10) + self.assertEqual(l2, 10) +

1 0

r25906: {website} Adding tlsdate to the volunteer page Adding tlsdate to the v (website/trunk/getinvolved/en)
by Damian Johnson 25 Nov '12

25 Nov '12

Author: atagar Date: 2012-11-25 23:22:26 +0000 (Sun, 25 Nov 2012) New Revision: 25906 Modified: website/trunk/getinvolved/en/volunteer.wml Log: Adding tlsdate to the volunteer page Adding tlsdate to the volunteer page as per Jake's request. Modified: website/trunk/getinvolved/en/volunteer.wml =================================================================== --- website/trunk/getinvolved/en/volunteer.wml 2012-11-21 20:36:41 UTC (rev 25905) +++ website/trunk/getinvolved/en/volunteer.wml 2012-11-25 23:22:26 UTC (rev 25906) @@ -253,6 +253,14 @@ </tr> <tr class="alt"> + <td><a href="#project-tlsdate">Tlsdate</a></td> + <td>Utility</td> + <td>C</td> + <td>Heavy</td> + <td>ioerror</td> + </tr> + + <tr> <td><a href="#project-metrics">Metrics</a></td> <td>Client Service</td> <td>Java</td> @@ -260,7 +268,7 @@ <td>karsten</td> </tr> - <tr> + <tr class="alt"> <td><a href="#project-atlas">Atlas</a></td> <td>Client Service</td> <td>JavaScript</td> @@ -268,7 +276,7 @@ <td>hellais</td> </tr> - <tr class="alt"> + <tr> <td><a href="#project-torstatus">TorStatus</a></td> <td>Client Service</td> <td>Python, Django</td> @@ -276,7 +284,7 @@ <td></td> </tr> - <tr> + <tr class="alt"> <td><a href="#project-compass">Compass</a></td> <td>Client Service</td> <td>Python</td> @@ -284,7 +292,7 @@ <td>gsathya, karsten</td> </tr> - <tr class="alt"> + <tr> <td><a href="#project-weather">Weather</a></td> <td>Client Service</td> <td>Python</td> @@ -292,7 +300,7 @@ <td>kaner</td> </tr> - <tr> + <tr class="alt"> <td><a href="#project-gettor">GetTor</a></td> <td>Client Service</td> <td>Python</td> @@ -300,7 +308,7 @@ <td>kaner</td> </tr> - <tr class="alt"> + <tr> <td><a href="#project-torcheck">TorCheck</a></td> <td>Client Service</td> <td>Python, Perl</td> @@ -308,7 +316,7 @@ <td>ioerror</td> </tr> - <tr> + <tr class="alt"> <td><a href="#project-onionoo">Onionoo</a></td> <td>Backend Service</td> <td>Java, Python</td> @@ -316,7 +324,7 @@ <td>karsten, gsathya</td> </tr> - <tr class="alt"> + <tr> <td><a href="#project-bridgedb">BridgeDB</a></td> <td>Backend Service</td> <td>Python</td> @@ -324,7 +332,7 @@ <td>kaner, nickm</td> </tr> - <tr> + <tr class="alt"> <td><a href="#project-torflow">TorFlow</a></td> <td>Backend Service</td> <td>Python</td> @@ -332,7 +340,7 @@ <td>aagbsn</td> </tr> - <tr class="alt"> + <tr> <td>*<a href="#project-torbel">TorBEL</a></td> <td>Backend Service</td> <td>Python</td> @@ -622,6 +630,21 @@ <i><a href="#stemUsability">Stem Usability Improvements</a></i> </p> + <a id="project-tlsdate"></a> + <h3>Tlsdate (<a href="https://github.com/ioerror/tlsdate">code</a>)</h3> + + <p> + tlsdate: secure parasitic rdate replacement + </p> + + <p> + tlsdate sets the local clock by securely connecting with TLS to remote + servers and extracting the remote time out of the secure handshake. Unlike + ntpdate, tlsdate uses TCP, for instance connecting to a remote HTTPS or TLS + enabled service, and provides some protection against adversaries that try + to feed you malicious time information. + </p> + <a id="project-metrics"></a> <h3><a href="https://metrics.torproject.org/">Metrics</a> (code: <a href="https://gitweb.torproject.org/metrics-db.git">db</a>, <a

1 0

[ooni-probe/master] Update README
by art＠torproject.org 25 Nov '12

25 Nov '12

commit 913742bac4ab230234ff45ca117bcbcc6852bab3 Author: aagbsn <aagbsn(a)extc.org> Date: Sun Nov 25 09:05:00 2012 -0800 Update README Adds additional required dependencies and instructions for using virtualenvwrapper. --- README.md | 41 ++++++++++++++++++++++++++++++++++------- 1 files changed, 34 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index e5c0a84..4100541 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Basic requirements: On debian based systems these can be installed with: - sudo apt-get install git-core python python-pip python-dev + sudo apt-get install git-core python python-pip python-dev build-essential The python dependencies required for running ooniprobe are: @@ -37,24 +37,47 @@ The python dependencies required for running ooniprobe are: * BeautifulSoup: http://www.crummy.com/software/BeautifulSoup/ * txtorcon: https://github.com/meejah/txtorcon +## Configurating a virtual environment + You are highly recommended to install python packages from inside of a virtual environment, since pip does not download the packages via SSL and you will need to install it system wide. This will require you to have installed virtualenv. - sudo apt-get install python-virtualenv + sudo apt-get install python-virtualenv virtualenvwrapper + +To create a new virtual environment do + + mkdir $HOME/.virtualenvs + mkvirtualenv ooni-probe + +You will automatically enter the environment. To re-enter this environment in the future, type: + + workon ooni-probe + +For convenience, you may want to add the following to your .bashrc: + + if [ -e ~/ooni-probe/bin ]; then + export PATH=~/ooni-probe/bin:$PATH + fi + if [ -e ~/ooni-probe ]; then + export PYTHONPATH=$PYTHONPATH:~/ooni-probe + fi + +Add the following to $HOME/.virtualenvs/ooni-probe/bin/postactivate to automatically cd into the working directory upon activation. + + if [ -e ~/ooni-probe ] ; then + cd ~/ooni-probe + fi + +## Installing ooni-probe Clone the ooniprobe repository: git clone https://git.torproject.org/ooni-probe.git cd ooni-probe -To create a new virtual environment do - - virtualenv env - source env/bin/activate - Then install OONI with: pip install -r requirements.txt @@ -67,6 +90,10 @@ If you are not in a virtualenv you will have to run the above command as root: It's ideal to install these manually since the ones in debian or ubuntu are not up to date. +If you don't already have Subversion installed: + + sudo apt-get install subversion + For libdnet: wget http://libdnet.googlecode.com/files/libdnet-1.12.tgz

1 0

[ooni-probe/master] Merge pull request #8 from aagbsn/update-setup-instruction
by art＠torproject.org 25 Nov '12

25 Nov '12

commit 0bb6ea25c84e55c7904ae2608e432ee4e8a46372 Merge: 7332e71 913742b Author: Arturo Filastò <hellais(a)gmail.com> Date: Sun Nov 25 15:06:27 2012 -0800 Merge pull request #8 from aagbsn/update-setup-instruction Update README README.md | 41 ++++++++++++++++++++++++++++++++++------- 1 files changed, 34 insertions(+), 7 deletions(-)

1 0