March 2012 - tor-commits - lists.torproject.org

[onionoo/master] Remove version, valid_after, and fresh_until lines from responses.
by karsten＠torproject.org 05 Mar '12

05 Mar '12

commit 5f98afeb716d5404d819c430ca842cdcfbc8caba Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Mar 5 08:11:06 2012 +0100 Remove version, valid_after, and fresh_until lines from responses. --- src/org/torproject/onionoo/ResourceServlet.java | 28 ++--------- src/org/torproject/onionoo/SummaryDataWriter.java | 10 +--- web/index.html | 56 --------------------- 3 files changed, 6 insertions(+), 88 deletions(-) diff --git a/src/org/torproject/onionoo/ResourceServlet.java b/src/org/torproject/onionoo/ResourceServlet.java index 529ed2b..63d0edb 100644 --- a/src/org/torproject/onionoo/ResourceServlet.java +++ b/src/org/torproject/onionoo/ResourceServlet.java @@ -28,9 +28,7 @@ public class ResourceServlet extends HttpServlet { long summaryFileLastModified = 0L; boolean readSummaryFile = false; - private String versionLine = null, validAfterLine = null, - freshUntilLine = null, relaysPublishedLine = null, - bridgesPublishedLine = null; + private String relaysPublishedLine = null, bridgesPublishedLine = null; private List<String> relayLines = new ArrayList<String>(), bridgeLines = new ArrayList<String>(); private void readSummaryFile() { @@ -40,7 +38,6 @@ public class ResourceServlet extends HttpServlet { return; } if (summaryFile.lastModified() > this.summaryFileLastModified) { - this.versionLine = this.validAfterLine = this.freshUntilLine = null; this.relayLines.clear(); this.bridgeLines.clear(); try { @@ -48,14 +45,9 @@ public class ResourceServlet extends HttpServlet { summaryFile)); String line; while ((line = br.readLine()) != null) { - if (line.startsWith("{\"version\":")) { - this.versionLine = line; - } else if (line.startsWith("\"valid_after\":")) { - this.validAfterLine = line; - } else if (line.startsWith("\"fresh_until\":")) { - this.freshUntilLine = line; - } else if (line.startsWith("\"relays_published\":")) { - this.relaysPublishedLine = line; + if (line.contains("\"relays_published\":")) { + this.relaysPublishedLine = line.startsWith("{") ? line : + "{" + line; } else if (line.startsWith("\"bridges_published\":")) { this.bridgesPublishedLine = line; } else if (line.startsWith("\"relays\":")) { @@ -129,25 +121,21 @@ public class ResourceServlet extends HttpServlet { response.setCharacterEncoding("utf-8"); PrintWriter pw = response.getWriter(); if (uri.equals("/" + resourceType + "/all")) { - this.writeHeader(pw); pw.print(this.relaysPublishedLine + "\n"); this.writeAllRelays(pw, resourceType); pw.print(this.bridgesPublishedLine + "\n"); this.writeAllBridges(pw, resourceType); } else if (uri.equals("/" + resourceType + "/running")) { - this.writeHeader(pw); pw.print(this.relaysPublishedLine + "\n"); this.writeRunningRelays(pw, resourceType); pw.print(this.bridgesPublishedLine + "\n"); this.writeRunningBridges(pw, resourceType); } else if (uri.equals("/" + resourceType + "/relays")) { - this.writeHeader(pw); pw.print(this.relaysPublishedLine + "\n"); this.writeAllRelays(pw, resourceType); pw.print(this.bridgesPublishedLine + "\n"); this.writeNoBridges(pw); } else if (uri.equals("/" + resourceType + "/bridges")) { - this.writeHeader(pw); pw.print(this.relaysPublishedLine + "\n"); this.writeNoRelays(pw); pw.print(this.bridgesPublishedLine + "\n"); @@ -155,7 +143,6 @@ public class ResourceServlet extends HttpServlet { } else if (uri.startsWith("/" + resourceType + "/search/")) { String searchParameter = this.parseSearchParameter(uri.substring( ("/" + resourceType + "/search/").length())); - this.writeHeader(pw); pw.print(this.relaysPublishedLine + "\n"); this.writeMatchingRelays(pw, searchParameter, resourceType); pw.print(this.bridgesPublishedLine + "\n"); @@ -163,7 +150,6 @@ public class ResourceServlet extends HttpServlet { } else if (uri.startsWith("/" + resourceType + "/lookup/")) { Set<String> fingerprintParameters = this.parseFingerprintParameters( uri.substring(("/" + resourceType + "/lookup/").length())); - this.writeHeader(pw); pw.print(this.relaysPublishedLine + "\n"); this.writeRelaysWithFingerprints(pw, fingerprintParameters, resourceType); @@ -198,12 +184,6 @@ public class ResourceServlet extends HttpServlet { return parsedFingerprints; } - private void writeHeader(PrintWriter pw) { - pw.print(this.versionLine + "\n"); - pw.print(this.validAfterLine + "\n"); - pw.print(this.freshUntilLine + "\n"); - } - private void writeAllRelays(PrintWriter pw, String resourceType) { pw.print("\"relays\":["); int written = 0; diff --git a/src/org/torproject/onionoo/SummaryDataWriter.java b/src/org/torproject/onionoo/SummaryDataWriter.java index b731900..55f2f11 100644 --- a/src/org/torproject/onionoo/SummaryDataWriter.java +++ b/src/org/torproject/onionoo/SummaryDataWriter.java @@ -40,9 +40,6 @@ public class SummaryDataWriter { "yyyy-MM-dd HH:mm:ss"); dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); long now = System.currentTimeMillis(); - String validAfterString = dateTimeFormat.format(now); - String freshUntilString = dateTimeFormat.format(now - + 75L * 60L * 1000L); String relaysPublishedString = dateTimeFormat.format( this.lastValidAfterMillis); String bridgesPublishedString = dateTimeFormat.format( @@ -50,11 +47,8 @@ public class SummaryDataWriter { this.relaySearchDataFile.getParentFile().mkdirs(); BufferedWriter bw = new BufferedWriter(new FileWriter( this.relaySearchDataFile)); - bw.write("{\"version\":1,\n" - + "\"valid_after\":\"" + validAfterString + "\",\n" - + "\"fresh_until\":\"" + freshUntilString + "\",\n" - + "\"relays_published\":\"" + relaysPublishedString + "\",\n" - + "\"relays\":["); + bw.write("{\"relays_published\":\"" + relaysPublishedString + + "\",\n\"relays\":["); int written = 0; for (Node entry : this.currentRelays.values()) { String nickname = !entry.getNickname().equals("Unnamed") ? diff --git a/web/index.html b/web/index.html index 3199ea5..8e5766c 100755 --- a/web/index.html +++ b/web/index.html @@ -43,30 +43,6 @@ fingerprints, IP addresses, and running information as well as bridges with hashed fingerprints and running information. Summary documents contain the following fields: <ul> -<li>"version": Version number of this -document. -Deprecated field. -Deploying a versioning scheme to this service seems like overkill. -Picking new field names and deprecating fields after one month means a lot -less maintenance overhead. -Announced on January 26, will be removed after March 4, 2012.</li> -<li>"valid_after": UTC timestamp -(YYYY-MM-DD hh:mm:ss) when the contents of this document were generated. -Deprecated field. -This field was a quite broken way to implement caching. -The current caching approach is to have clients cache documents for five -minutes. -A future caching mechanism is going to use HTTP headers. -Announced on January 26, will be removed after March 4, 2012.</li> -<li>"fresh_until": UTC timestamp -(YYYY-MM-DD hh:mm:ss) until when the contents of this document shall be -considered fresh. -Deprecated field. -This field was a quite broken way to implement caching. -The current caching approach is to have clients cache documents for five -minutes. -A future caching mechanism is going to use HTTP headers. -Announced on January 26, will be removed after March 4, 2012.</li> <li>"relays_published": UTC timestamp (YYYY-MM-DD hh:mm:ss) when the last contained relay network status consensus started being valid. Indicates how recent the relay summaries in this document are. @@ -159,22 +135,6 @@ Detail documents are based on the network statuses published by the Tor directories and the server descriptors published by relays and bridges. Detail documents contain the following fields: <ul> -<li>"version": Version number of this -document. -Deprecated field. -Deprecated for the same reason as stated for summary documents. -Announced on January 26, will be removed after March 4, 2012.</li> -<li>"valid_after": UTC timestamp -(YYYY-MM-DD hh:mm:ss) when the contents of this document were generated. -Deprecated field. -Deprecated for the same reason as stated for summary documents. -Announced on January 26, will be removed after March 4, 2012.</li> -<li>"fresh_until": UTC timestamp -(YYYY-MM-DD hh:mm:ss) until when the contents of this document shall be -considered fresh. -Deprecated field. -Deprecated for the same reason as stated for summary documents. -Announced on January 26, will be removed after March 4, 2012.</li> <li>"relays_published": UTC timestamp (YYYY-MM-DD hh:mm:ss) when the last contained relay network status consensus started being valid. Indicates how recent the relay details in this document are. @@ -449,22 +409,6 @@ Bandwidth documents are available for all relays and bridges that have been running in the past week. Bandwidth documents contain the following fields: <ul> -<li>"version": Version number of this -document. -Deprecated field. -Deprecated for the same reason as stated for summary documents. -Announced on January 26, will be removed after March 4, 2012.</li> -<li>"valid_after": UTC timestamp -(YYYY-MM-DD hh:mm:ss) when the contents of this document were generated. -Deprecated field. -Deprecated for the same reason as stated for summary documents. -Announced on January 26, will be removed after March 4, 2012.</li> -<li>"fresh_until": UTC timestamp -(YYYY-MM-DD hh:mm:ss) until when the contents of this document shall be -considered fresh. -Deprecated field. -Deprecated for the same reason as stated for summary documents. -Announced on January 26, will be removed after March 4, 2012.</li> <li>"relays_published": UTC timestamp (YYYY-MM-DD hh:mm:ss) when the last contained relay network status consensus started being valid. Indicates how recent the relay bandwidth documents in this document are.

1 0

[ooni-probe/master] getting serious now
by art＠torproject.org 05 Mar '12

05 Mar '12

commit 393cb623d307f12f87f59159aadb99e99adb1cfa Author: Isis Lovecruft <isis(a)patternsinthevoid.net> Date: Sat Mar 3 20:09:07 2012 -0800 getting serious now --- logo.py | 30 ++++++++++++++++++++++++++++++ 1 files changed, 30 insertions(+), 0 deletions(-) diff --git a/logo.py b/logo.py index b9849b1..5669eae 100644 --- a/logo.py +++ b/logo.py @@ -181,4 +181,34 @@ o O o O O o O o O O o O o O O | / | / ~-. `-. _ _ _ |_____| |_____| ~ - . _ _ _ _ _> """) + logo.append(""" + _________________ + ,/~~, | | + /`` \ /| OONI-PROBE... | + `( O^O ) / | | + `\_-_/` | ...DOES THIS | + ~~~ | EXCITE YOU? | + __| |__ |_________________| + | | + | af | + | | | | + | | | \ + | | |\ \ + | | | \ \ + | | | \ \ + | |___| ((( + (((|/*\| + | | | + | | | + | | | + | | | + | | | + | | | + | | | + | | | + | | | + | | | + _| | |_ + (___|___) +""") return random.choice(logo)

1 0

[ooni-probe/master] Merge branch 'master' of github.com:isislovecruft/ooni-probe
by art＠torproject.org 05 Mar '12

05 Mar '12

commit e222596c8f5df5a859df82a08d3fa1da22094821 Merge: 393cb62 0705e9a Author: Isis Lovecruft <isis(a)patternsinthevoid.net> Date: Sat Mar 3 20:09:49 2012 -0800 Merge branch 'master' of github.com:isislovecruft/ooni-probe ooniprobe.py | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-)

1 0

[ooni-probe/master] Introduce node.py.
by art＠torproject.org 05 Mar '12

05 Mar '12

commit 46a130d1a28025cc5060560d734deed11b4346c9 Author: George Kadianakis <desnacked(a)riseup.net> Date: Sat Mar 3 23:48:23 2012 -0800 Introduce node.py. --- node.py | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 66 insertions(+), 0 deletions(-) diff --git a/node.py b/node.py new file mode 100644 index 0000000..eea78df --- /dev/null +++ b/node.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 + +import os +import sys +import socks + +class Node(object): + def __init__(self, address, port): + self.address = address + self.port = port + +""" +[]: node = NetworkNode("192.168.0.112", 5555, "SOCKS5") +[]: node_socket = node.wrap_socket() +""" +class NetworkNode(Node): + def __init__(self, address, port, node_type="SOCKS5", auth_creds=None): + self.node = Node(address,port) + + # XXX support for multiple types + # node type (SOCKS proxy, HTTP proxy, GRE tunnel, ...) + self.node_type = node_type + # type-specific authentication credentials + self.auth_creds = auth_creds + + def _get_socksipy_socket(self, proxy_type, auth_creds): + import socks + s = socks.socksocket() + # auth_creds[0] -> username + # auth_creds[1] -> password + s.setproxy(proxy_type, self.node.address, self.node.port, + self.auth_creds[0], self.auth_creds[1]) + return s + + def _get_socket_wrapper(self): + if (self.node_type.startswith("SOCKS")): # SOCKS proxies + if (self.node_type != "SOCKS5"): + proxy_type = socks.PROXY_TYPE_SOCKS5 + elif (self.node_type != "SOCKS4"): + proxy_type = socks.PROXY_TYPE_SOCKS4 + else: + print "We don't know this proxy type." + sys.exit(1) + + return self._get_socksipy_socket(proxy_type) + elif (self.node_type == "HTTP"): # HTTP proxies + return self._get_socksipy_socket(PROXY_TYPE_HTTP) + else: # Unknown proxies + print "We don't know this proxy type." + sys.exit(1) + + def wrap_socket(self): + return self._get_socket_wrapper() + +class CodeExecNode(Node): + def __init__(self, address, port, node_type, auth_creds): + self.node = Node(address,port) + + # node type (SSH proxy, etc.) + self.node_type = node_type + # type-specific authentication credentials + self.auth_creds = auth_creds + + def add_unit(self): + pass

1 0

[ooni-probe/master] Merge branch 'node', remote-tracking branch 'remotes/gitorious/node' into node
by art＠torproject.org 05 Mar '12

05 Mar '12

commit 71a52ff25b7dfeebe036cc6669fe9002889d5bd5 Author: Arturo Filastò <hellais(a)gmail.com> Date: Sun Mar 4 13:02:22 2012 -0800 Merge branch 'node', remote-tracking branch 'remotes/gitorious/node' into node

1 0

[ooni-probe/master] PL addnodes exclude
by art＠torproject.org 05 Mar '12

05 Mar '12

commit e78bf9357a9c4c16373f42bac656a3d1ed890d66 Author: Isis Lovecruft <isis(a)patternsinthevoid.net> Date: Sat Mar 3 23:55:40 2012 -0800 PL addnodes exclude --- .gitignore | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/.gitignore b/.gitignore index b1e99aa..6433d66 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ proxy-lists/ip-cc.txt proxy-lists/ips.txt proxy-lists/italy-dns-ips.txt proxy-lists/italy-http-ips.txt +private/* \ No newline at end of file

1 0

[ooni-probe/master] Merge branch 'isis', remote-tracking branch 'remotes/isis/master' into isis
by art＠torproject.org 05 Mar '12

05 Mar '12

commit e1deb9a3a0f8962e145a30371f033b8cc48b4626 Author: Arturo Filastò <hellais(a)gmail.com> Date: Sun Mar 4 15:12:49 2012 -0800 Merge branch 'isis', remote-tracking branch 'remotes/isis/master' into isis

1 0

[ooni-probe/master] Start implementing the work unit classes
by art＠torproject.org 05 Mar '12

05 Mar '12

commit 4b02a217d4f43182b1634615bfc935dc213034c1 Author: Arturo Filastò <hellais(a)gmail.com> Date: Sun Mar 4 15:13:54 2012 -0800 Start implementing the work unit classes * Fix ascii art --- logo.py | 32 ++++++++++++++++---------------- plugoo.py | 46 +++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 59 insertions(+), 19 deletions(-) diff --git a/logo.py b/logo.py index 5669eae..3b47775 100644 --- a/logo.py +++ b/logo.py @@ -141,16 +141,16 @@ o O o O O o O o O O o O o O O """) logo.append(""" ________________________________________ -/ OONI-PROBE -- WE FIGHT CENSORSHIP LIKE \ +/ OONI-PROBE -- WE FIGHT CENSORSHIP LIKE \\ \ DRAGONS EAT KNIGHTS! / ---------------------------------------- \ ^ /^ - \ / \ // \ - \ |\___/| / \// .\ + \ / \ // \\ + \ |\___/| / \// .\\ \ /O O \__ / // | \ \ *----* / / \/_/ // | \ \ \ | - @___@` \/_ // | \ \ \/\ \ - 0/0/| \/_ // | \ \ \ \ + @___@` \/_ // | \ \ \/\ \\ + 0/0/| \/_ // | \ \ \ \\ 0/0/0/0/| \/// | \ \ | | 0/0/0/0/0/_|_ / ( // | \ _\ | / 0/0/0/0/0/0/`/,_ _ _/ ) ; -. | _ _\.-~ / / @@ -158,8 +158,8 @@ o O o O O o O o O O o O o O O \ \__/ `/\ / ~-. _ .-~ / \____(oo) *. } { / ( (--) .----~-.\ \-` .~ - //__\\ \__ Ack! ///.----..< \ _ -~ - // \\ ///-._ _ _ _ _ _ _{^ - - - - ~ + //__\\\\ \__ Ack! ///.----..< \ _ -~ + // \\\\ ///-._ _ _ _ _ _ _{^ - - - - ~ """) logo.append(""" _________________ @@ -167,17 +167,17 @@ o O o O O o O o O O o O o O O | DON'T CENSOR ME | |_________________| \ . . - \ / `. .' " + \ / `. .' " \ .---. < > < > .---. \ | \ \ - ~ ~ - / / | _____ ..-~ ~-..-~ | (A)| \~~~\.' `./~~~/ --------- \__/ \__/ - .' O \ / / \ " + .' O \ / / \ " (_____, `._.' | } \/~~~/ `----. / } | / \__/ `-. | / | / `. ,~~| - ~-.__| /_ - ~ ^| /- _ `..-' + ~-.__| /_ - ~ ^| /- _ `..-' | / | / ~-. `-. _ _ _ |_____| |_____| ~ - . _ _ _ _ _> """) @@ -189,16 +189,16 @@ o O o O O o O o O O o O o O O `\_-_/` | ...DOES THIS | ~~~ | EXCITE YOU? | __| |__ |_________________| - | | + | | | af | | | | | - | | | \ - | | |\ \ - | | | \ \ - | | | \ \ + | | | \\ + | | |\ \\ + | | | \ \\ + | | | \ \\ | |___| ((( (((|/*\| - | | | + | | | | | | | | | | | | diff --git a/plugoo.py b/plugoo.py index 0ce1d28..07f553d 100644 --- a/plugoo.py +++ b/plugoo.py @@ -26,7 +26,8 @@ import itertools import gevent class Asset: - """This is an ooni-probe asset. It is a python + """ + This is an ooni-probe asset. It is a python iterator object, allowing it to be efficiently looped. To create your own custom asset your should subclass this and override the next_asset method and the len method for @@ -106,7 +107,7 @@ class Report: import paramiko except: self.scp = None - self.logger.warn("Could not import paramiko. SCP will not be disabled") + self.logger.warn("Could not import paramiko. SCP will be disabled") def __call__(self, data): """ @@ -231,6 +232,44 @@ class Report: self.logger.info("Reporting to %s" % type) getattr(self, type+"_report").__call__(data) +class Worker(): + """ + The worker is responsible for keeping track + of WorkUnits. Generating them, reading them. + This allows for resume support and dispatching + work units to remote OONI control centers. + """ + def __init__(self, status=None): + self.status = 0 + if status: + self.resumeStatus(status) + + def resumeStatus(self, status): + pass + + def next(self): + pass + +class WorkUnits(): + """ + This is a unit of work. It is associated with a node on + which it should be performed and a set of tasks to fulfil. + """ + def __init__(self): + pass + + def __iter__(self): + return self + + def next(self): + try: + return self + except: + raise StopIteration + + + + class Plugoo(): def __init__(self, ooni): self.config = ooni.config @@ -323,7 +362,8 @@ class Plugoo(): jobs = [] class torify(object): - """This is the torify decorator. It should be used to + """ + This is the torify decorator. It should be used to decorate functions that should use to for connecting to the interwebz. The suggary syntax is the following: @torify([urllib2])

1 0

[ooni-probe/master] Merge branch 'node'
by art＠torproject.org 05 Mar '12

05 Mar '12

commit af274db0a78dd0287bf537b1935b3e4062dc85c6 Merge: fc0ee97 4258d6e Author: Arturo Filastò <hellais(a)gmail.com> Date: Sun Mar 4 22:54:33 2012 -0800 Merge branch 'node' Conflicts: plugoo.py ooniprobe.py | 3 +- plugoo.py | 388 --------------------------------------------------- plugoo/__init__.py | 53 +++++++ plugoo/assets.py | 45 ++++++ plugoo/nodes.py | 71 ++++++++++ plugoo/reports.py | 170 ++++++++++++++++++++++ plugoo/tests.py | 105 ++++++++++++++ tests/bridget.py | 10 +- tests/dnstamper.py | 8 +- tests/traceroute.py | 7 +- 10 files changed, 460 insertions(+), 400 deletions(-)

1 0

[ooni-probe/master] Big refactor and code reorganization.
by art＠torproject.org 05 Mar '12

05 Mar '12

commit 4258d6e33547ab24d604d6e800dc1ee23df43b64 Author: Arturo Filastò <hellais(a)gmail.com> Date: Sun Mar 4 22:53:20 2012 -0800 Big refactor and code reorganization. --- node.py | 66 ---------- ooniprobe.py | 3 +- plugoo.py | 348 --------------------------------------------------- plugoo/__init__.py | 53 ++++++++ plugoo/assets.py | 45 +++++++ plugoo/nodes.py | 71 +++++++++++ plugoo/reports.py | 170 +++++++++++++++++++++++++ plugoo/tests.py | 105 +++++++++++++++ tests/bridget.py | 10 +- tests/dnstamper.py | 8 +- tests/traceroute.py | 7 +- 11 files changed, 460 insertions(+), 426 deletions(-) diff --git a/node.py b/node.py deleted file mode 100644 index eea78df..0000000 --- a/node.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 - -import os -import sys -import socks - -class Node(object): - def __init__(self, address, port): - self.address = address - self.port = port - -""" -[]: node = NetworkNode("192.168.0.112", 5555, "SOCKS5") -[]: node_socket = node.wrap_socket() -""" -class NetworkNode(Node): - def __init__(self, address, port, node_type="SOCKS5", auth_creds=None): - self.node = Node(address,port) - - # XXX support for multiple types - # node type (SOCKS proxy, HTTP proxy, GRE tunnel, ...) - self.node_type = node_type - # type-specific authentication credentials - self.auth_creds = auth_creds - - def _get_socksipy_socket(self, proxy_type, auth_creds): - import socks - s = socks.socksocket() - # auth_creds[0] -> username - # auth_creds[1] -> password - s.setproxy(proxy_type, self.node.address, self.node.port, - self.auth_creds[0], self.auth_creds[1]) - return s - - def _get_socket_wrapper(self): - if (self.node_type.startswith("SOCKS")): # SOCKS proxies - if (self.node_type != "SOCKS5"): - proxy_type = socks.PROXY_TYPE_SOCKS5 - elif (self.node_type != "SOCKS4"): - proxy_type = socks.PROXY_TYPE_SOCKS4 - else: - print "We don't know this proxy type." - sys.exit(1) - - return self._get_socksipy_socket(proxy_type) - elif (self.node_type == "HTTP"): # HTTP proxies - return self._get_socksipy_socket(PROXY_TYPE_HTTP) - else: # Unknown proxies - print "We don't know this proxy type." - sys.exit(1) - - def wrap_socket(self): - return self._get_socket_wrapper() - -class CodeExecNode(Node): - def __init__(self, address, port, node_type, auth_creds): - self.node = Node(address,port) - - # node type (SSH proxy, etc.) - self.node_type = node_type - # type-specific authentication credentials - self.auth_creds = auth_creds - - def add_unit(self): - pass diff --git a/ooniprobe.py b/ooniprobe.py index d8501c1..a542153 100755 --- a/ooniprobe.py +++ b/ooniprobe.py @@ -96,7 +96,8 @@ class ooni(object): test_name = fname if not self.config.main.testdir in sys.path: sys.path.insert(0, self.config.main.testdir) - + #print "Fname: %s\n__import__(%s)" % (fname, fname) + #print sys.path module = __import__(fname) try: test.name = module.__plugoo__ diff --git a/plugoo.py b/plugoo.py deleted file mode 100644 index 0ce1d28..0000000 --- a/plugoo.py +++ /dev/null @@ -1,348 +0,0 @@ -# -*- coding: UTF-8 -""" - plugoo - ****** - - This contains all of the "goo" necessary for creating - ooni-probe plugoonies. - - :copyright: (c) 2012 by Arturo Filastò. - :license: see LICENSE for more details. - -""" - -import os -from datetime import datetime -import yaml - -try: - import socks -except: - "Error SocksiPy is not installed!" -import socket - -import logging -import itertools -import gevent - -class Asset: - """This is an ooni-probe asset. It is a python - iterator object, allowing it to be efficiently looped. - To create your own custom asset your should subclass this - and override the next_asset method and the len method for - computing the length of the asset. - """ - def __init__(self, file=None): - self.fh = None - if file: - self.name = file - self.fh = open(file, 'r') - self.eof = False - - def __iter__(self): - return self - - def len(self): - """Returns the length of the asset - """ - for i, l in enumerate(self.fh): - pass - # rewind the file - self.fh.seek(0) - return i + 1 - - def next_asset(self): - """Return the next asset. - """ - # XXX this is really written with my feet. - # clean me up please... - line = self.fh.readline() - if line: - return line.replace('\n','') - else: - self.fh.seek(0) - raise StopIteration - - def next(self): - try: - return self.next_asset() - except: - raise StopIteration - - -class Report: - """This is the ooni-probe reporting mechanism. It allows - reporting to multiple destinations and file formats. - - :scp the string of <host>:<port> of an ssh server - - :yaml the filename of a the yaml file to write - - :file the filename of a simple txt file to write - - :tcp the <host>:<port> of a TCP server that will just listen for - inbound connection and accept a stream of data (think of it - as a `nc -l -p <port> > filename.txt`) - """ - def __init__(self, ooni, - scp="127.0.0.1:22", - file="test.report", - tcp="127.0.0.1:9000"): - - self.file = file - self.tcp = tcp - self.scp = scp - self.config = ooni.config.report - self.logger = ooni.logger - - if self.config.timestamp: - tmp = self.file.split('.') - self.file = '.'.join(tmp[:-1]) + "-" + \ - datetime.now().isoformat('-') + '.' + \ - tmp[-1] - print self.file - - try: - import paramiko - except: - self.scp = None - self.logger.warn("Could not import paramiko. SCP will not be disabled") - - def __call__(self, data): - """ - This should be invoked every time you wish to write some - data to the reporting system - """ - #print "Writing report(s)" - dump = '--- \n' - dump += yaml.dump(data) - reports = [] - - if self.file: - reports.append("file") - - if self.tcp: - reports.append("tcp") - - if self.scp: - reports.append("scp") - - jobs = [gevent.spawn(self.send_report, *(dump, report)) for report in reports] - gevent.joinall(jobs) - ret = [] - for job in jobs: - #print job.value - ret.append(job.value) - return ret - - def file_report(self, data, file=None, mode='a+'): - """ - This reports to a file in YAML format - """ - if not file: - file = self.file - try: - f = open(file, mode) - f.write(data) - except Exception, e: - raise e - finally: - f.close() - - - def tcp_report(self, data): - """This connect to the specified tcp server - and writes the data passed as argument. - """ - host, port = self.tcp.split(":") - tcp = socket.getprotobyname('tcp') - send_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM, tcp) - try: - send_socket.connect((host, int(port))) - send_socket.send(data) - - except Exception, e: - raise e - - finally: - send_socket.close() - - - def scp_report(self, data, rfile=None, mode='a+'): - """Push data to the remote ssh server. - :rfile the remote filename to write - :data the raw data content that should be written - :mode in what mode the file should be created - """ - if not rfile: - rfile = self.file - host, port = self.scp.split(":") - transport = paramiko.Transport((host, port)) - - # The remote path of the remote file to write - rfpath = os.path.join(self.config.ssh_rpath, rfile) - - try: - username = self.config.ssh_username - except: - raise "No username provided" - - # Load the local known host key file - transport.load_host_keys(os.path.expanduser("~/.ssh/known_hosts")) - - # We prefer to use an ssh keyfile fo authentication - if self.config.ssh_keyfile: - keyfile = os.path.expanduser(self.config.ssh_keyfile) - key = paramiko.RSAKey.from_private_key_file(keylocfile) - try: - transport.connect(username=username, pkey=key) - except Exception, e: - raise e - - # If not even a password is fine - elif self.config.ssh_password: - try: - transport.connect(username=username, password=self.config.ssh_password) - except Exception, e: - raise e - - # ... but no authentication, that is madness! - else: - raise "No key or password provided for ssh" - - sftp = paramiko.SFTPClient.from_transport(transport) - try: - sftp = ssh.open_sftp() - remote_file = sftp.file(rfile, mode) - remote_file.set_pipelined(True) - remote_file.write(data) - - except Exception, e: - raise e - sftp.close() - transport.close() - - - def send_report(self, data, type): - """This sends the report using the - specified type. - """ - #print "Reporting %s to %s" % (data, type) - self.logger.info("Reporting to %s" % type) - getattr(self, type+"_report").__call__(data) - -class Plugoo(): - def __init__(self, ooni): - self.config = ooni.config - self.logger = ooni.logger - self.name = "test" - self.report = Report(ooni, - scp=ooni.config.report.ssh, - file=ooni.config.report.file, - tcp=ooni.config.report.tcp) - - - def control(self, *a, **b): - pass - - def experiment(self, *a, **b): - """Override this method to write your own - Plugoo. - """ - pass - - def load_assets(self, assets): - """Takes as input an array of Asset objects and - outputs an iterator for the loaded assets. - example: - assets = [hostlist, portlist, requestlist] - - """ - asset_count = len(assets) - bigsize = 0 - bigidx = 0 - - if asset_count > 1: - # If we have more than on asset we try to do some - # optimizations as how to iterate through them by - # picking the largest asset set as the main iterator - # and do a cartesian product on the smaller sets - for i, v in enumerate(assets): - size = v.len() - if size > bigsize: - bigidx, bigsize = (i, size) - - smallassets = list(assets) - smallassets.pop(bigidx) - - for x in assets[bigidx]: - if asset_count > 1: - # XXX this will only work in python 2.6, maybe refactor? - for comb in itertools.product(*smallassets): - yield (x,) + comb - else: - yield (x) - - def srun(self, assets=None, buffer=10, timeout=2): - self.logger.info("Starting %s", self.name) - if assets: - self.logger.debug("Running through tests") - for i, data in enumerate(self.load_assets(assets)): - args = {'data': data} - ret = self.experiment(**args) - print ret - self.report(ret) - - def run(self, assets=None, buffer=10, timeout=100000): - self.logger.info("Starting %s", self.name) - jobs = [] - if assets: - self.logger.debug("Running through tests") - for i, data in enumerate(self.load_assets(assets)): - args = {'data': data} - # Append to the job queue - jobs.append(gevent.spawn(self.experiment, **args)) - # If the buffer is full run the jobs - if i % buffer == (buffer-1): - # Run the jobs with the selected timeout - gevent.joinall(jobs, timeout=timeout) - for job in jobs: - #print "JOB VAL: %s" % job.value - self.logger.info("Writing report(s)") - self.report(job.value) - job.kill() - jobs = [] - - if len(jobs) > 0: - gevent.joinall(jobs, timeout=timeout) - for job in jobs: - #print "JOB VAL: %s" % job.value - self.logger.info("Writing report(s)") - self.report(job.value) - job.kill() - jobs = [] - -class torify(object): - """This is the torify decorator. It should be used to - decorate functions that should use to for connecting to - the interwebz. The suggary syntax is the following: - @torify([urllib2]) - def myfunction(): - f = urllib2.urlopen('https://torproject.org/') - remember to set the proxyaddress in the config file. - """ - def __init__(self, f): - print f - self.f = f - - def __get__(self, instance, owner): - self.modules = instance.modules - def decorator(*args): - print instance.config.main.proxyaddress - host, port = instance.config.main.proxyaddress.split(":") - socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, host, int(port)) - # Wrap the modules into socks - for module in self.modules: - socks.wrapmodule(module) - return self.f(instance, *args) - return decorator diff --git a/plugoo/__init__.py b/plugoo/__init__.py new file mode 100644 index 0000000..1bd4446 --- /dev/null +++ b/plugoo/__init__.py @@ -0,0 +1,53 @@ +# -*- coding: UTF-8 +""" + plugoo + ****** + + This contains all of the "goo" necessary for creating + ooni-probe plugoonies. + + :copyright: (c) 2012 by Arturo Filastò. + :license: see LICENSE for more details. + +""" + +__all__ = ['assets', 'reports', 'nodes'] + +import os +from datetime import datetime +import yaml + +try: + import socks +except: + "Error SocksiPy is not installed!" +import socket + +import logging +import itertools +import gevent + +class torify(object): + """This is the torify decorator. It should be used to + decorate functions that should use to for connecting to + the interwebz. The suggary syntax is the following: + @torify([urllib2]) + def myfunction(): + f = urllib2.urlopen('https://torproject.org/') + remember to set the proxyaddress in the config file. + """ + def __init__(self, f): + print f + self.f = f + + def __get__(self, instance, owner): + self.modules = instance.modules + def decorator(*args): + print instance.config.main.proxyaddress + host, port = instance.config.main.proxyaddress.split(":") + socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, host, int(port)) + # Wrap the modules into socks + for module in self.modules: + socks.wrapmodule(module) + return self.f(instance, *args) + return decorator diff --git a/plugoo/assets.py b/plugoo/assets.py new file mode 100644 index 0000000..5f93d1b --- /dev/null +++ b/plugoo/assets.py @@ -0,0 +1,45 @@ +class Asset: + """This is an ooni-probe asset. It is a python + iterator object, allowing it to be efficiently looped. + To create your own custom asset your should subclass this + and override the next_asset method and the len method for + computing the length of the asset. + """ + def __init__(self, file=None, *args, **argv): + self.fh = None + if file: + self.name = file + self.fh = open(file, 'r') + self.eof = False + + def __iter__(self): + return self + + def len(self): + """Returns the length of the asset + """ + for i, l in enumerate(self.fh): + pass + # rewind the file + self.fh.seek(0) + return i + 1 + + def next_asset(self): + """Return the next asset. + """ + # XXX this is really written with my feet. + # clean me up please... + line = self.fh.readline() + if line: + return line.replace('\n','') + else: + self.fh.seek(0) + raise StopIteration + + def next(self): + try: + return self.next_asset() + except: + raise StopIteration + + diff --git a/plugoo/nodes.py b/plugoo/nodes.py new file mode 100644 index 0000000..a76601d --- /dev/null +++ b/plugoo/nodes.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 + +import os +import sys +import socks + +class Node(object): + def __init__(self, address, port): + self.address = address + self.port = port + +""" +[]: node = NetworkNode("192.168.0.112", 5555, "SOCKS5") +[]: node_socket = node.wrap_socket() +""" +class NetworkNode(Node): + def __init__(self, address, port, node_type="SOCKS5", auth_creds=None): + self.node = Node(address,port) + + # XXX support for multiple types + # node type (SOCKS proxy, HTTP proxy, GRE tunnel, ...) + self.node_type = node_type + # type-specific authentication credentials + self.auth_creds = auth_creds + + def _get_socksipy_socket(self, proxy_type, auth_creds): + import socks + s = socks.socksocket() + # auth_creds[0] -> username + # auth_creds[1] -> password + s.setproxy(proxy_type, self.node.address, self.node.port, + self.auth_creds[0], self.auth_creds[1]) + return s + + def _get_socket_wrapper(self): + if (self.node_type.startswith("SOCKS")): # SOCKS proxies + if (self.node_type != "SOCKS5"): + proxy_type = socks.PROXY_TYPE_SOCKS5 + elif (self.node_type != "SOCKS4"): + proxy_type = socks.PROXY_TYPE_SOCKS4 + else: + print "We don't know this proxy type." + sys.exit(1) + + return self._get_socksipy_socket(proxy_type) + elif (self.node_type == "HTTP"): # HTTP proxies + return self._get_socksipy_socket(PROXY_TYPE_HTTP) + else: # Unknown proxies + print "We don't know this proxy type." + sys.exit(1) + + def wrap_socket(self): + return self._get_socket_wrapper() + +class CodeExecNode(Node): + def __init__(self, address, port, node_type, auth_creds): + self.node = Node(address,port) + + # node type (SSH proxy, etc.) + self.node_type = node_type + # type-specific authentication credentials + self.auth_creds = auth_creds + + def add_unit(self): + pass + + def get_status(self): + pass + + diff --git a/plugoo/reports.py b/plugoo/reports.py new file mode 100644 index 0000000..5a9fb8b --- /dev/null +++ b/plugoo/reports.py @@ -0,0 +1,170 @@ +import os +from datetime import datetime +import yaml + +import logging +import itertools +import gevent + +class Report: + """This is the ooni-probe reporting mechanism. It allows + reporting to multiple destinations and file formats. + + :scp the string of <host>:<port> of an ssh server + + :yaml the filename of a the yaml file to write + + :file the filename of a simple txt file to write + + :tcp the <host>:<port> of a TCP server that will just listen for + inbound connection and accept a stream of data (think of it + as a `nc -l -p <port> > filename.txt`) + """ + def __init__(self, ooni, + scp="127.0.0.1:22", + file="test.report", + tcp="127.0.0.1:9000"): + + self.file = file + self.tcp = tcp + self.scp = scp + self.config = ooni.config.report + self.logger = ooni.logger + + if self.config.timestamp: + tmp = self.file.split('.') + self.file = '.'.join(tmp[:-1]) + "-" + \ + datetime.now().isoformat('-') + '.' + \ + tmp[-1] + print self.file + + try: + import paramiko + except: + self.scp = None + self.logger.warn("Could not import paramiko. SCP will not be disabled") + + def __call__(self, data): + """ + This should be invoked every time you wish to write some + data to the reporting system + """ + #print "Writing report(s)" + dump = '--- \n' + dump += yaml.dump(data) + reports = [] + + if self.file: + reports.append("file") + + if self.tcp: + reports.append("tcp") + + if self.scp: + reports.append("scp") + + jobs = [gevent.spawn(self.send_report, *(dump, report)) for report in reports] + gevent.joinall(jobs) + ret = [] + for job in jobs: + #print job.value + ret.append(job.value) + return ret + + def file_report(self, data, file=None, mode='a+'): + """ + This reports to a file in YAML format + """ + if not file: + file = self.file + try: + f = open(file, mode) + f.write(data) + except Exception, e: + raise e + finally: + f.close() + + + def tcp_report(self, data): + """This connect to the specified tcp server + and writes the data passed as argument. + """ + host, port = self.tcp.split(":") + tcp = socket.getprotobyname('tcp') + send_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM, tcp) + try: + send_socket.connect((host, int(port))) + send_socket.send(data) + + except Exception, e: + raise e + + finally: + send_socket.close() + + + def scp_report(self, data, rfile=None, mode='a+'): + """Push data to the remote ssh server. + :rfile the remote filename to write + :data the raw data content that should be written + :mode in what mode the file should be created + """ + if not rfile: + rfile = self.file + host, port = self.scp.split(":") + transport = paramiko.Transport((host, port)) + + # The remote path of the remote file to write + rfpath = os.path.join(self.config.ssh_rpath, rfile) + + try: + username = self.config.ssh_username + except: + raise "No username provided" + + # Load the local known host key file + transport.load_host_keys(os.path.expanduser("~/.ssh/known_hosts")) + + # We prefer to use an ssh keyfile fo authentication + if self.config.ssh_keyfile: + keyfile = os.path.expanduser(self.config.ssh_keyfile) + key = paramiko.RSAKey.from_private_key_file(keylocfile) + try: + transport.connect(username=username, pkey=key) + except Exception, e: + raise e + + # If not even a password is fine + elif self.config.ssh_password: + try: + transport.connect(username=username, password=self.config.ssh_password) + except Exception, e: + raise e + + # ... but no authentication, that is madness! + else: + raise "No key or password provided for ssh" + + sftp = paramiko.SFTPClient.from_transport(transport) + try: + sftp = ssh.open_sftp() + remote_file = sftp.file(rfile, mode) + remote_file.set_pipelined(True) + remote_file.write(data) + + except Exception, e: + raise e + sftp.close() + transport.close() + + + def send_report(self, data, type): + """This sends the report using the + specified type. + """ + #print "Reporting %s to %s" % (data, type) + self.logger.info("Reporting to %s" % type) + getattr(self, type+"_report").__call__(data) + + diff --git a/plugoo/tests.py b/plugoo/tests.py new file mode 100644 index 0000000..24986b2 --- /dev/null +++ b/plugoo/tests.py @@ -0,0 +1,105 @@ +import os +from datetime import datetime +import yaml + +import logging +import itertools +import gevent +from plugoo.reports import Report + +class Test: + """ + This is a ooni probe Test. + Also known as a Plugoo! + """ + def __init__(self, ooni): + self.config = ooni.config + self.logger = ooni.logger + self.name = "test" + self.report = Report(ooni, + scp=ooni.config.report.ssh, + file=ooni.config.report.file, + tcp=ooni.config.report.tcp) + + + def control(self, *a, **b): + pass + + def experiment(self, *a, **b): + """Override this method to write your own + Plugoo. + """ + pass + + def load_assets(self, assets): + """Takes as input an array of Asset objects and + outputs an iterator for the loaded assets. + example: + assets = [hostlist, portlist, requestlist] + + """ + asset_count = len(assets) + bigsize = 0 + bigidx = 0 + + if asset_count > 1: + # If we have more than on asset we try to do some + # optimizations as how to iterate through them by + # picking the largest asset set as the main iterator + # and do a cartesian product on the smaller sets + for i, v in enumerate(assets): + size = v.len() + if size > bigsize: + bigidx, bigsize = (i, size) + + smallassets = list(assets) + smallassets.pop(bigidx) + + for x in assets[bigidx]: + if asset_count > 1: + # XXX this will only work in python 2.6, maybe refactor? + for comb in itertools.product(*smallassets): + yield (x,) + comb + else: + yield (x) + + def srun(self, assets=None, buffer=10, timeout=2): + self.logger.info("Starting %s", self.name) + if assets: + self.logger.debug("Running through tests") + for i, data in enumerate(self.load_assets(assets)): + args = {'data': data} + ret = self.experiment(**args) + print ret + self.report(ret) + + def run(self, assets=None, buffer=10, timeout=100000): + self.logger.info("Starting %s", self.name) + jobs = [] + if assets: + self.logger.debug("Running through tests") + for i, data in enumerate(self.load_assets(assets)): + args = {'data': data} + # Append to the job queue + jobs.append(gevent.spawn(self.experiment, **args)) + # If the buffer is full run the jobs + if i % buffer == (buffer-1): + # Run the jobs with the selected timeout + gevent.joinall(jobs, timeout=timeout) + for job in jobs: + #print "JOB VAL: %s" % job.value + self.logger.info("Writing report(s)") + self.report(job.value) + job.kill() + jobs = [] + + if len(jobs) > 0: + gevent.joinall(jobs, timeout=timeout) + for job in jobs: + #print "JOB VAL: %s" % job.value + self.logger.info("Writing report(s)") + self.report(job.value) + job.kill() + jobs = [] + + diff --git a/tests/bridget.py b/tests/bridget.py index 0a2cc4f..61fc8b7 100644 --- a/tests/bridget.py +++ b/tests/bridget.py @@ -23,11 +23,11 @@ from subprocess import Popen, PIPE from datetime import datetime import shutil -import plugoo import gevent from gevent import socket import fcntl -from plugoo import Plugoo, Asset, torify +from plugoo.assets import Asset +from plugoo.tests import Test import urllib2 import httplib import json @@ -71,7 +71,7 @@ class BridgeTAsset(Asset): def __init__(self, file=None): self = Asset.__init__(self, file) -class BridgeT(Plugoo): +class BridgeT(Test): # This is the timeout value after which # we will give up timeout = 20 @@ -368,11 +368,11 @@ def run(ooni): bridges = BridgeTAsset(os.path.join(config.main.assetdir, \ config.tests.tor_bridges)) - assets = [bridges] + bridgelist = [bridges] bridget = BridgeT(ooni) ooni.logger.info("Starting bridget test") - bridget.run(assets) + bridget.run(bridgelist) bridget.print_failures() bridget.clean() ooni.logger.info("Testing completed!") diff --git a/tests/dnstamper.py b/tests/dnstamper.py index 06b3551..68be12d 100644 --- a/tests/dnstamper.py +++ b/tests/dnstamper.py @@ -5,16 +5,18 @@ except: import gevent import os import plugoo -from plugoo import Plugoo, Asset +from plugoo.assets import Asset +from plugoo.tests import Test + __plugoo__ = "DNST" __desc__ = "DNS censorship detection test" class DNSTAsset(Asset): def __init__(self, file=None): - self = Asset.__init__(self, file) + self = asset.__init__(self, file) -class DNST(Plugoo): +class DNST(Test): def lookup(self, hostname, ns): res = resolver.Resolver(configure=False) res.nameservers = [ns] diff --git a/tests/traceroute.py b/tests/traceroute.py index f2272c9..891d356 100644 --- a/tests/traceroute.py +++ b/tests/traceroute.py @@ -11,7 +11,8 @@ try: except: print "Error: traceroute plugin requires scapy to be installed (http://www.secdev.org/projects/scapy)" -from plugoo import Plugoo, Asset +from plugoo.assets import Asset +from plugoo.tests import Test import socket @@ -20,10 +21,10 @@ __desc__ = "Performs TTL walking tests" class TracerouteAsset(Asset): def __init__(self, file=None): - self = Asset.__init__(self, file) + self = asset.__init__(self, file) -class Traceroute(Plugoo): +class Traceroute(Test): """A *very* quick and dirty traceroute implementation, UDP and TCP """ def traceroute(self, dst, dst_port=3880, src_port=3000, proto="tcp", max_hops=30):

1 0