[or-cvs] r18443: {torflow} Add initial cut at a snakeinspector.py to dump specific resu (torflow/trunk/NetworkScanners)
mikeperry at seul.org
mikeperry at seul.org
Mon Feb 9 10:36:42 UTC 2009
Author: mikeperry
Date: 2009-02-09 05:36:42 -0500 (Mon, 09 Feb 2009)
New Revision: 18443
Added:
torflow/trunk/NetworkScanners/snakeinspector.py
Modified:
torflow/trunk/NetworkScanners/README.ExitScanning
torflow/trunk/NetworkScanners/libsoat.py
torflow/trunk/NetworkScanners/soat.py
Log:
Add initial cut at a snakeinspector.py to dump specific
results in a more manageable format.
Modified: torflow/trunk/NetworkScanners/README.ExitScanning
===================================================================
--- torflow/trunk/NetworkScanners/README.ExitScanning 2009-02-09 09:56:43 UTC (rev 18442)
+++ torflow/trunk/NetworkScanners/README.ExitScanning 2009-02-09 10:36:42 UTC (rev 18443)
@@ -17,6 +17,7 @@
Python 2.4+
Custom patched Tor 0.2.1
Super Secret SoaT Sauce
+py-openssl/pyOpenSSL
Bonus: 500M of disk space
Bonus: Secondary external IP address
@@ -95,7 +96,8 @@
IV. Running Tor, The Metatroller, and SoaT
Once you have everything compiled and configured, you should be ready to
-run the pieces.
+run the pieces. You probably want to do this as a separate, unprivileged
+user.
First, start up your custom Tor with the sample torrc provided in the
TorFlow svn root:
Modified: torflow/trunk/NetworkScanners/libsoat.py
===================================================================
--- torflow/trunk/NetworkScanners/libsoat.py 2009-02-09 09:56:43 UTC (rev 18442)
+++ torflow/trunk/NetworkScanners/libsoat.py 2009-02-09 10:36:42 UTC (rev 18443)
@@ -10,13 +10,13 @@
import time
import traceback
sys.path.append("./libs")
-from BeautifulSoup.BeautifulSoup import BeautifulSoup, Tag
+from BeautifulSoup.BeautifulSoup import Tag
import sets
from sets import Set
+from soat_config import *
-from soat_config import *
sys.path.append("../")
from TorCtl.TorUtil import *
@@ -34,6 +34,9 @@
TEST_INCONCLUSIVE = 1
TEST_FAILURE = 2
+# Sorry, we sort of rely on the ordinal nature of the above constants
+RESULT_STRINGS = {TEST_SUCCESS:"Success", TEST_INCONCLUSIVE:"Inconclusive", TEST_FAILURE:"Failure"}
+
# Inconclusive reasons
INCONCLUSIVE_NOEXITCONTENT = "InconclusiveNoExitContent"
INCONCLUSIVE_NOLOCALCONTENT = "InconclusiveNoLocalContent"
@@ -55,12 +58,14 @@
class TestResult(object):
''' Parent class for all test result classes '''
- def __init__(self, exit_node, site, status):
+ def __init__(self, exit_node, site, status, reason=None):
self.exit_node = exit_node
self.site = site
self.timestamp = time.time()
self.status = status
+ self.reason = reason
self.false_positive=False
+ self.false_positive_reason="None"
def mark_false_positive(self, reason):
pass
@@ -77,6 +82,18 @@
plog("WARN", "Error moving "+file+" to "+dir)
return file
+ def __str__(self):
+ ret = self.__class__.__name__+" for "+self.site+"\n"
+ ret += " Time: "+time.ctime(self.timestamp)+"\n"
+ ret += " Exit: "+self.exit_node+"\n"
+ ret += " "+str(RESULT_STRINGS[self.status])
+ if self.reason:
+ ret += " Reason: "+self.reason
+ if self.false_positive:
+ ret += " (False positive: "+self.false_positive_reason+")"
+ ret += "\n"
+ return ret
+
class SSLTestResult(TestResult):
''' Represents the result of an openssl test '''
def __init__(self, exit_node, ssl_site, cert_file, status):
@@ -110,17 +127,14 @@
def __init__(self, exit_node, website, status, reason=None,
sha1sum=None, exit_sha1sum=None, content=None,
content_exit=None, content_old=None, sha1sum_old=None):
- super(HttpTestResult, self).__init__(exit_node, website, status)
+ super(HttpTestResult, self).__init__(exit_node, website, status, reason)
self.proto = "http"
- self.reason = reason
self.sha1sum = sha1sum
self.sha1sum_old = sha1sum_old
self.exit_sha1sum = exit_sha1sum
self.content = content
self.content_exit = content_exit
self.content_old = content_old
- self.false_positive=False
- self.false_positive_reason="None"
def mark_false_positive(self, reason):
self.false_positive=True
@@ -137,6 +151,16 @@
try: os.unlink(self.content_exit)
except: pass
+ def __str__(self):
+ ret = TestResult.__str__(self)
+ if self.content:
+ ret += " "+self.content+" (SHA1: "+self.sha1sum+")\n"
+ if self.content_old:
+ ret += " "+self.content_old+" (SHA1: "+self.sha1sum_old+")\n"
+ if self.content_exit:
+ ret += " "+self.content_exit+" (SHA1: "+self.exit_sha1sum+")\n"
+ return ret
+
class CookieTestResult(TestResult):
def __init__(self, exit_node, status, reason, plain_cookies,
tor_cookies):
@@ -150,14 +174,11 @@
''' Represents the result of a JS test '''
def __init__(self, exit_node, website, status, reason=None,
content=None, content_exit=None, content_old=None):
- super(JsTestResult, self).__init__(exit_node, website, status)
+ super(JsTestResult, self).__init__(exit_node, website, status, reason)
self.proto = "http"
- self.reason = reason
self.content = content
self.content_exit = content_exit
self.content_old = content_old
- self.false_positive=False
- self.false_positive_reason="None"
def mark_false_positive(self, reason):
self.false_positive=True
@@ -174,18 +195,26 @@
try: os.unlink(self.content_exit)
except: pass
+ def __str__(self):
+ # XXX: Re-run the JSDiffer and compare these differences
+ ret = TestResult.__str__(self)
+ if self.content:
+ ret += " "+self.content+"\n"
+ if self.content_old:
+ ret += " "+self.content_old+"\n"
+ if self.content_exit:
+ ret += " "+self.content_exit+"\n"
+ return ret
+
class HtmlTestResult(TestResult):
''' Represents the result of a http test '''
def __init__(self, exit_node, website, status, reason=None,
content=None, content_exit=None, content_old=None):
- super(HtmlTestResult, self).__init__(exit_node, website, status)
+ super(HtmlTestResult, self).__init__(exit_node, website, status, reason)
self.proto = "http"
- self.reason = reason
self.content = content
self.content_exit = content_exit
self.content_old = content_old
- self.false_positive=False
- self.false_positive_reason="None"
def mark_false_positive(self, reason):
self.false_positive=True
@@ -202,6 +231,17 @@
try: os.unlink(self.content_exit)
except: pass
+ def __str__(self):
+ # XXX: Re-run the SoupDiffer+JSDiffer and compare these differences
+ ret = TestResult.__str__(self)
+ if self.content:
+ ret += " "+self.content+"\n"
+ if self.content_old:
+ ret += " "+self.content_old+"\n"
+ if self.content_exit:
+ ret += " "+self.content_exit+"\n"
+ return ret
+
class SSHTestResult(TestResult):
''' Represents the result of an ssh test '''
def __init__(self, exit_node, ssh_site, status):
@@ -382,7 +422,7 @@
""" Create a map of changed tags to ALL attributes that tag
has ever had (changed or not) """
changed_tags = {}
- for tags in map(BeautifulSoup, self.changed_tags()):
+ for tags in map(TheChosenSoup, self.changed_tags()):
for t in tags.findAll():
if t.name not in changed_tags:
changed_tags[t.name] = sets.Set([])
@@ -394,7 +434,7 @@
""" Returns true if we have additional tags with additional
attributes that were not present in tag_attr_map
(returned from changed_tags_with_attrs) """
- for tags in map(BeautifulSoup, self.changed_tags()):
+ for tags in map(TheChosenSoup, self.changed_tags()):
for t in tags.findAll():
if t.name not in tag_attr_map:
return True
Added: torflow/trunk/NetworkScanners/snakeinspector.py
===================================================================
--- torflow/trunk/NetworkScanners/snakeinspector.py (rev 0)
+++ torflow/trunk/NetworkScanners/snakeinspector.py 2009-02-09 10:36:42 UTC (rev 18443)
@@ -0,0 +1,39 @@
+#!/usr/bin/python
+
+import dircache
+import operator
+import os
+import pickle
+import sys
+import time
+
+import sets
+from sets import Set
+
+import libsoat
+from libsoat import *
+
+sys.path.append("../")
+from TorCtl.TorUtil import *
+
+
+def main(argv):
+ dh = DataHandler()
+ # FIXME: Handle this better.. maybe explicit --file or --exit options?
+ # For now, I should be the only one runnin this so...
+ # XXX: Also want to filter on reason, false positive, and
+ # failure/inconclusive
+ if len(argv) == 1:
+ results = dh.getAll()
+ elif argv[1][0] == '$':
+ results = dh.filterByNode(dh.getAll(), argv[1])
+ else:
+ results = [dh.getResult(argv[1])]
+
+ for r in results:
+ if r.status == TEST_FAILURE:
+ print r
+ print "\n-----------------------------\n"
+
+if __name__ == "__main__":
+ main(sys.argv)
Property changes on: torflow/trunk/NetworkScanners/snakeinspector.py
___________________________________________________________________
Added: svn:executable
+ *
Modified: torflow/trunk/NetworkScanners/soat.py
===================================================================
--- torflow/trunk/NetworkScanners/soat.py 2009-02-09 09:56:43 UTC (rev 18442)
+++ torflow/trunk/NetworkScanners/soat.py 2009-02-09 10:36:42 UTC (rev 18443)
@@ -59,9 +59,9 @@
import OpenSSL
from OpenSSL import *
+
sys.path.append("./libs/")
-# XXX: Try to determine if we should be using MinimalSoup
-from BeautifulSoup.BeautifulSoup import BeautifulSoup, SoupStrainer, Tag
+from BeautifulSoup.BeautifulSoup import SoupStrainer, Tag
from SocksiPy import socks
import Pyssh.pyssh
@@ -286,7 +286,7 @@
links = SoupStrainer('a')
try:
- soup = BeautifulSoup(content, parseOnlyThese=links)
+ soup = TheChosenSoup(content, parseOnlyThese=links)
except Exception:
plog('ERROR', 'Soup-scraping of http://'+host+search_path+" failed")
traceback.print_exc()
@@ -875,17 +875,17 @@
elements = SoupStrainer(lambda name, attrs: name in tags_to_check or
len(Set(map(lambda a: a[0], attrs)).intersection(Set(attrs_to_check))) > 0)
- orig_soup = self._recursive_strain(BeautifulSoup(orig_html.decode('ascii',
+ orig_soup = self._recursive_strain(TheChosenSoup(orig_html.decode('ascii',
'ignore'), parseOnlyThese=elements))
- tor_soup = self._recursive_strain(BeautifulSoup(tor_html.decode('ascii',
+ tor_soup = self._recursive_strain(TheChosenSoup(tor_html.decode('ascii',
'ignore'), parseOnlyThese=elements))
# Also find recursive urls
recurse_elements = SoupStrainer(lambda name, attrs:
name in tags_to_recurse and
len(Set(map(lambda a: a[0], attrs)).intersection(Set(attrs_to_recurse))) > 0)
- self._add_recursive_targets(BeautifulSoup(tor_html.decode('ascii',
+ self._add_recursive_targets(TheChosenSoup(tor_html.decode('ascii',
'ignore'), recurse_elements), address)
# compare the content
@@ -909,7 +909,7 @@
return TEST_INCONCLUSIVE
- new_soup = self._recursive_strain(BeautifulSoup(content_new,
+ new_soup = self._recursive_strain(TheChosenSoup(content_new,
parseOnlyThese=elements))
# compare the new and old content
# if they match, means the node has been changing the content
@@ -1034,6 +1034,16 @@
# return the cert
return c.get_peer_certificate()
+ def get_resolved_ip(self, hostname):
+ mappings = self.mt.__control.get_address_mappings("cache")
+ ret = None
+ for m in mappings:
+ if m.from_name == hostname:
+ if ret:
+ plog("WARN", "Multiple maps for "+hostname)
+ ret = m.to_name
+ return ret
+
def check_openssl(self, address):
''' check whether an https connection to a given address is molested '''
plog('INFO', 'Conducting an ssl test with destination ' + address)
More information about the tor-commits
mailing list