- tor-commits - lists.torproject.org

[ooni-probe/master] Removed cerealizer because it sometimes causes the reporter to hang.
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit 20137d4b331ca8d3016708a8641e6f31f0b9ef72 Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun Nov 25 13:40:14 2012 +0000 Removed cerealizer because it sometimes causes the reporter to hang. --- nettests/bridge_reachability/tcpsyn.py | 22 ++-------------------- 1 files changed, 2 insertions(+), 20 deletions(-) diff --git a/nettests/bridge_reachability/tcpsyn.py b/nettests/bridge_reachability/tcpsyn.py index ed11c9d..54271bf 100644 --- a/nettests/bridge_reachability/tcpsyn.py +++ b/nettests/bridge_reachability/tcpsyn.py @@ -62,7 +62,6 @@ class TCPSynTest(nettest.NetTestCase): destinations = {} - @log.catcher def setUp(self, *a, **kw): """Configure commandline parameters for TCPSynTest.""" if self.localOptions: @@ -70,26 +69,12 @@ class TCPSynTest(nettest.NetTestCase): setattr(self, key, value) if not self.interface: try: - iface = net.getDefaultIface() + iface = log.catcher(net.getDefaultIface()) except net.IfaceError, ie: - log.msg("Could not find a working network interface!") - except Exception, ex: - log.exception(ex) + log.warn("Could not find a working network interface!") else: log.msg("Using system default interface: %s" % iface) self.interface = iface - if self.cerealize: - if True: - raise NotImplemented("need handler for type(dictproxy)...") - else: - from Cerealize import cerealizer - self.cheerios = Cerealize.cerealizer() - mind = ['scapy.layers.inet.IP', - 'scapy.base_classes.Packet_metaclass', - 'scapy.plist.SndRcvList'] - for spoon in mind: - __import__(spoon) - self.cheerios.register(spoon) def addToDestinations(self, addr, port): dst, dport = net.checkIPandPort(addr, port) @@ -161,9 +146,6 @@ class TCPSynTest(nettest.NetTestCase): if self.hexdump: request_data.update('hexdump', q.hexdump()) response_data.update('hexdump', r.hexdump()) - if self.cerealize: - pass - for dest, data in self.destinations.items(): if data['dst'] == response_data['src']: if not 'reachable' in data:

1 0

[ooni-probe/master] Still working (amend me).
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit eaecafea34c0acc1eb52bc0ade09796e45a335bf Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Nov 23 05:49:20 2012 +0000 Still working (amend me). --- nettests/bridge_reachability/tcpsyn.py | 191 +++++++++++++++----------------- 1 files changed, 89 insertions(+), 102 deletions(-) diff --git a/nettests/bridge_reachability/tcpsyn.py b/nettests/bridge_reachability/tcpsyn.py index e261a83..ed11c9d 100644 --- a/nettests/bridge_reachability/tcpsyn.py +++ b/nettests/bridge_reachability/tcpsyn.py @@ -62,11 +62,11 @@ class TCPSynTest(nettest.NetTestCase): destinations = {} + @log.catcher def setUp(self, *a, **kw): """Configure commandline parameters for TCPSynTest.""" if self.localOptions: for key, value in self.localOptions.items(): - log.debug("setting self.%s = %s" % (key, value)) setattr(self, key, value) if not self.interface: try: @@ -78,7 +78,6 @@ class TCPSynTest(nettest.NetTestCase): else: log.msg("Using system default interface: %s" % iface) self.interface = iface - if self.cerealize: if True: raise NotImplemented("need handler for type(dictproxy)...") @@ -97,111 +96,99 @@ class TCPSynTest(nettest.NetTestCase): if not dst in self.destinations.keys(): self.destinations[dst] = {'dst': dst, 'dport': dport} else: - ## XXX implement multiple port or portrange options - log.msg("Multiple port scanning not yet implemented.") + log.debug("XXX Multiple port scanning not yet implemented.") return (dst, dport) def inputProcessor(self, input_file=None): """ - Pull the IPs and PORTs from the input file, and place them in a dict - for storing test results as they arrive. + Pull the IPs and PORTs from the commandline options first, and then + from the input file, and place them in a dict for storing test results + as they arrive. """ - try: - ## get the commandline input, if there is one: - if self.localOptions['dst'] is not None \ - and self.localOptions['port'] is not None: - log.debug("processing commandline destination input") - yield self.addToDestinations(self.localOptions['dst'], - self.localOptions['port']) - ## get the inputs from inputFile: - if input_file and os.path.isfile(input_file): - log.debug("processing input file %s" % input_file) - with open(input_file) as f: - for line in f.readlines(): - if line.startswith('#'): - continue - one = line.strip() - raw_ip, raw_port = one.rsplit(':', 1) ## XXX not ipv6 safe! - yield self.addToDestinations(raw_ip, raw_port) - except Exception, ex: - log.exception(ex) - + if self.localOptions['dst'] is not None \ + and self.localOptions['port'] is not None: + log.debug("processing commandline destination input") + yield self.addToDestinations(self.localOptions['dst'], + self.localOptions['port']) + if input_file and os.path.isfile(input_file): + log.debug("processing input file %s" % input_file) + with open(input_file) as f: + for line in f.readlines(): + if line.startswith('#'): + continue + one = line.strip() + raw_ip, raw_port = one.rsplit(':', 1) ## XXX not ipv6 safe! + yield self.addToDestinations(raw_ip, raw_port) + + @staticmethod + def build_packets(addr, port, flags=None, count=3): + """Construct a list of packets to send out.""" + packets = [] + for x in xrange(count): + packets.append( IP(dst=addr)/TCP(dport=port, flags=flags) ) + return packets + + @log.catcher def test_tcp_syn(self): """Send the list of SYN packets.""" - try: - def build_packets(addr, port): - """Construct a list of packets to send out.""" - packets = [] - for x in xrange(self.count): - pkt = IP(dst=addr)/TCP(dport=port, flags="S") - packets.append(pkt) - return packets - - def process_packets(packet_list): - """xxx""" - results, unanswered = packet_list - - if self.pdf: - pdf_name = self.name +'_'+ timestamp() - try: - results.pdfdump(pdf_name) - except Exception, ex: - log.exception(ex) - else: - log.msg("Visual packet conversation saved to %s.pdf" - % pdf_name) - - for (q, r) in results: - request_data = {'summary': q.summary(), - 'command': q.command(), - 'hash': q.hashret(), - 'display': q.display(), - 'sent_time': q.time} - response_data = {'summary': r.summary(), - 'command': r.command(), - 'hash': r.hashret(), - 'src': r['IP'].src, - 'flags': r['IP'].flags, - 'display': r.display(), - 'recv_time': r.time, - 'delay': r.time - q.time} - if self.hexdump: - request_data.update('hexdump', q.hexdump()) - response_data.update('hexdump', r.hexdump()) - if self.cerealize: - pass - result_data = (request_data, response_data) - - for dest, data in self.destinations.items(): - if data['dst'] == response_data['src']: - if not 'received_response' in data: - if self.hexdump: - log.msg("%s" % request.hexdump()) - log.msg("%s" % response.hexdump()) - else: - log.msg("\n %s\n ==> %s" % (q.summary(), - r.summary())) - data['result'] = [result_data, ] - data['received_response'] = True - data['reachable'] = True - else: - data['result'].append(result_data) - - def process_unanswered(unanswer): - """Callback function to process unanswered packets.""" - #log.debug("%s" % str(unanswer)) - return unanswer - (addr, port) = self.input - packets = build_packets(addr, port) - - results = [] - - d = txscapy.sr(packets, iface=self.interface) - d.addCallbacks(process_packets, log.exception) - d.addCallbacks(process_unanswered, log.exception) - self.report['destinations'] = self.destinations - return d - - except Exception, e: - log.exception(e) + def process_packets(packet_list): + """xxx""" + results, unanswered = packet_list + + if self.pdf: + pdf_name = self.name +'_'+ timestamp() + try: + results.pdfdump(pdf_name) + except Exception, ex: + log.exception(ex) + else: + log.msg("Visual packet conversation saved to %s.pdf" + % pdf_name) + + for (q, r) in results: + request_data = {'dst': q.dst, + 'dport': q.dport, + 'summary': q.summary(), + 'command': q.command(), + 'sent_time': q.time} + response_data = {'summary': r.summary(), + 'command': r.command(), + 'src': r['IP'].src, + 'flags': r['IP'].flags, + 'recv_time': r.time, + 'delay': r.time - q.time} + if self.hexdump: + request_data.update('hexdump', q.hexdump()) + response_data.update('hexdump', r.hexdump()) + if self.cerealize: + pass + + for dest, data in self.destinations.items(): + if data['dst'] == response_data['src']: + if not 'reachable' in data: + if self.hexdump: + log.msg("%s\n%s" % (q.hexdump(), r.hexdump())) + else: + log.msg(" Received response:\n%s ==> %s" + % (q.mysummary(), r.mysummary())) + data.update( {'reachable': True, + 'request': request_data, + 'response': response_data} ) + return unanswered + + def process_unanswered(unanswered): + """Callback function to process unanswered packets.""" + if unanswered is not None and len(unanswered) > 0: + log.msg("Waiting on responses from\n%s" % + '\n'.join([unans.summary() for unans in unanswered])) + log.msg("Writing response packet information to report...") + self.report = (self.destinations) + return self.destinations + + (addr, port) = self.input + packets = self.build_packets(addr, port, "S", self.count) + d = txscapy.sr(packets, iface=self.interface) + d.addCallbacks(process_packets, log.exception) + d.addCallbacks(process_unanswered, log.exception) + return d

1 0

[ooni-probe/master] Trying out cerealizer for passing objects.
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit f6f8aae1a78d2086103718eaf41dd80f663aaef1 Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Nov 23 05:49:20 2012 +0000 Trying out cerealizer for passing objects. --- nettests/bridge_reachability/tcpsyn.py | 191 +++++++++++++++----------------- 1 files changed, 89 insertions(+), 102 deletions(-) diff --git a/nettests/bridge_reachability/tcpsyn.py b/nettests/bridge_reachability/tcpsyn.py index e261a83..ed11c9d 100644 --- a/nettests/bridge_reachability/tcpsyn.py +++ b/nettests/bridge_reachability/tcpsyn.py @@ -62,11 +62,11 @@ class TCPSynTest(nettest.NetTestCase): destinations = {} + @log.catcher def setUp(self, *a, **kw): """Configure commandline parameters for TCPSynTest.""" if self.localOptions: for key, value in self.localOptions.items(): - log.debug("setting self.%s = %s" % (key, value)) setattr(self, key, value) if not self.interface: try: @@ -78,7 +78,6 @@ class TCPSynTest(nettest.NetTestCase): else: log.msg("Using system default interface: %s" % iface) self.interface = iface - if self.cerealize: if True: raise NotImplemented("need handler for type(dictproxy)...") @@ -97,111 +96,99 @@ class TCPSynTest(nettest.NetTestCase): if not dst in self.destinations.keys(): self.destinations[dst] = {'dst': dst, 'dport': dport} else: - ## XXX implement multiple port or portrange options - log.msg("Multiple port scanning not yet implemented.") + log.debug("XXX Multiple port scanning not yet implemented.") return (dst, dport) def inputProcessor(self, input_file=None): """ - Pull the IPs and PORTs from the input file, and place them in a dict - for storing test results as they arrive. + Pull the IPs and PORTs from the commandline options first, and then + from the input file, and place them in a dict for storing test results + as they arrive. """ - try: - ## get the commandline input, if there is one: - if self.localOptions['dst'] is not None \ - and self.localOptions['port'] is not None: - log.debug("processing commandline destination input") - yield self.addToDestinations(self.localOptions['dst'], - self.localOptions['port']) - ## get the inputs from inputFile: - if input_file and os.path.isfile(input_file): - log.debug("processing input file %s" % input_file) - with open(input_file) as f: - for line in f.readlines(): - if line.startswith('#'): - continue - one = line.strip() - raw_ip, raw_port = one.rsplit(':', 1) ## XXX not ipv6 safe! - yield self.addToDestinations(raw_ip, raw_port) - except Exception, ex: - log.exception(ex) - + if self.localOptions['dst'] is not None \ + and self.localOptions['port'] is not None: + log.debug("processing commandline destination input") + yield self.addToDestinations(self.localOptions['dst'], + self.localOptions['port']) + if input_file and os.path.isfile(input_file): + log.debug("processing input file %s" % input_file) + with open(input_file) as f: + for line in f.readlines(): + if line.startswith('#'): + continue + one = line.strip() + raw_ip, raw_port = one.rsplit(':', 1) ## XXX not ipv6 safe! + yield self.addToDestinations(raw_ip, raw_port) + + @staticmethod + def build_packets(addr, port, flags=None, count=3): + """Construct a list of packets to send out.""" + packets = [] + for x in xrange(count): + packets.append( IP(dst=addr)/TCP(dport=port, flags=flags) ) + return packets + + @log.catcher def test_tcp_syn(self): """Send the list of SYN packets.""" - try: - def build_packets(addr, port): - """Construct a list of packets to send out.""" - packets = [] - for x in xrange(self.count): - pkt = IP(dst=addr)/TCP(dport=port, flags="S") - packets.append(pkt) - return packets - - def process_packets(packet_list): - """xxx""" - results, unanswered = packet_list - - if self.pdf: - pdf_name = self.name +'_'+ timestamp() - try: - results.pdfdump(pdf_name) - except Exception, ex: - log.exception(ex) - else: - log.msg("Visual packet conversation saved to %s.pdf" - % pdf_name) - - for (q, r) in results: - request_data = {'summary': q.summary(), - 'command': q.command(), - 'hash': q.hashret(), - 'display': q.display(), - 'sent_time': q.time} - response_data = {'summary': r.summary(), - 'command': r.command(), - 'hash': r.hashret(), - 'src': r['IP'].src, - 'flags': r['IP'].flags, - 'display': r.display(), - 'recv_time': r.time, - 'delay': r.time - q.time} - if self.hexdump: - request_data.update('hexdump', q.hexdump()) - response_data.update('hexdump', r.hexdump()) - if self.cerealize: - pass - result_data = (request_data, response_data) - - for dest, data in self.destinations.items(): - if data['dst'] == response_data['src']: - if not 'received_response' in data: - if self.hexdump: - log.msg("%s" % request.hexdump()) - log.msg("%s" % response.hexdump()) - else: - log.msg("\n %s\n ==> %s" % (q.summary(), - r.summary())) - data['result'] = [result_data, ] - data['received_response'] = True - data['reachable'] = True - else: - data['result'].append(result_data) - - def process_unanswered(unanswer): - """Callback function to process unanswered packets.""" - #log.debug("%s" % str(unanswer)) - return unanswer - (addr, port) = self.input - packets = build_packets(addr, port) - - results = [] - - d = txscapy.sr(packets, iface=self.interface) - d.addCallbacks(process_packets, log.exception) - d.addCallbacks(process_unanswered, log.exception) - self.report['destinations'] = self.destinations - return d - - except Exception, e: - log.exception(e) + def process_packets(packet_list): + """xxx""" + results, unanswered = packet_list + + if self.pdf: + pdf_name = self.name +'_'+ timestamp() + try: + results.pdfdump(pdf_name) + except Exception, ex: + log.exception(ex) + else: + log.msg("Visual packet conversation saved to %s.pdf" + % pdf_name) + + for (q, r) in results: + request_data = {'dst': q.dst, + 'dport': q.dport, + 'summary': q.summary(), + 'command': q.command(), + 'sent_time': q.time} + response_data = {'summary': r.summary(), + 'command': r.command(), + 'src': r['IP'].src, + 'flags': r['IP'].flags, + 'recv_time': r.time, + 'delay': r.time - q.time} + if self.hexdump: + request_data.update('hexdump', q.hexdump()) + response_data.update('hexdump', r.hexdump()) + if self.cerealize: + pass + + for dest, data in self.destinations.items(): + if data['dst'] == response_data['src']: + if not 'reachable' in data: + if self.hexdump: + log.msg("%s\n%s" % (q.hexdump(), r.hexdump())) + else: + log.msg(" Received response:\n%s ==> %s" + % (q.mysummary(), r.mysummary())) + data.update( {'reachable': True, + 'request': request_data, + 'response': response_data} ) + return unanswered + + def process_unanswered(unanswered): + """Callback function to process unanswered packets.""" + if unanswered is not None and len(unanswered) > 0: + log.msg("Waiting on responses from\n%s" % + '\n'.join([unans.summary() for unans in unanswered])) + log.msg("Writing response packet information to report...") + self.report = (self.destinations) + return self.destinations + + (addr, port) = self.input + packets = self.build_packets(addr, port, "S", self.count) + d = txscapy.sr(packets, iface=self.interface) + d.addCallbacks(process_packets, log.exception) + d.addCallbacks(process_unanswered, log.exception) + return d

1 0

[ooni-probe/master] Added better reporting of statistics and safe packet information to TCPflags
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit 1fa46bcb6d65a11b3209c07d8e14507d43303f1a Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Nov 27 14:07:52 2012 +0000 Added better reporting of statistics and safe packet information to TCPflags test, which is a combined and condensed version of TCPsyn and TCPfin tests. * Tested pcap support. * Updated documentation. --- nettests/bridge_reachability/tcpsyn.py | 199 ++++++++++++++++++++------------ 1 files changed, 124 insertions(+), 75 deletions(-) diff --git a/nettests/bridge_reachability/tcpsyn.py b/nettests/bridge_reachability/tcpsyn.py index 54271bf..555fdef 100644 --- a/nettests/bridge_reachability/tcpsyn.py +++ b/nettests/bridge_reachability/tcpsyn.py @@ -16,12 +16,13 @@ import os import sys -from ipaddr import IPAddress -from twisted.python import usage -from twisted.internet import reactor, defer, address -from ooni import nettest -from ooni.utils import net, log -from ooni.utils.otime import timestamp +from ipaddr import IPAddress +from twisted.python import usage +from twisted.python.failure import Failure +from twisted.internet import reactor, defer, address +from ooni import nettest, config +from ooni.utils import net, log +from ooni.utils.otime import timestamp try: from scapy.all import TCP, IP @@ -30,8 +31,8 @@ except: log.msg("This test requires scapy, see www.secdev.org/projects/scapy") -class UsageOptions(usage.Options): - """Options for TCPSynTest.""" +class TCPFlagOptions(usage.Options): + """Options for TCPTest.""" optParameters = [ ['dst', 'd', None, 'Host IP to ping'], ['port', 'p', None, 'Host port'], @@ -43,7 +44,7 @@ class UsageOptions(usage.Options): ['cerealize', 'z', False, 'Cerealize scapy objects for further scripting']] -class TCPSynTest(nettest.NetTestCase): +class TCPFlagTest(nettest.NetTestCase): """ Sends only a TCP SYN packet to a host IP:PORT, and waits for either a SYN/ACK, a RST, or an ICMP error. @@ -51,37 +52,55 @@ class TCPSynTest(nettest.NetTestCase): TCPSynTest can take an input file containing one IP:Port pair per line, or the commandline switches --dst <IP> and --port <PORT> can be used. """ - name = 'TCP SYN' + name = 'TCP Flag' author = 'Isis Lovecruft <isis(a)torproject.org>' - description = 'A TCP SYN test to see if a host is reachable.' + description = 'A TCP SYN/ACK/FIN test to see if a host is reachable.' version = '0.0.1' requiresRoot = True - usageOptions = UsageOptions + usageOptions = TCPFlagOptions inputFile = ['file', 'f', None, 'File of list of IP:PORTs to ping'] - destinations = {} + #destinations = {} def setUp(self, *a, **kw): """Configure commandline parameters for TCPSynTest.""" + self.report = {} + if self.localOptions: for key, value in self.localOptions.items(): setattr(self, key, value) if not self.interface: try: - iface = log.catcher(net.getDefaultIface()) + iface = log.catch(net.getDefaultIface()) except net.IfaceError, ie: log.warn("Could not find a working network interface!") + log.fail(ie) else: log.msg("Using system default interface: %s" % iface) self.interface = iface + if config.debug: + defer.setDebugging(on) + + def addToDestinations(self, addr='0.0.0.0', port='443'): + """ + Validate and add an IP address and port to the dictionary of + destinations to send to. If the host's IP is already in the + destinations, then only add the port. - def addToDestinations(self, addr, port): + @param addr: A string representing an IPv4 or IPv6 address. + @param port: A string representing a port number. + @returns: A 2-tuple containing the address and port. + """ dst, dport = net.checkIPandPort(addr, port) - if not dst in self.destinations.keys(): - self.destinations[dst] = {'dst': dst, 'dport': dport} + #if not dst in self.destinations.keys(): + if not dst in self.report.keys(): + #self.destinations[dst] = {'dst': dst, 'dport': [dport]} + self.report[dst] = {'dst': dst, 'dport': [dport]} else: - log.debug("XXX Multiple port scanning not yet implemented.") + log.debug("Got additional port for destination.") + #self.destinations[dst]['dport'].append(dport) + self.report[dst]['dport'].append(dport) return (dst, dport) def inputProcessor(self, input_file=None): @@ -105,6 +124,12 @@ class TCPSynTest(nettest.NetTestCase): raw_ip, raw_port = one.rsplit(':', 1) ## XXX not ipv6 safe! yield self.addToDestinations(raw_ip, raw_port) + @log.catch + def createPDF(self, results): + pdfname = self.name + '_' + timestamp() + results.pdfdump(pdfname) + log.msg("Visual packet conversation saved to %s.pdf" % pdfname) + @staticmethod def build_packets(addr, port, flags=None, count=3): """Construct a list of packets to send out.""" @@ -113,64 +138,88 @@ class TCPSynTest(nettest.NetTestCase): packets.append( IP(dst=addr)/TCP(dport=port, flags=flags) ) return packets - @log.catcher - def test_tcp_syn(self): - """Send the list of SYN packets.""" - - def process_packets(packet_list): - """xxx""" - results, unanswered = packet_list - - if self.pdf: - pdf_name = self.name +'_'+ timestamp() - try: - results.pdfdump(pdf_name) - except Exception, ex: - log.exception(ex) - else: - log.msg("Visual packet conversation saved to %s.pdf" - % pdf_name) - - for (q, r) in results: - request_data = {'dst': q.dst, - 'dport': q.dport, - 'summary': q.summary(), - 'command': q.command(), - 'sent_time': q.time} - response_data = {'summary': r.summary(), - 'command': r.command(), - 'src': r['IP'].src, - 'flags': r['IP'].flags, - 'recv_time': r.time, - 'delay': r.time - q.time} - if self.hexdump: - request_data.update('hexdump', q.hexdump()) - response_data.update('hexdump', r.hexdump()) - for dest, data in self.destinations.items(): - if data['dst'] == response_data['src']: - if not 'reachable' in data: - if self.hexdump: - log.msg("%s\n%s" % (q.hexdump(), r.hexdump())) - else: - log.msg(" Received response:\n%s ==> %s" - % (q.mysummary(), r.mysummary())) - data.update( {'reachable': True, - 'request': request_data, - 'response': response_data} ) - return unanswered - - def process_unanswered(unanswered): - """Callback function to process unanswered packets.""" - if unanswered is not None and len(unanswered) > 0: - log.msg("Waiting on responses from\n%s" % - '\n'.join([unans.summary() for unans in unanswered])) - log.msg("Writing response packet information to report...") - self.report = (self.destinations) - return self.destinations + @staticmethod + def process_packets(packet_list): + """ + If the source address of packet in :param:packet_list matches one of our input + destinations, then extract some of the information from it to the test report. + + @param packet_list: + A :class:scapy.plist.PacketList + """ + results, unanswered = packet_list + + if self.pdf: + self.createPDF(results) + + for (q, r) in results: + request_data = {'dst': q.dst, + 'dport': q.dport, + 'summary': q.summary(), + 'command': q.command(), + 'sent_time': q.time} + response_data = {'src': r['IP'].src, + 'flags': r['IP'].flags, + 'summary': r.summary(), + 'command': r.command(), + 'recv_time': r.time, + 'delay': r.time - q.time} + if self.hexdump: + request_data.update('hexdump', q.hexdump()) + response_data.update('hexdump', r.hexdump()) + for dest, data in self.destinations.items(): + if data['dst'] == response_data['src']: + if not 'reachable' in data: + if self.hexdump: + log.msg("%s\n%s" % (q.hexdump(), r.hexdump())) + else: + log.msg(" Received response:\n%s ==> %s" + % (q.mysummary(), r.mysummary())) + data.update( {'reachable': True, + 'request': request_data, + 'response': response_data} ) + return unanswered + @staticmethod + def process_unanswered(unanswered): + """Callback function to process unanswered packets.""" + if unanswered is not None and len(unanswered) > 0: + log.msg("Waiting on responses from\n%s" % + '\n'.join( [unans.summary() for unans in unanswered] )) + log.msg("Writing response packet information to report...") + self.report = (self.destinations) + return self.destinations + + @log.catch + def tcp_flags(self, flags="S"): + """ + Generate, send, and listen for responses to, a list of TCP/IP packets + to an address and port pair taken from the current input, and a string + specifying the TCP flags to set. + + @param flags: + A string representing the TCP flags to be set, i.e. "SA" or "F". + Defaults to "S". + """ (addr, port) = self.input - packets = self.build_packets(addr, port, "S", self.count) + packets = self.build_packets(addr, port, str(flags), self.count) d = txscapy.sr(packets, iface=self.interface) - d.addCallbacks(process_packets, log.exception) - d.addCallbacks(process_unanswered, log.exception) + d.addCallbacks(self.process_packets, log.exception) + d.addCallbacks(self.process_unanswered, log.exception) return d + + def test_tcp_fin(self): + """Send a list of FIN packets to an address and port pair from inputs.""" + return self.tcp_flags("F") + + def test_tcp_syn(self): + """Send a list of SYN packets to an address and port pair from inputs.""" + return self.tcp_flags("S") + + def test_tcp_synack(self): + """Send a list of SYN/ACK packets to an address and port pair from inputs.""" + return self.tcp_flags("SA") + + def test_tcp_ack(self): + """Send a list of SYN packets to an address and port pair from inputs.""" + return self.tcp_flags("A")

1 0

[ooni-probe/master] Added warning suppression for scapy's logger to the beginning of oonicli.py.
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit a382177448f004a29a26b970be9e01bd799f1f0b Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Nov 27 14:12:33 2012 +0000 Added warning suppression for scapy's logger to the beginning of oonicli.py. --- ooni/oonicli.py | 10 +++++++--- 1 files changed, 7 insertions(+), 3 deletions(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 263494b..9c2b4e6 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -19,13 +19,17 @@ from twisted.application import app from twisted.python import usage, failure from twisted.python.util import spewer -from ooni import nettest, runner, reporter, config - -from ooni.inputunit import InputUnitFactory +## Get rid of the annoying "No route found for +## IPv6 destination warnings": +import logging as pylogging +pylogging.getLogger("scapy.runtime").setLevel(pylogging.ERROR) +from ooni import nettest, runner, reporter, config from ooni.utils import net from ooni.utils import checkForRoot, PermissionsError from ooni.utils import log +from ooni.inputunit import InputUnitFactory + class Options(usage.Options, app.ReactorSelectionMixin): synopsis = """%s [options] [path to test].py

1 0

[ooni-probe/master] Added log.warn() and log.fail() functions to ooni's logger, and fixed the
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit 037b4218e455b49240f1fe7a6826d03e60137765 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Nov 27 14:22:21 2012 +0000 Added log.warn() and log.fail() functions to ooni's logger, and fixed the deferred debugger in tcpsyn.setUp. --- nettests/bridge_reachability/tcpsyn.py | 4 ++-- ooni/utils/log.py | 12 +++++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/nettests/bridge_reachability/tcpsyn.py b/nettests/bridge_reachability/tcpsyn.py index 555fdef..dd7f494 100644 --- a/nettests/bridge_reachability/tcpsyn.py +++ b/nettests/bridge_reachability/tcpsyn.py @@ -79,8 +79,8 @@ class TCPFlagTest(nettest.NetTestCase): else: log.msg("Using system default interface: %s" % iface) self.interface = iface - if config.debug: - defer.setDebugging(on) + if config.advanced.debug: + defer.setDebugging('on') def addToDestinations(self, addr='0.0.0.0', port='443'): """ diff --git a/ooni/utils/log.py b/ooni/utils/log.py index c751571..3b2b92a 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -49,6 +49,9 @@ def msg(msg, *arg, **kw): def debug(msg, *arg, **kw): txlog.msg(msg, logLevel=logging.DEBUG, *arg, **kw) +def warn(msg, category=None, filename=None, lineno=None, *arg, **kw): + txlog.warnings.warn(msg, category, filename, lineno, *arg, **kw) + def err(msg, *arg, **kw): txlog.err("Error: " + str(msg), logLevel=logging.ERROR, *arg, **kw) @@ -60,7 +63,10 @@ def exception(msg): def exception(*msg): logging.exception(msg) -def catcher(func): +def fail(failure): + txlog.failure(failure) + +def catch(func): """ Quick wrapper to add around test methods for debugging purposes, catches the given Exception. Use like so: @@ -71,12 +77,12 @@ def catcher(func): raise Exception("catch me no matter what I am") foo("baz") """ - def _catcher(*args, **kwargs): + def _catch(*args, **kwargs): try: func(*args, **kwargs) except Exception, exc: exception(exc) - return _catcher + return _catch class LoggerFactory(object): """

1 0

[ooni-probe/master] Added connection timeout to geoIP requests.
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit 989910435a48a7df612807717fc56ece9ce52832 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Nov 27 14:16:53 2012 +0000 Added connection timeout to geoIP requests. --- ooni/reporter.py | 19 ++++++++++++------- ooni/utils/geodata.py | 16 ++++++++++++++-- ooniprobe.conf | 8 ++++++-- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/ooni/reporter.py b/ooni/reporter.py index 05ea94a..aec1e1b 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -19,16 +19,12 @@ from twisted.python.util import untilConcludes from twisted.trial import reporter from twisted.internet import defer, reactor +from ooni import config from ooni.templates.httpt import BodyReceiver, StringProducer from ooni.utils import otime, log, geodata - from ooni.utils.hacks import OSafeRepresenter, OSafeDumper -from ooni import config try: - ## Get rid of the annoying "No route found for - ## IPv6 destination warnings": - logging.getLogger("scapy.runtime").setLevel(logging.ERROR) from scapy.all import packet except: class FooClass: @@ -54,8 +50,17 @@ def getTestDetails(options): config.privacy.includeasn or \ config.privacy.includecountry or \ config.privacy.includecity: - log.msg("Running geo IP lookup via check.torproject.org") - client_ip = yield geodata.myIP() + log.msg("Running geoIP lookup via check.torproject.org") + if config.privacy.checktimeout is not None and \ + isinstance(config.privacy.checktimeout, int): + my_ip_timeout = config.privacy.checktimeout + else: + log.debug( + "reporter.getTestDetails(): bad config.privacy.checktimeout %s" + % str(config.privacy.checktimeout) + ) + my_ip_timeout = 15 + client_ip = yield geodata.myIP(connectTimeout=my_ip_timeout) client_location = geodata.IPToLocation(client_ip) else: client_ip = "127.0.0.1" diff --git a/ooni/utils/geodata.py b/ooni/utils/geodata.py index 447d2eb..29fa59d 100644 --- a/ooni/utils/geodata.py +++ b/ooni/utils/geodata.py @@ -23,10 +23,22 @@ except ImportError: log.err("Unable to import pygeoip. We will not be able to run geo IP related measurements") @defer.inlineCallbacks -def myIP(): +def myIP(*args, **kwargs): + """ + Get the client's public IP address. + + @param: + All parameters are passed to the instantiation of + :class:`twisted.client.Agent`. + @ivar connectTimeout: + Seconds to timeout after for HTTP requests. + """ target_site = 'https://check.torproject.org/' regexp = "Your IP address appears to be: <b>(.+?)<\/b>" - myAgent = Agent(reactor) + if 'connectTimeout' in kwargs: + myAgent = Agent(reactor, connectTimeout=connectTimeout) + else: + myAgent = Agent(reactor, *args, **kwargs) result = yield myAgent.request('GET', target_site) diff --git a/ooniprobe.conf b/ooniprobe.conf index 191d809..e3a7198 100644 --- a/ooniprobe.conf +++ b/ooniprobe.conf @@ -6,8 +6,12 @@ basic: # Where OONIProbe should be writing it's log file logfile: /tmp/ooniprobe.log privacy: - # Should we include the IP address of the probe in the report? + # Should we include the client's IP address of the probe in the report? + # (Client IP is obtained by an HTTP request to https://check.torproject.org) includeip: false + # If we're including the client's IP address, how long should we wait for + # the connection to complete before timing out? + checktimeout: 15 # Should we include the ASN of the probe in the report? includeasn: false # Should we include the ASN of the probe in the report? @@ -24,4 +28,4 @@ advanced: geoip_data_dir: /home/x/code/networking/ooni-probe/data/ debug: true threadpool_size: 10 - + # Connection timeout in seconds for \ No newline at end of file

1 0

[ooni-probe/master] Rearranged the logic of ensuring that the reactor is shutdown, and removed a
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit c18718548bf0551e707fe4d2871bffe794b0e876 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Nov 28 18:38:31 2012 +0000 Rearranged the logic of ensuring that the reactor is shutdown, and removed a couple extra blank lines at the end of /ooni/utils/__init__.py and /ooni/utils/hacks.py. --- ooni/runner.py | 4 ++-- ooni/utils/__init__.py | 2 -- ooni/utils/hacks.py | 1 - 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/ooni/runner.py b/ooni/runner.py index 41f3178..8d750ae 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -253,10 +253,10 @@ def runTestCases(test_cases, options, input_list, oreporter) except Exception, ex: # XXX we probably want to add a log.warn() at some point - log.msg("Problem in running test") + log.warn("Problem in running test") log.exception(ex) + oreporter.allDone() if reactor.running: reactor.stop() - oreporter.allDone() diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index 74e1fc2..be5b38a 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -88,5 +88,3 @@ def randomStr(length, num=True): if num: chars += string.digits return ''.join(random.choice(chars) for x in range(length)) - - diff --git a/ooni/utils/hacks.py b/ooni/utils/hacks.py index 8912bbb..9121982 100644 --- a/ooni/utils/hacks.py +++ b/ooni/utils/hacks.py @@ -107,4 +107,3 @@ class OSafeDumper(Emitter, Serializer, OSafeRepresenter, Resolver): OSafeRepresenter.__init__(self, default_style=default_style, default_flow_style=default_flow_style) Resolver.__init__(self) -

1 0

[ooni-probe/master] Started implementing a way for a test to abort itself,
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit 3750cb43e72eb7de8b01fc824bac3121fa9678f0 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Nov 28 18:34:06 2012 +0000 Started implementing a way for a test to abort itself, nettest.NetTestCase.__test_done__(). --- ooni/nettest.py | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index a2f98c4..a868ac4 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -166,3 +166,8 @@ class NetTestCase(object): def __repr__(self): return "<%s inputs=%s>" % (self.__class__, self.inputs) + def __test_done__(self): + up = inspect.stack() + parent = up[1] + # XXX call oreporter.allDone() from parent stack frame + raise NotImplemented

1 0

[ooni-probe/master] Merge branch 'tcpsyn-report' into tcpsyn-port
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit 4d15117d2cf23af454d29494fdc42a084f658af0 Merge: eaecafe 214c626 Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Nov 30 12:08:43 2012 +0000 Merge branch 'tcpsyn-report' into tcpsyn-port Conflicts: nettests/bridge_reachability/tcpsyn.py nettests/bridge_reachability/tcpsyn.py | 220 ++++++++++++++++++-------------- ooni/nettest.py | 5 + ooni/oonicli.py | 10 +- ooni/reporter.py | 19 ++- ooni/runner.py | 4 +- ooni/utils/__init__.py | 2 - ooni/utils/geodata.py | 16 ++- ooni/utils/hacks.py | 1 - ooni/utils/log.py | 14 ++- ooniprobe.conf | 8 +- 10 files changed, 183 insertions(+), 116 deletions(-) diff --cc nettests/bridge_reachability/tcpsyn.py index ed11c9d,dd7f494..39c882b --- a/nettests/bridge_reachability/tcpsyn.py +++ b/nettests/bridge_reachability/tcpsyn.py @@@ -57,14 -58,15 +58,16 @@@ class TCPFlagTest(nettest.NetTestCase) version = '0.0.1' requiresRoot = True - usageOptions = UsageOptions + usageOptions = TCPFlagOptions inputFile = ['file', 'f', None, 'File of list of IP:PORTs to ping'] - destinations = {} + #destinations = {} + @log.catcher def setUp(self, *a, **kw): """Configure commandline parameters for TCPSynTest.""" + self.report = {} + if self.localOptions: for key, value in self.localOptions.items(): setattr(self, key, value)

1 0