commit 514bb499295b6f1884cd1a5cb083522719acfe45 Author: Arturo Filastò art@fuffa.org Date: Sun Nov 11 20:31:28 2012 +0100

Start rewriting daphn3 --- nettests/core/daphn3.py | 89 ++++++---------- ooni/kit/daphn3.py | 261 +++++------------------------------------------ 2 files changed, 58 insertions(+), 292 deletions(-)

diff --git a/nettests/core/daphn3.py b/nettests/core/daphn3.py index 8d7dbbd..ed65963 100644 --- a/nettests/core/daphn3.py +++ b/nettests/core/daphn3.py @@ -1,6 +1,5 @@ from twisted.python import usage - -from twisted.internet import protocol, endpoints +from twisted.internet import protocol, endpoints, reactor

from ooni.kit import daphn3 from ooni.utils import log @@ -45,55 +44,45 @@ class Daphn3ClientFactory(protocol.ClientFactory): print "Connection Lost."

class daphn3Args(usage.Options): - optParameters = [['pcap', 'f', None, - 'PCAP to read for generating the YAML output'], - - ['output', 'o', 'daphn3.yaml', - 'What file should be written'], - - ['yaml', 'y', None, - 'The input file to the test'], - + optParameters = [ ['host', 'h', None, 'Target Hostname'], - ['port', 'p', None, 'Target port number'], - ['resume', 'r', 0, 'Resume at this index']] + ['port', 'p', None, 'Target port number']] + + optFlags = [['pcap', 'c', 'Specify that the input file is a pcap file'], + ['yaml', 'y', 'Specify that the input file is a YAML file (default)']]

class daphn3Test(nettest.NetTestCase):

- shortName = "daphn3" - description = "daphn3" - requirements = None - options = daphn3Args - blocking = False + name = "Daphn3" + usageOptions = daphn3Args + inputFile = ['file', 'f', None, + 'Specify the pcap or YAML file to be used as input to the test']

- local_options = None + requiredOptions = ['file']

steps = None

- def initialize(self): - if not self.local_options: - self.end() - return + def inputProcessor(self, fp): + if self.localOptions['pcap']: + self.steps = daphn3.read_pcap(self.localOptions['pcap']) + else: + self.steps = daphn3.read_yaml(self.localOptions['yaml'])

+ def setUp(self): self.factory = Daphn3ClientFactory() self.factory.test = self

- if self.local_options['pcap']: - self.tool = True - - elif self.local_options['yaml']: - self.steps = daphn3.read_yaml(self.local_options['yaml']) - + if self.localOptions['pcap']: + self.steps = daphn3.read_pcap(self.localOptions['pcap']) + elif self.localOptions['yaml']: + self.steps = daphn3.read_yaml(self.localOptions['yaml']) else: - log.msg("Not enough inputs specified to the test") - self.end() + raise usage.UsageError("You must specify either --pcap or --yaml")

- def runTool(self): - import yaml - pcap = daphn3.read_pcap(self.local_options['pcap']) - f = open(self.local_options['output'], 'w') - f.write(yaml.dump(pcap)) - f.close() + mutations = 0 + for x in self.steps: + mutations += len(x['data']) + return {'mutation': range(mutations)}

def control(self, exp_res, args): try: @@ -106,34 +95,20 @@ class daphn3Test(nettest.NetTestCase): 'value': mutation}

def _failure(self, *argc, **kw): - self.result['censored'] = True - self.result['error'] = ('Failed in connecting', (argc, kw)) - self.end() + self.report['censored'] = True + self.report['mutation'] = + self.report['error'] = ('Failed in connecting', (argc, kw))

- def experiment(self, args): + def test_daphn3(self): log.msg("Doing mutation %s" % args['mutation']) self.factory.steps = self.steps + host = self.local_options['host'] port = int(self.local_options['port']) log.msg("Connecting to %s:%s" % (host, port))

- if self.ended: - return - - endpoint = endpoints.TCP4ClientEndpoint(self.reactor, host, port) + endpoint = endpoints.TCP4ClientEndpoint(reactor, host, port) d = endpoint.connect(self.factory) d.addErrback(self._failure) return d

- def load_assets(self): - if not self.local_options: - return {} - if not self.steps: - print "Error: No assets!" - self.end() - return {} - mutations = 0 - for x in self.steps: - mutations += len(x['data']) - return {'mutation': range(mutations)} - diff --git a/ooni/kit/daphn3.py b/ooni/kit/daphn3.py index 37c94c7..2660af0 100644 --- a/ooni/kit/daphn3.py +++ b/ooni/kit/daphn3.py @@ -7,7 +7,6 @@ from twisted.internet.error import ConnectionDone from scapy.all import IP, Raw, rdpcap

from ooni.utils import log -from ooni.plugoo import reports

def read_pcap(filename): """ @@ -28,9 +27,9 @@ def read_pcap(filename): """ pcap assumptions:

- pcap only contains packets exchanged between a Tor client and a Tor server. - (This assumption makes sure that there are only two IP addresses in the - pcap file) + pcap only contains packets exchanged between a Tor client and a Tor + server. (This assumption makes sure that there are only two IP addresses + in the pcap file)

The first packet of the pcap is sent from the client to the server. (This assumption is used to get the IP address of the client.) @@ -39,8 +38,10 @@ def read_pcap(filename): establishment/teardown packets should be filtered out (no SYN/SYN+ACK) """

- """Minimally validate the pcap and also find out what's the client - and server IP addresses.""" + """ + Minimally validate the pcap and also find out what's the client + and server IP addresses. + """ for packet in packets: if checking_first_packet: client_ip_addr = packet[IP].src @@ -72,240 +73,30 @@ def read_yaml(filename): f.close() return obj

-class Mutator: - idx = 0 - step = 0 - - waiting = False - waiting_step = 0 - - def __init__(self, steps): - """ - @param steps: array of dicts for the steps that must be gone over by - the mutator. Looks like this: - [{"sender": "client", "data": "\xde\xad\xbe\xef"}, - {"sender": "server", "data": "\xde\xad\xbe\xef"}] - """ - self.steps = steps - - def _mutate(self, data, idx): - """ - Mutate the idx bytes by increasing it's value by one - - @param data: the data to be mutated. - - @param idx: what byte should be mutated. - """ - print "idx: %s, data: %s" % (idx, data) - ret = data[:idx] - ret += chr(ord(data[idx]) + 1) - ret += data[idx+1:] - return ret - - def state(self): - """ - Return the current mutation state. As in what bytes are being mutated. - - Returns a dict containg the packet index and the step number. - """ - print "[Mutator.state()] Giving out my internal state." - current_state = {'idx': self.idx, 'step': self.step} - return current_state - - def next(self): - """ - Increases by one the mutation state. - - ex. (* is the mutation state, i.e. the byte to be mutated) - before [___*] [____] - step1 step2 - after [____] [*___] - - Should be called every time you need to proceed onto the next mutation. - It changes the internal state of the mutator to that of the next - mutatation. - - returns True if another mutation is available. - returns False if all the possible mutations have been done. - """ - if (self.step) == len(self.steps): - # Hack to stop once we have gone through all the steps - print "[Mutator.next()] I believe I have gone over all steps" - print " Stopping!" - self.waiting = True - return False - - self.idx += 1 - current_idx = self.idx - current_step = self.step - current_data = self.steps[current_step]['data'] - - if 0: - print "current_step: %s" % current_step - print "current_idx: %s" % current_idx - print "current_data: %s" % current_data - print "steps: %s" % len(self.steps) - print "waiting_step: %s" % self.waiting_step - - data_to_receive = len(self.steps[current_step]['data']) - - if self.waiting and self.waiting_step == data_to_receive: - print "[Mutator.next()] I am no longer waiting" - log.debug("I am no longer waiting.") - self.waiting = False - self.waiting_step = 0 - self.idx = 0 - - elif self.waiting: - print "[Mutator.next()] Waiting some more." - log.debug("Waiting some more.") - self.waiting_step += 1 - - elif current_idx >= len(current_data): - print "[Mutator.next()] Entering waiting mode." - log.debug("Entering waiting mode.") - self.step += 1 - self.idx = 0 - self.waiting = True - - log.debug("current index %s" % current_idx) - log.debug("current data %s" % len(current_data)) - return True - - def get(self, step): - """ - Returns the current packet to be sent to the wire. - If no mutation is necessary it will return the plain data. - Should be called when you are interested in obtaining the data to be - sent for the selected state. - - @param step: the current step you want the mutation for - - returns the mutated packet for the specified step. - """ - if step != self.step or self.waiting: - log.debug("[Mutator.get()] I am not going to do anything :)") - return self.steps[step]['data'] - - data = self.steps[step]['data'] - #print "Mutating %s with idx %s" % (data, self.idx) - return self._mutate(data, self.idx) +class NoInputSpecified(Exception): + pass

class Daphn3Protocol(protocol.Protocol): - """ - This implements the Daphn3 protocol for the server side. - It gets instanced once for every client that connects to the oonib. - For every instance of protocol there is only 1 mutation. - Once the last step is reached the connection is closed on the serverside. - """ - steps = [] - mutator = None - - current_state = None - - role = 'client' - state = 0 - total_states = len(steps) - 1 - received_data = 0 - to_receive_data = 0 - report = reports.Report('daphn3', 'daphn3.yamlooni') - - test = None - - def next_state(self): - """ - This is called once I have completed one step of the protocol and need - to proceed to the next step. - """ - if not self.mutator: - print "[Daphn3Protocol.next_state] No mutator. There is no point to stay on this earth." - self.transport.loseConnection() - return - - if self.role is self.steps[self.state]['sender']: - print "[Daphn3Protocol.next_state] I am a sender" - data = self.mutator.get(self.state) - self.transport.write(data) - self.to_receive_data = 0 - + def __init__(self, yaml_file=None, pcap_file=None, role="client"): + if yaml_file: + self.packets = read_yaml(yaml_file) + elif pcap_file: + self.packets = read_pcap(pcap_file) else: - print "[Daphn3Protocol.next_state] I am a receiver" - self.to_receive_data = len(self.steps[self.state]['data']) - - self.state += 1 - self.received_data = 0 + raise NoInputSpecified

- def dataReceived(self, data): - """ - This is called every time some data is received. I transition to the - next step once the amount of data that I expect to receive is received. - - @param data: the data that has been sent by the client. - """ - if not self.mutator: - print "I don't have a mutator. My life means nothing." - self.transport.loseConnection() - return - - if len(self.steps) == self.state: - self.transport.loseConnection() - return - - self.received_data += len(data) - if self.received_data >= self.to_receive_data: - print "Moving to next state %s" % self.state - self.next_state() - - def censorship_detected(self, report): - """ - I have detected the possible presence of censorship we need to write a - report on it. - - @param report: a dict containing the report to be written. Must contain - the keys 'reason', 'proto_state' and 'mutator_state'. - The reason is the reason for which the connection was - closed. The proto_state is the current state of the - protocol instance and mutator_state is what was being - mutated. - """ - print "The connection was closed because of %s" % report['reason'] - print "State %s, Mutator %s" % (report['proto_state'], - report['mutator_state']) - if self.test: - self.test.result['censored'] = True - self.test.result['state'] = report - self.mutator.next() + self.role = role + # We use this index to keep track of where we are in the state machine + self.current_step = 0

- def connectionLost(self, reason): - """ - The connection was closed. This may be because of a legittimate reason - or it may be because of a censorship event. - """ - if not self.mutator: - print "Terminated because of little interest in life." - return - report = {'reason': reason, 'proto_state': self.state, - 'trigger': None, 'mutator_state': self.current_state} + # 0 indicates we are waiting to receive data, while 1 indicates we are + # sending data + self.current_state = 0 + self.current_data_received = 0

- if self.state < self.total_states: - report['trigger'] = 'did not finish state walk' - self.censorship_detected(report) - - else: - print "I have reached the end of the state machine" - print "Censorship fingerprint bruteforced!" - if self.test: - print "In the test thing" - self.test.result['censored'] = False - self.test.result['state'] = report - self.test.result['state_walk_finished'] = True - self.test.report(self.test.result) - return - - if reason.check(ConnectionDone): - print "Connection closed cleanly" - else: - report['trigger'] = 'unclean connection closure' - self.censorship_detected(report) + def dataReceived(self, data): + self.current_data_received += len(data) + expected_data_in_this_state = len(self.packets[self.current_step][self.role]) + if len(self.current_data_received)