commit 78f768540a0912619b566e1f75e518edcb29b7ff Author: Isis Lovecruft isis@patternsinthevoid.net Date: Mon Aug 27 04:50:54 2012 +0000
Changed the ooni/lib structure to use gitsubmodules so that OONI users do not accidentally git-clone broken or incompatible upstreams. Each submodule has it's own .git with both hellais' and my forks configured, though each submodule is also configured with a special HEAD state so that unneeded files are not downloaded from upstream (this seems better than git-/hg- cloning an unknown version of a repo and copying out the files we do want).
I still need to update the ooni-probe imports to reflect the slight directory structure change. Also, I rm'd the Makefile. Now, new OONI users should do a "git submodule init && git submodule update"...I'll probably write a script for it. I will add documentation for what to do with committed changes in submodule directories, so that OONI devs can make sure that users get the right versions. --- .gitmodules | 9 ++ ooni/lib/Makefile | 30 ---- ooni/lib/txscapy | 1 + ooni/lib/txscapy.py | 363 ------------------------------------------------- ooni/lib/txtorcon | 1 + ooni/lib/txtraceroute | 1 + 6 files changed, 12 insertions(+), 393 deletions(-)
diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..07a4c56 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,9 @@ +[submodule "ooni/lib/txscapy"] + path = ooni/lib/txscapy + url = git://github.com/hellais/txscapy.git +[submodule "ooni/lib/txtraceroute"] + path = ooni/lib/txtraceroute + url = git://github.com/hellais/txtraceroute.git +[submodule "ooni/lib/txtorcon"] + path = ooni/lib/txtorcon + url = git://github.com/meejah/txtorcon.git diff --git a/ooni/lib/Makefile b/ooni/lib/Makefile deleted file mode 100644 index 3b0c922..0000000 --- a/ooni/lib/Makefile +++ /dev/null @@ -1,30 +0,0 @@ -all: txtorcon txtraceroute - -txtraceroute: - echo "Processing dependency txtraceroute..." - git clone https://github.com/hellais/txtraceroute.git txtraceroute.git - mv txtraceroute.git/txtraceroute.py txtraceroute.py - rm -rf txtraceroute.git - -txtorcon: - echo "Processing dependency txtorcon..." - git clone https://github.com/meejah/txtorcon.git txtorcon.git - mv txtorcon.git/txtorcon txtorcon - rm -rf txtorcon.git - -clean: - rm -rf txtorcon - rm -rf txtraceroute.py - -#txscapy: -# echo "Processing dependency txscapy" -# git clone https://github.com/hellais/txscapy.git txscapy.git -# mv txscapy.git/txscapy.py txscapy.py -# rm -rf txscapy.git - -#rfc3339: -# echo "Processing RFC3339 dependency" -# hg clone https://bitbucket.org/henry/rfc3339 rfc3339 -# mv rfc3339/rfc3339.py rfc3339.py -# rm -rf rfc3339 - diff --git a/ooni/lib/txscapy b/ooni/lib/txscapy new file mode 160000 index 0000000..19fb281 --- /dev/null +++ b/ooni/lib/txscapy @@ -0,0 +1 @@ +Subproject commit 19fb28150c0b31f16a1ae2bc0aadeb6fd3c259bf diff --git a/ooni/lib/txscapy.py b/ooni/lib/txscapy.py deleted file mode 100644 index 8996f75..0000000 --- a/ooni/lib/txscapy.py +++ /dev/null @@ -1,363 +0,0 @@ -# -*- coding:utf8 -*- -""" - txscapy - ****** - (c) 2012 Arturo Filastò - a twisted wrapper for scapys send and receive functions. - - This software has been written to be part of OONI, the Open Observatory of - Network Interference. More information on that here: http://ooni.nu/ - -""" - -import struct -import socket -import os -import sys -import time - -from twisted.internet import protocol, base, fdesc, error, defer -from twisted.internet import reactor, threads -from twisted.python import log -from zope.interface import implements - -from scapy.all import Gen -from scapy.all import SetGen - -LINUX=sys.platform.startswith("linux") -OPENBSD=sys.platform.startswith("openbsd") -FREEBSD=sys.platform.startswith("freebsd") -NETBSD=sys.platform.startswith("netbsd") -DARWIN=sys.platform.startswith("darwin") -SOLARIS=sys.platform.startswith("sunos") -WINDOWS=sys.platform.startswith("win32") - -from scapy.all import RawPcapWriter, MTU, BasePacketList, conf -class PcapWriter(RawPcapWriter): - def __init__(self, filename, linktype=None, gz=False, endianness="", - append=False, sync=False): - RawPcapWriter.__init__(self, filename, linktype=None, gz=False, - endianness="", append=False, sync=False) - fdesc.setNonBlocking(self.f) - - def _write_header(self, pkt): - if self.linktype == None: - if type(pkt) is list or type(pkt) is tuple or isinstance(pkt, BasePacketList): - pkt = pkt[0] - try: - self.linktype = conf.l2types[pkt.__class__] - except KeyError: - self.linktype = 1 - RawPcapWriter._write_header(self, pkt) - - def _write_packet(self, packet): - sec = int(packet.time) - usec = int(round((packet.time-sec)*1000000)) - s = str(packet) - caplen = len(s) - RawPcapWriter._write_packet(self, s, sec, usec, caplen, caplen) - -class ScapySocket(object): - MTU = 1500 - def __init__(self, filter=None, iface=None, nofilter=None): - from scapy.all import conf - self.ssocket = conf.L3socket(filter=filter, iface=iface, nofilter=nofilter) - - def fileno(self): - return self.ssocket.ins.fileno() - - def send(self, data): - return self.ssocket.send(data) - - def recv(self): - if FREEBSD or DARWIN: - return self.ssocket.nonblock_recv() - else: - return self.ssocket.recv(self.MTU) - -class Scapy(object): - """ - A twisted based wrapper for scapy send and receive functionality. - - It sends packets inside of a threadpool and receives packets using the - libdnet receive non blocking file descriptor. - """ - min = 2 - max = 6 - debug = True - write_only_answers = False - pcapwriter = None - recv = False - timeout = None - - def __init__(self, pkts=None, maxPacketSize=8192, reactor=None, filter=None, - iface=None, nofilter=None, pcapfile=None, timeout=None, *args, **kw): - - self.timeout = timeout - self.last_answer = None - if self.debug: - log.startLogging(sys.stdout) - - self.maxPacketSize = maxPacketSize - if not reactor: - from twisted.internet import reactor - - self._reactor = reactor - - if pkts: - self._buildPacketQueues(pkts) - self._buildSocket() - - self.cthreads = 0 - self.mthreads = 80 - - self.running = False - self.done = False - self.finished = False - - import thread - from twisted.python import threadpool - self.threadID = thread.get_ident - self.threadpool = threadpool.ThreadPool(self.min, self.max) - self.startID = self._reactor.callWhenRunning(self._start) - - self.deferred = defer.Deferred() - - if pcapfile: - self.pcapwriter = PcapWriter(pcapfile) - - def _buildSocket(self, filter=None, iface=None, nofilter=None): - self.socket = ScapySocket(filter, iface, nofilter) - if self.recv: - self._reactor.addReader(self) - - def _buildPacketQueues(self, pkts): - """ - Converts the list of packets to a Scapy generator and sets up all the - necessary attributes for understanding if all the needed responses have - been received. - """ - if not isinstance(pkts, Gen): - self.pkts = SetGen(pkts) - - self.outqueue = [p for p in pkts] - - self.total_count = len(self.outqueue) - self.answer_count = 0 - self.out_count = 0 - - self.hsent = {} - for p in self.outqueue: - h = p.hashret() - if h in self.hsent: - self.hsent[h].append(p) - else: - self.hsent[h] = [p] - - - def gotAnswer(self, answer, question): - """ - Got a packet that has been identified as an answer to one of the sent - out packets. - - If the answer count matches the sent count the finish callback is - fired. - - @param answer: the packet received on the wire. - - @param question: the sent packet that matches that response. - - """ - - if self.pcapwriter and self.write_only_answers: - self.pcapwriter.write(question) - self.pcapwriter.write(answer) - self.answer_count += 1 - self.last_answer = time.time() - - if self.answer_count >= self.total_count: - print "Got all the answers I need" - self.deferred.callback(None) - - - def processAnswer(self, pkt, hlst): - """ - Checks if the potential answer is in fact an answer to one of the - matched sent packets. Uses the scapy .answers() function to verify - this. - - @param pkt: The packet to be tested if is the answer to a sent packet. - - @param hlst: a list of packets that match the hash for an answer to - pkt. - """ - for i in range(len(hlst)): - if pkt.answers(hlst[i]): - self.gotAnswer(pkt, hlst[i]) - - def fileno(self): - """ - Returns a fileno for use by twisteds Reader. - """ - return self.socket.fileno() - - def processPacket(self, pkt): - """ - Override this method to process your packets. - - @param pkt: the packet that has been received. - """ - #pkt.show() - - - def doRead(self): - """ - There is something to be read on the wire. Do all the processing on the - received packet. - """ - if self.timeout and (time.time() - self.last_answer) > self.timeout and\ - not self.outqueue: - print "Timing out.." - self.deferred.callback(None) - - pkt = self.socket.recv() - if self.pcapwriter and not self.write_only_answers: - self.pcapwriter.write(pkt) - self.processPacket(pkt) - - h = pkt.hashret() - if h in self.hsent: - hlst = self.hsent[h] - self.processAnswer(pkt, hlst) - - def logPrefix(self): - """ - The prefix to be prepended in logging. - """ - return "txScapy" - - def _start(self): - """ - Start the twisted thread pool. - """ - self.startID = None - return self.start() - - def start(self): - """ - Actually start the thread pool. - """ - if not self.running: - self.threadpool.start() - self.shutdownID = self._reactor.addSystemEventTrigger( - 'during', 'shutdown', self.finalClose) - self.running = True - - def sendPkt(self, pkt): - """ - Send a packet to the wire. - - @param pkt: The packet to be sent. - """ - self.socket.send(pkt) - - def sr(self, pkts, filter=None, iface=None, nofilter=0, timeout=None, *args, **kw): - """ - Wraps the scapy sr function. - - @param nofilter: put 1 to avoid use of bpf filters - - @param retry: if positive, how many times to resend unanswered packets - if negative, how many times to retry when no more packets are - answered (XXX to be implemented) - - @param timeout: how much time to wait after the last packet has - been sent (XXX to be implemented) - - @param multi: whether to accept multiple answers for the same - stimulus (XXX to be implemented) - - @param filter: provide a BPF filter - @param iface: listen answers only on the given interface - """ - self.timeout = timeout - self.recv = True - self._sendrcv(pkts, filter=filter, iface=iface, nofilter=nofilter) - - def send(self, pkts, filter=None, iface=None, nofilter=0, *args, **kw): - """ - Wraps the scapy send function. Its the same as send and receive, except - it does not receive. Who would have ever guessed? ;) - - @param nofilter: put 1 to avoid use of bpf filters - - @param retry: if positive, how many times to resend unanswered packets - if negative, how many times to retry when no more packets are - answered (XXX to be implemented) - - @param timeout: how much time to wait after the last packet has - been sent (XXX to be implemented) - - @param multi: whether to accept multiple answers for the same - stimulus (XXX to be implemented) - - @param filter: provide a BPF filter - @param iface: listen answers only on the given interface - """ - self.recv = False - self._sendrcv(pkts, filter=filter, iface=iface, nofilter=nofilter) - - def _sendrcv(self, pkts, filter=None, iface=None, nofilter=0): - self._buildSocket(filter, iface, nofilter) - self._buildPacketQueues(pkts) - if not self.last_answer: - self.last_answer = time.time() - - def sent(cb): - if self.cthreads < self.mthreads and not self.done: - pkt = None - try: - pkt = self.outqueue.pop() - except: - self.done = True - if not self.recv: - self.deferred.callback(None) - return - d = threads.deferToThreadPool(reactor, self.threadpool, - self.sendPkt, pkt) - d.addCallback(sent) - return d - - for x in range(self.mthreads): - try: - pkt = self.outqueue.pop() - except: - self.done = True - return - if self.cthreads >= self.mthreads and self.done: - return - d = threads.deferToThreadPool(reactor, self.threadpool, - self.sendPkt, pkt) - d.addCallback(sent) - return d - - def connectionLost(self, why): - pass - - def finalClose(self): - """ - Clean all the thread related stuff up. - """ - self.shutdownID = None - self.threadpool.stop() - self.running = False - -def txsr(*args, **kw): - tr = Scapy(*args, **kw) - tr.sr(*args, **kw) - return tr.deferred - -def txsend(*arg, **kw): - tr = Scapy(*arg, **kw) - tr.send(*arg, **kw) - return tr.deferred diff --git a/ooni/lib/txtorcon b/ooni/lib/txtorcon new file mode 160000 index 0000000..ac791b4 --- /dev/null +++ b/ooni/lib/txtorcon @@ -0,0 +1 @@ +Subproject commit ac791b4ddfc29a2dd29a7044f34c3394df38653d diff --git a/ooni/lib/txtraceroute b/ooni/lib/txtraceroute new file mode 160000 index 0000000..067a260 --- /dev/null +++ b/ooni/lib/txtraceroute @@ -0,0 +1 @@ +Subproject commit 067a2609390e77bf9187275638cf8786efef7e13