[tor-commits] [ooni-probe/master] Changed the ooni/lib structure to use gitsubmodules so that OONI users do not
isis at torproject.org
isis at torproject.org
Thu Sep 13 13:04:15 UTC 2012
commit 78f768540a0912619b566e1f75e518edcb29b7ff
Author: Isis Lovecruft <isis at patternsinthevoid.net>
Date: Mon Aug 27 04:50:54 2012 +0000
Changed the ooni/lib structure to use gitsubmodules so that OONI users do not
accidentally git-clone broken or incompatible upstreams. Each submodule has
it's own .git with both hellais' and my forks configured, though each
submodule is also configured with a special HEAD state so that unneeded files
are not downloaded from upstream (this seems better than git-/hg- cloning an
unknown version of a repo and copying out the files we do want).
I still need to update the ooni-probe imports to reflect the slight directory
structure change. Also, I rm'd the Makefile. Now, new OONI users should do a
"git submodule init && git submodule update"...I'll probably write a script
for it. I will add documentation for what to do with committed changes in
submodule directories, so that OONI devs can make sure that users get the
right versions.
---
.gitmodules | 9 ++
ooni/lib/Makefile | 30 ----
ooni/lib/txscapy | 1 +
ooni/lib/txscapy.py | 363 -------------------------------------------------
ooni/lib/txtorcon | 1 +
ooni/lib/txtraceroute | 1 +
6 files changed, 12 insertions(+), 393 deletions(-)
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 0000000..07a4c56
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,9 @@
+[submodule "ooni/lib/txscapy"]
+ path = ooni/lib/txscapy
+ url = git://github.com/hellais/txscapy.git
+[submodule "ooni/lib/txtraceroute"]
+ path = ooni/lib/txtraceroute
+ url = git://github.com/hellais/txtraceroute.git
+[submodule "ooni/lib/txtorcon"]
+ path = ooni/lib/txtorcon
+ url = git://github.com/meejah/txtorcon.git
diff --git a/ooni/lib/Makefile b/ooni/lib/Makefile
deleted file mode 100644
index 3b0c922..0000000
--- a/ooni/lib/Makefile
+++ /dev/null
@@ -1,30 +0,0 @@
-all: txtorcon txtraceroute
-
-txtraceroute:
- echo "Processing dependency txtraceroute..."
- git clone https://github.com/hellais/txtraceroute.git txtraceroute.git
- mv txtraceroute.git/txtraceroute.py txtraceroute.py
- rm -rf txtraceroute.git
-
-txtorcon:
- echo "Processing dependency txtorcon..."
- git clone https://github.com/meejah/txtorcon.git txtorcon.git
- mv txtorcon.git/txtorcon txtorcon
- rm -rf txtorcon.git
-
-clean:
- rm -rf txtorcon
- rm -rf txtraceroute.py
-
-#txscapy:
-# echo "Processing dependency txscapy"
-# git clone https://github.com/hellais/txscapy.git txscapy.git
-# mv txscapy.git/txscapy.py txscapy.py
-# rm -rf txscapy.git
-
-#rfc3339:
-# echo "Processing RFC3339 dependency"
-# hg clone https://bitbucket.org/henry/rfc3339 rfc3339
-# mv rfc3339/rfc3339.py rfc3339.py
-# rm -rf rfc3339
-
diff --git a/ooni/lib/txscapy b/ooni/lib/txscapy
new file mode 160000
index 0000000..19fb281
--- /dev/null
+++ b/ooni/lib/txscapy
@@ -0,0 +1 @@
+Subproject commit 19fb28150c0b31f16a1ae2bc0aadeb6fd3c259bf
diff --git a/ooni/lib/txscapy.py b/ooni/lib/txscapy.py
deleted file mode 100644
index 8996f75..0000000
--- a/ooni/lib/txscapy.py
+++ /dev/null
@@ -1,363 +0,0 @@
-# -*- coding:utf8 -*-
-"""
- txscapy
- ******
- (c) 2012 Arturo Filastò
- a twisted wrapper for scapys send and receive functions.
-
- This software has been written to be part of OONI, the Open Observatory of
- Network Interference. More information on that here: http://ooni.nu/
-
-"""
-
-import struct
-import socket
-import os
-import sys
-import time
-
-from twisted.internet import protocol, base, fdesc, error, defer
-from twisted.internet import reactor, threads
-from twisted.python import log
-from zope.interface import implements
-
-from scapy.all import Gen
-from scapy.all import SetGen
-
-LINUX=sys.platform.startswith("linux")
-OPENBSD=sys.platform.startswith("openbsd")
-FREEBSD=sys.platform.startswith("freebsd")
-NETBSD=sys.platform.startswith("netbsd")
-DARWIN=sys.platform.startswith("darwin")
-SOLARIS=sys.platform.startswith("sunos")
-WINDOWS=sys.platform.startswith("win32")
-
-from scapy.all import RawPcapWriter, MTU, BasePacketList, conf
-class PcapWriter(RawPcapWriter):
- def __init__(self, filename, linktype=None, gz=False, endianness="",
- append=False, sync=False):
- RawPcapWriter.__init__(self, filename, linktype=None, gz=False,
- endianness="", append=False, sync=False)
- fdesc.setNonBlocking(self.f)
-
- def _write_header(self, pkt):
- if self.linktype == None:
- if type(pkt) is list or type(pkt) is tuple or isinstance(pkt, BasePacketList):
- pkt = pkt[0]
- try:
- self.linktype = conf.l2types[pkt.__class__]
- except KeyError:
- self.linktype = 1
- RawPcapWriter._write_header(self, pkt)
-
- def _write_packet(self, packet):
- sec = int(packet.time)
- usec = int(round((packet.time-sec)*1000000))
- s = str(packet)
- caplen = len(s)
- RawPcapWriter._write_packet(self, s, sec, usec, caplen, caplen)
-
-class ScapySocket(object):
- MTU = 1500
- def __init__(self, filter=None, iface=None, nofilter=None):
- from scapy.all import conf
- self.ssocket = conf.L3socket(filter=filter, iface=iface, nofilter=nofilter)
-
- def fileno(self):
- return self.ssocket.ins.fileno()
-
- def send(self, data):
- return self.ssocket.send(data)
-
- def recv(self):
- if FREEBSD or DARWIN:
- return self.ssocket.nonblock_recv()
- else:
- return self.ssocket.recv(self.MTU)
-
-class Scapy(object):
- """
- A twisted based wrapper for scapy send and receive functionality.
-
- It sends packets inside of a threadpool and receives packets using the
- libdnet receive non blocking file descriptor.
- """
- min = 2
- max = 6
- debug = True
- write_only_answers = False
- pcapwriter = None
- recv = False
- timeout = None
-
- def __init__(self, pkts=None, maxPacketSize=8192, reactor=None, filter=None,
- iface=None, nofilter=None, pcapfile=None, timeout=None, *args, **kw):
-
- self.timeout = timeout
- self.last_answer = None
- if self.debug:
- log.startLogging(sys.stdout)
-
- self.maxPacketSize = maxPacketSize
- if not reactor:
- from twisted.internet import reactor
-
- self._reactor = reactor
-
- if pkts:
- self._buildPacketQueues(pkts)
- self._buildSocket()
-
- self.cthreads = 0
- self.mthreads = 80
-
- self.running = False
- self.done = False
- self.finished = False
-
- import thread
- from twisted.python import threadpool
- self.threadID = thread.get_ident
- self.threadpool = threadpool.ThreadPool(self.min, self.max)
- self.startID = self._reactor.callWhenRunning(self._start)
-
- self.deferred = defer.Deferred()
-
- if pcapfile:
- self.pcapwriter = PcapWriter(pcapfile)
-
- def _buildSocket(self, filter=None, iface=None, nofilter=None):
- self.socket = ScapySocket(filter, iface, nofilter)
- if self.recv:
- self._reactor.addReader(self)
-
- def _buildPacketQueues(self, pkts):
- """
- Converts the list of packets to a Scapy generator and sets up all the
- necessary attributes for understanding if all the needed responses have
- been received.
- """
- if not isinstance(pkts, Gen):
- self.pkts = SetGen(pkts)
-
- self.outqueue = [p for p in pkts]
-
- self.total_count = len(self.outqueue)
- self.answer_count = 0
- self.out_count = 0
-
- self.hsent = {}
- for p in self.outqueue:
- h = p.hashret()
- if h in self.hsent:
- self.hsent[h].append(p)
- else:
- self.hsent[h] = [p]
-
-
- def gotAnswer(self, answer, question):
- """
- Got a packet that has been identified as an answer to one of the sent
- out packets.
-
- If the answer count matches the sent count the finish callback is
- fired.
-
- @param answer: the packet received on the wire.
-
- @param question: the sent packet that matches that response.
-
- """
-
- if self.pcapwriter and self.write_only_answers:
- self.pcapwriter.write(question)
- self.pcapwriter.write(answer)
- self.answer_count += 1
- self.last_answer = time.time()
-
- if self.answer_count >= self.total_count:
- print "Got all the answers I need"
- self.deferred.callback(None)
-
-
- def processAnswer(self, pkt, hlst):
- """
- Checks if the potential answer is in fact an answer to one of the
- matched sent packets. Uses the scapy .answers() function to verify
- this.
-
- @param pkt: The packet to be tested if is the answer to a sent packet.
-
- @param hlst: a list of packets that match the hash for an answer to
- pkt.
- """
- for i in range(len(hlst)):
- if pkt.answers(hlst[i]):
- self.gotAnswer(pkt, hlst[i])
-
- def fileno(self):
- """
- Returns a fileno for use by twisteds Reader.
- """
- return self.socket.fileno()
-
- def processPacket(self, pkt):
- """
- Override this method to process your packets.
-
- @param pkt: the packet that has been received.
- """
- #pkt.show()
-
-
- def doRead(self):
- """
- There is something to be read on the wire. Do all the processing on the
- received packet.
- """
- if self.timeout and (time.time() - self.last_answer) > self.timeout and\
- not self.outqueue:
- print "Timing out.."
- self.deferred.callback(None)
-
- pkt = self.socket.recv()
- if self.pcapwriter and not self.write_only_answers:
- self.pcapwriter.write(pkt)
- self.processPacket(pkt)
-
- h = pkt.hashret()
- if h in self.hsent:
- hlst = self.hsent[h]
- self.processAnswer(pkt, hlst)
-
- def logPrefix(self):
- """
- The prefix to be prepended in logging.
- """
- return "txScapy"
-
- def _start(self):
- """
- Start the twisted thread pool.
- """
- self.startID = None
- return self.start()
-
- def start(self):
- """
- Actually start the thread pool.
- """
- if not self.running:
- self.threadpool.start()
- self.shutdownID = self._reactor.addSystemEventTrigger(
- 'during', 'shutdown', self.finalClose)
- self.running = True
-
- def sendPkt(self, pkt):
- """
- Send a packet to the wire.
-
- @param pkt: The packet to be sent.
- """
- self.socket.send(pkt)
-
- def sr(self, pkts, filter=None, iface=None, nofilter=0, timeout=None, *args, **kw):
- """
- Wraps the scapy sr function.
-
- @param nofilter: put 1 to avoid use of bpf filters
-
- @param retry: if positive, how many times to resend unanswered packets
- if negative, how many times to retry when no more packets are
- answered (XXX to be implemented)
-
- @param timeout: how much time to wait after the last packet has
- been sent (XXX to be implemented)
-
- @param multi: whether to accept multiple answers for the same
- stimulus (XXX to be implemented)
-
- @param filter: provide a BPF filter
- @param iface: listen answers only on the given interface
- """
- self.timeout = timeout
- self.recv = True
- self._sendrcv(pkts, filter=filter, iface=iface, nofilter=nofilter)
-
- def send(self, pkts, filter=None, iface=None, nofilter=0, *args, **kw):
- """
- Wraps the scapy send function. Its the same as send and receive, except
- it does not receive. Who would have ever guessed? ;)
-
- @param nofilter: put 1 to avoid use of bpf filters
-
- @param retry: if positive, how many times to resend unanswered packets
- if negative, how many times to retry when no more packets are
- answered (XXX to be implemented)
-
- @param timeout: how much time to wait after the last packet has
- been sent (XXX to be implemented)
-
- @param multi: whether to accept multiple answers for the same
- stimulus (XXX to be implemented)
-
- @param filter: provide a BPF filter
- @param iface: listen answers only on the given interface
- """
- self.recv = False
- self._sendrcv(pkts, filter=filter, iface=iface, nofilter=nofilter)
-
- def _sendrcv(self, pkts, filter=None, iface=None, nofilter=0):
- self._buildSocket(filter, iface, nofilter)
- self._buildPacketQueues(pkts)
- if not self.last_answer:
- self.last_answer = time.time()
-
- def sent(cb):
- if self.cthreads < self.mthreads and not self.done:
- pkt = None
- try:
- pkt = self.outqueue.pop()
- except:
- self.done = True
- if not self.recv:
- self.deferred.callback(None)
- return
- d = threads.deferToThreadPool(reactor, self.threadpool,
- self.sendPkt, pkt)
- d.addCallback(sent)
- return d
-
- for x in range(self.mthreads):
- try:
- pkt = self.outqueue.pop()
- except:
- self.done = True
- return
- if self.cthreads >= self.mthreads and self.done:
- return
- d = threads.deferToThreadPool(reactor, self.threadpool,
- self.sendPkt, pkt)
- d.addCallback(sent)
- return d
-
- def connectionLost(self, why):
- pass
-
- def finalClose(self):
- """
- Clean all the thread related stuff up.
- """
- self.shutdownID = None
- self.threadpool.stop()
- self.running = False
-
-def txsr(*args, **kw):
- tr = Scapy(*args, **kw)
- tr.sr(*args, **kw)
- return tr.deferred
-
-def txsend(*arg, **kw):
- tr = Scapy(*arg, **kw)
- tr.send(*arg, **kw)
- return tr.deferred
diff --git a/ooni/lib/txtorcon b/ooni/lib/txtorcon
new file mode 160000
index 0000000..ac791b4
--- /dev/null
+++ b/ooni/lib/txtorcon
@@ -0,0 +1 @@
+Subproject commit ac791b4ddfc29a2dd29a7044f34c3394df38653d
diff --git a/ooni/lib/txtraceroute b/ooni/lib/txtraceroute
new file mode 160000
index 0000000..067a260
--- /dev/null
+++ b/ooni/lib/txtraceroute
@@ -0,0 +1 @@
+Subproject commit 067a2609390e77bf9187275638cf8786efef7e13
More information about the tor-commits
mailing list