- tor-commits - lists.torproject.org

[pytorctl/master] Handling for connections to non-control sockets
by mikeperry＠torproject.org 18 Jun '11

18 Jun '11

commit e76418e507319887626b9d3921bdc0ac44b28e76 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jun 13 09:50:13 2011 -0700 Handling for connections to non-control sockets When we attempt to initiate a control connection to a non-control port (for instance tor's socks port instead)... - the socket shutdown issues an error since it's not connected yet - checking the authentication type fails with a TorCtlClosed rather than error reply For more information see bug 2580. --- TorCtl.py | 5 +++-- TorUtil.py | 7 ++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/TorCtl.py b/TorCtl.py index 056a65c..44a612b 100755 --- a/TorCtl.py +++ b/TorCtl.py @@ -590,8 +590,9 @@ class Connection: # check PROTOCOLINFO for authentication type try: authInfo = self.sendAndRecv("PROTOCOLINFO\r\n")[1][1] - except ErrorReply, exc: - raise IOError("Unable to query PROTOCOLINFO for the authentication type: %s" % exc) + except Exception, exc: + excMsg = ": %s" % exc if exc.message else "" + raise IOError("Unable to query PROTOCOLINFO for the authentication type%s" % excMsg) authType, cookiePath = None, None if authInfo.startswith("AUTH METHODS=NULL"): diff --git a/TorUtil.py b/TorUtil.py index 29c44bf..fcbef70 100644 --- a/TorUtil.py +++ b/TorUtil.py @@ -231,7 +231,12 @@ class BufSock: def close(self): self._isDone = True - self._s.shutdown(socket.SHUT_RDWR) + + # if we haven't yet established a connection then this raises an error + # socket.error: [Errno 107] Transport endpoint is not connected + try: self._s.shutdown(socket.SHUT_RDWR) + except socket.error: pass + self._s.close() # SocketServer.TCPServer is nuts..

1 0

[pytorctl/master] Unblocking waits for a response when we error
by mikeperry＠torproject.org 18 Jun '11

18 Jun '11

commit 9de33409bb240d8fe50e135dfb895536dc6a45bd Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jun 13 09:30:42 2011 -0700 Unblocking waits for a response when we error When the signal sent by _sendImpl causes the control connection to close we block indefinitely waiting for a response. For more information see bug 1329. --- TorCtl.py | 12 +++++++++++- TorUtil.py | 1 + 2 files changed, 12 insertions(+), 1 deletions(-) diff --git a/TorCtl.py b/TorCtl.py index 2758033..056a65c 100755 --- a/TorCtl.py +++ b/TorCtl.py @@ -681,8 +681,18 @@ class Connection: while 1: try: isEvent, reply = self._read_reply() - except TorCtlClosed: + except TorCtlClosed, exc: plog("NOTICE", "Tor closed control connection. Exiting event thread.") + + # notify anything blocking on a response of the error, for details see: + # https://trac.torproject.org/projects/tor/ticket/1329 + try: + self._closedEx = exc + cb = self._queue.get(timeout=0) + if cb != "CLOSE": + cb("EXCEPTION") + except Queue.Empty: pass + return except Exception,e: if not self._closed: diff --git a/TorUtil.py b/TorUtil.py index 60d2b2f..29c44bf 100644 --- a/TorUtil.py +++ b/TorUtil.py @@ -208,6 +208,7 @@ class BufSock: s = self._s.recv(128) except socket.timeout: if not self._isDone: continue + except: s = None if not s: return None # XXX: This really does need an exception

1 0

[pytorctl/master] Option to get get_network_status as an iterator
by mikeperry＠torproject.org 18 Jun '11

18 Jun '11

commit 45d7753b7c947b141210017d1b987673c51d97c5 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Jun 15 19:27:12 2011 -0700 Option to get get_network_status as an iterator Setting the 'getIterator' argument drops the memory usage of calling get_network_status by 71% (from 3.5 MB to 1 MB). This is still higher than what I was expecting from a generator, though certainly much better. Unfortunately this didn't have an impact on the ConsensusTracker. The memory usage from its constructor dwarfs anything else I've looked at (18.8 MB) and didn't drop like I'd expect when consensus_only was false. :( --- TorCtl.py | 54 +++++++++++++++++++++++++++++++++++------------------- 1 files changed, 35 insertions(+), 19 deletions(-) diff --git a/TorCtl.py b/TorCtl.py index 44a612b..8680ee6 100755 --- a/TorCtl.py +++ b/TorCtl.py @@ -1040,10 +1040,21 @@ class Connection: TorCtl.NetworkStatus instances.""" return parse_ns_body(self.sendAndRecv("GETINFO dir/status-vote/current/consensus\r\n")[0][2]) - def get_network_status(self, who="all"): + def get_network_status(self, who="all", getIterator=False): """Get the entire network status list. Returns a list of - TorCtl.NetworkStatus instances.""" - return parse_ns_body(self.sendAndRecv("GETINFO ns/"+who+"\r\n")[0][2]) + TorCtl.NetworkStatus instances. + + Be aware that by default this reads the whole consensus into memory at + once which can be fairly sizable (as of writing 3.5 MB), and even if + freed it may remain allocated to the interpretor: + http://effbot.org/pyfaq/why-doesnt-python-release-the-memory-when-i-delete-… + + To avoid this use the iterator instead. + """ + + nsData = self.sendAndRecv("GETINFO ns/"+who+"\r\n")[0][2] + if getIterator: return ns_body_iter(nsData) + else: return parse_ns_body(nsData) def get_address_mappings(self, type="all"): # TODO: Also parse errors and GMTExpiry @@ -1232,21 +1243,26 @@ class Connection: def parse_ns_body(data): """Parse the body of an NS event or command into a list of NetworkStatus instances""" - if not data: return [] - nsgroups = re.compile(r"^r ", re.M).split(data) - nsgroups.pop(0) - nslist = [] - for nsline in nsgroups: - m = re.search(r"^s((?:[ ]\S*)+)", nsline, re.M) - flags = m.groups() - flags = flags[0].strip().split(" ") - m = re.match(r"(\S+)\s(\S+)\s(\S+)\s(\S+\s\S+)\s(\S+)\s(\d+)\s(\d+)", nsline) - w = re.search(r"^w Bandwidth=(\d+)", nsline, re.M) - if w: - nslist.append(NetworkStatus(*(m.groups()+(flags,)+(int(w.group(1))*1000,)))) - else: - nslist.append(NetworkStatus(*(m.groups() + (flags,)))) - return nslist + return list(ns_body_iter(data)) + +def ns_body_iter(data): + """Generator for NetworkStatus instances of an NS event""" + if data: + nsgroups = re.compile(r"^r ", re.M).split(data) + nsgroups.pop(0) + + while nsgroups: + nsline = nsgroups.pop(0) + m = re.search(r"^s((?:[ ]\S*)+)", nsline, re.M) + flags = m.groups() + flags = flags[0].strip().split(" ") + m = re.match(r"(\S+)\s(\S+)\s(\S+)\s(\S+\s\S+)\s(\S+)\s(\d+)\s(\d+)", nsline) + w = re.search(r"^w Bandwidth=(\d+)", nsline, re.M) + + if w: + yield NetworkStatus(*(m.groups()+(flags,)+(int(w.group(1))*1000,))) + else: + yield NetworkStatus(*(m.groups() + (flags,))) class EventSink: def heartbeat_event(self, event): pass @@ -1687,7 +1703,7 @@ class ConsensusTracker(EventHandler): if self.consensus_only: self._update_consensus(self.c.get_consensus()) else: - self._update_consensus(self.c.get_network_status()) + self._update_consensus(self.c.get_network_status(getIterator=True)) self._read_routers(self.ns_map.values()) def new_consensus_event(self, n):

1 0

[pytorctl/master] TorCtl connect method inflexible
by mikeperry＠torproject.org 18 Jun '11

18 Jun '11

commit 565e18a6ff6189dd11d8a43ca77eed287743b479 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Jun 16 07:50:58 2011 -0700 TorCtl connect method inflexible The connect method is a nice helper for cli initiation, but lacks sufficient call visibility to help in more general use cases. In particular it... - sends feedback to stdout - prompts for the controller password on stdin - suppresses exceptions To get around this I'm adding a more generic 'connectionComp' that does the icky bits of the connection negotiation I was trying to hide while still giving the caller what it needs to handle the connection process however it'd like. Tested by exercising the connect functionality with cookies, password, and connection components to handle my TBB cookie auth renegotiation fix. --- TorCtl.py | 62 +++++++++++++++++++++++++++++++++++++++++------------------- 1 files changed, 42 insertions(+), 20 deletions(-) diff --git a/TorCtl.py b/TorCtl.py index 8680ee6..cb3b53f 100755 --- a/TorCtl.py +++ b/TorCtl.py @@ -108,36 +108,20 @@ def connect(controlAddr="127.0.0.1", controlPort=9051, passphrase=None): passphrase - authentication passphrase (if defined this is used rather than prompting the user) """ - - conn = None + try: - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.connect((controlAddr, controlPort)) - conn = Connection(s) - authType, authValue = conn.get_auth_type(), "" - + conn, authType, authValue = connectionComp(controlAddr, controlPort) + if authType == AUTH_TYPE.PASSWORD: # password authentication, promting for the password if it wasn't provided if passphrase: authValue = passphrase else: try: authValue = getpass.getpass() except KeyboardInterrupt: return None - elif authType == AUTH_TYPE.COOKIE: - authValue = conn.get_auth_cookie_path() - + conn.authenticate(authValue) return conn - except socket.error, exc: - if "Connection refused" in exc.args: - # most common case - tor control port isn't available - print "Connection refused. Is the ControlPort enabled?" - else: print "Failed to establish socket: %s" % exc - - if conn: conn.close() - return None except Exception, exc: - if conn: conn.close() - if passphrase and str(exc) == "Unable to authenticate: password incorrect": # provide a warning that the provided password didn't work, then try # again prompting for the user to enter it @@ -147,6 +131,44 @@ def connect(controlAddr="127.0.0.1", controlPort=9051, passphrase=None): print exc return None +def connectionComp(controlAddr="127.0.0.1", controlPort=9051): + """ + Provides an uninitiated torctl connection components for the control port, + returning a tuple of the form... + (torctl connection, authType, authValue) + + The authValue corresponds to the cookie path if using an authentication + cookie, otherwise this is the empty string. This raises an IOError in case + of failure. + + Arguments: + controlAddr - ip address belonging to the controller + controlPort - port belonging to the controller + """ + + conn = None + try: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.connect((controlAddr, controlPort)) + conn = Connection(s) + authType, authValue = conn.get_auth_type(), "" + + if authType == AUTH_TYPE.COOKIE: + authValue = conn.get_auth_cookie_path() + + return (conn, authType, authValue) + except socket.error, exc: + if conn: conn.close() + + if "Connection refused" in exc.args: + # most common case - tor control port isn't available + raise IOError("Connection refused. Is the ControlPort enabled?") + + raise IOError("Failed to establish socket: %s" % exc) + except Exception, exc: + if conn: conn.close() + raise IOError(exc) + class TorCtlError(Exception): "Generic error raised by TorControl code." pass

1 0

[pytorctl/master] Add iterator support to get_consensus().
by mikeperry＠torproject.org 18 Jun '11

18 Jun '11

commit 5b73e36159cab89d76f481047f8524e8e9938ed5 Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Fri Jun 17 16:24:50 2011 -0700 Add iterator support to get_consensus(). Also rename getIterator to get_iterator. --- TorCtl.py | 23 +++++++++++++++++------ 1 files changed, 17 insertions(+), 6 deletions(-) diff --git a/TorCtl.py b/TorCtl.py index 2eddabc..b1387a0 100755 --- a/TorCtl.py +++ b/TorCtl.py @@ -1060,12 +1060,23 @@ class Connection: """ return self.sendAndRecv("RESETCONF %s\r\n"%(" ".join(keylist))) - def get_consensus(self): + def get_consensus(self, get_iterator=False): """Get the pristine Tor Consensus. Returns a list of - TorCtl.NetworkStatus instances.""" - return parse_ns_body(self.sendAndRecv("GETINFO dir/status-vote/current/consensus\r\n")[0][2]) + TorCtl.NetworkStatus instances. + + Be aware that by default this reads the whole consensus into memory at + once which can be fairly sizable (as of writing 3.5 MB), and even if + freed it may remain allocated to the interpretor: + http://effbot.org/pyfaq/why-doesnt-python-release-the-memory-when-i-delete-… + + To avoid this use the iterator instead. + """ + + nsData = self.sendAndRecv("GETINFO dir/status-vote/current/consensus\r\n")[0][2] + if get_iterator: return ns_body_iter(nsData) + else: return parse_ns_body(nsData) - def get_network_status(self, who="all", getIterator=False): + def get_network_status(self, who="all", get_iterator=False): """Get the entire network status list. Returns a list of TorCtl.NetworkStatus instances. @@ -1078,7 +1089,7 @@ class Connection: """ nsData = self.sendAndRecv("GETINFO ns/"+who+"\r\n")[0][2] - if getIterator: return ns_body_iter(nsData) + if get_iterator: return ns_body_iter(nsData) else: return parse_ns_body(nsData) def get_address_mappings(self, type="all"): @@ -1728,7 +1739,7 @@ class ConsensusTracker(EventHandler): if self.consensus_only: self._update_consensus(self.c.get_consensus()) else: - self._update_consensus(self.c.get_network_status(getIterator=True)) + self._update_consensus(self.c.get_network_status(get_iterator=True)) self._read_routers(self.ns_map.values()) def new_consensus_event(self, n):

1 0

[pytorctl/master] Merge commit '40f7cbc' into bug2812
by mikeperry＠torproject.org 18 Jun '11

18 Jun '11

commit 3c6a3268326d928474b4a1093cf74a2c32898208 Merge: 42b5f0e 40f7cbc Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Jun 16 19:15:38 2011 -0700 Merge commit '40f7cbc' into bug2812 Conflicts: TorUtil.py TorCtl.py | 186 ++++++++++++++++++++++++++++++------------------------------ TorUtil.py | 9 ++- example.py | 34 +++++++++++ 3 files changed, 134 insertions(+), 95 deletions(-) diff --cc TorUtil.py index 82c2a12,fcbef70..b8d6425 --- a/TorUtil.py +++ b/TorUtil.py @@@ -200,7 -204,11 +200,8 @@@ class BufSock return result while 1: - s = self._s.recv(128) - try: - s = self._s.recv(128) - except socket.timeout: - if not self._isDone: continue ++ try: s = self._s.recv(128) + except: s = None if not s: return None # XXX: This really does need an exception @@@ -222,7 -230,13 +223,11 @@@ self._s.send(s) def close(self): - self._s.shutdown(socket.SHUT_RDWR) - self._isDone = True - + # if we haven't yet established a connection then this raises an error + # socket.error: [Errno 107] Transport endpoint is not connected + try: self._s.shutdown(socket.SHUT_RDWR) + except socket.error: pass + self._s.close() # SocketServer.TCPServer is nuts..

1 0

[flashproxy/master] Be consistent about prefixing prompt indicators in README.
by dcf＠torproject.org 18 Jun '11

18 Jun '11

commit 4d2318d5b93d445d0d6e27bb4464ffe14336f0ce Author: David Fifield <david(a)bamsoftware.com> Date: Fri Jun 17 12:45:54 2011 -0700 Be consistent about prefixing prompt indicators in README. --- README | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README b/README index ef7e826..ef2cc65 100644 --- a/README +++ b/README @@ -39,12 +39,12 @@ Copy the resulting swfcat.swf file to a web server. === Connecting On the Tor relay, run - sudo ./crossdomaind.py + # ./crossdomaind.py In general, any computer that a proxy connects to needs to serve a crossdomain policy, but the connector and facilitator have it built in. On the facilitator, run - ./facilitator.py -r <relay-ip> + $ ./facilitator.py -r <relay-ip> The facilitator runs on port 9002 by default. In a browser somewhere, open swfcat.swf and pass it a parameters telling it @@ -58,7 +58,7 @@ coordinates connections using another server (which may be the same as the facilitator) and works when the client is behind a NAT. To connect with TCP, run - ./connector.py -f <facilitator-ip> + $ ./connector.py -f <facilitator-ip> (If you are running the facilitator locally, be sure to use an external IP address, not 127.0.0.1.) The connector informs the facilitator that it wants a connection, then listens on 0.0.0.0:9000 and 127.0.0.1:9001. @@ -66,7 +66,7 @@ The proxy will connect on port 9000 and the local Tor will connect on 9001. To connect with RTMFP, run - ./connector.py 127.0.0.1:9001 127.0.0.1:9002 + $ ./connector.py 127.0.0.1:9001 127.0.0.1:9002 and open in a browser http://www.example.com/swfcat.swf?client=1&facilitator=<facilitator-ip>:9002 In this case, it is swfcat running in client mode that registers with

1 0

[tor/master] Abandon rendezvous circuits on SIGNAL NEWNYM
by nickm＠torproject.org 18 Jun '11

18 Jun '11

commit 010b8dd4f6e8e3c3d2e44ff589ff61cbf64b952a Author: Robert Ransom <rransom.8774(a)gmail.com> Date: Thu Jun 9 19:56:40 2011 -0700 Abandon rendezvous circuits on SIGNAL NEWNYM --- changes/abandon-rend-circs-on-newnym | 8 ++++++++ src/or/circuituse.c | 3 ++- 2 files changed, 10 insertions(+), 1 deletions(-) diff --git a/changes/abandon-rend-circs-on-newnym b/changes/abandon-rend-circs-on-newnym new file mode 100644 index 0000000..67cb2dc --- /dev/null +++ b/changes/abandon-rend-circs-on-newnym @@ -0,0 +1,8 @@ + o Security fixes: + - Don't attach new streams to old rendezvous circuits after SIGNAL + NEWNYM. Previously, we would keep using an existing rendezvous + circuit if it remained open (i.e. if it were kept open by a + long-lived stream or if a new stream were attached to it before + Tor could notice that it was old and no longer in use and close + it). Bugfix on 0.1.1.15-rc; fixes bug 3375. + diff --git a/src/or/circuituse.c b/src/or/circuituse.c index 41c1899..48fc198 100644 --- a/src/or/circuituse.c +++ b/src/or/circuituse.c @@ -59,7 +59,8 @@ circuit_is_acceptable(circuit_t *circ, edge_connection_t *conn, return 0; } - if (purpose == CIRCUIT_PURPOSE_C_GENERAL) + if (purpose == CIRCUIT_PURPOSE_C_GENERAL || + purpose == CIRCUIT_PURPOSE_C_REND_JOINED) if (circ->timestamp_dirty && circ->timestamp_dirty+get_options()->MaxCircuitDirtiness <= now) return 0;

1 0

[tor/master] Merge remote-tracking branch 'origin/maint-0.2.2'
by nickm＠torproject.org 18 Jun '11

18 Jun '11

commit 42d6f3459053aaafbed2ab215e9e85ae5d594b5a Merge: 31b9b1a 85d4c29 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Jun 17 15:25:18 2011 -0400 Merge remote-tracking branch 'origin/maint-0.2.2' changes/abandon-rend-circs-on-newnym | 8 ++++++++ src/or/circuituse.c | 3 ++- 2 files changed, 10 insertions(+), 1 deletions(-)

1 0

[tor/master] Merge remote-tracking branch 'origin/maint-0.2.1' into maint-0.2.2
by nickm＠torproject.org 18 Jun '11

18 Jun '11

commit 85d4c290d79264d334baa9e5c8fdf3b06d0b3d79 Merge: 209229f 010b8dd Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Jun 17 15:24:23 2011 -0400 Merge remote-tracking branch 'origin/maint-0.2.1' into maint-0.2.2 changes/abandon-rend-circs-on-newnym | 8 ++++++++ src/or/circuituse.c | 3 ++- 2 files changed, 10 insertions(+), 1 deletions(-)

1 0