- tor-commits - lists.torproject.org

[torflow/master] Update submodule.
by mikeperry＠torproject.org 18 Jun '11

18 Jun '11

commit 8db5826def8209a518c8e473fd4cb6d30e55b47f Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Fri Jun 17 16:32:58 2011 -0700 Update submodule. --- TorCtl | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/TorCtl b/TorCtl index b99ecc1..0ec171e 160000 --- a/TorCtl +++ b/TorCtl @@ -1 +1 @@ -Subproject commit b99ecc1de55cdefbcd803e267d2635534ff9e511 +Subproject commit 0ec171e6e0286b84802d09d399f7ff620dc37803

1 0

[pytorctl/master] Rename connectionComp and export ns_body_iter.
by mikeperry＠torproject.org 18 Jun '11

18 Jun '11

commit 0ec171e6e0286b84802d09d399f7ff620dc37803 Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Fri Jun 17 16:29:36 2011 -0700 Rename connectionComp and export ns_body_iter. --- TorCtl.py | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diff --git a/TorCtl.py b/TorCtl.py index b1387a0..250dab1 100755 --- a/TorCtl.py +++ b/TorCtl.py @@ -35,7 +35,7 @@ __all__ = ["EVENT_TYPE", "connect", "TorCtlError", "TorCtlClosed", "NewDescEvent", "CircuitEvent", "StreamEvent", "ORConnEvent", "StreamBwEvent", "LogEvent", "AddrMapEvent", "BWEvent", "BuildTimeoutSetEvent", "UnknownEvent", "ConsensusTracker", - "EventListener", "EVENT_STATE" ] + "EventListener", "EVENT_STATE", "ns_body_iter" ] import os import re @@ -111,7 +111,7 @@ def connect(controlAddr="127.0.0.1", controlPort=9051, passphrase=None): conn = None try: - conn, authType, authValue = connectionComp(controlAddr, controlPort) + conn, authType, authValue = _make_connection(controlAddr, controlPort) if authType == AUTH_TYPE.PASSWORD: # password authentication, promting for the password if it wasn't provided @@ -134,7 +134,7 @@ def connect(controlAddr="127.0.0.1", controlPort=9051, passphrase=None): print exc return None -def connectionComp(controlAddr="127.0.0.1", controlPort=9051): +def _make_connection(controlAddr="127.0.0.1", controlPort=9051): """ Provides an uninitiated torctl connection components for the control port, returning a tuple of the form...

1 0

[pytorctl/master] Unblocking waits for a response when we error
by mikeperry＠torproject.org 18 Jun '11

18 Jun '11

commit 9de33409bb240d8fe50e135dfb895536dc6a45bd Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jun 13 09:30:42 2011 -0700 Unblocking waits for a response when we error When the signal sent by _sendImpl causes the control connection to close we block indefinitely waiting for a response. For more information see bug 1329. --- TorCtl.py | 12 +++++++++++- TorUtil.py | 1 + 2 files changed, 12 insertions(+), 1 deletions(-) diff --git a/TorCtl.py b/TorCtl.py index 2758033..056a65c 100755 --- a/TorCtl.py +++ b/TorCtl.py @@ -681,8 +681,18 @@ class Connection: while 1: try: isEvent, reply = self._read_reply() - except TorCtlClosed: + except TorCtlClosed, exc: plog("NOTICE", "Tor closed control connection. Exiting event thread.") + + # notify anything blocking on a response of the error, for details see: + # https://trac.torproject.org/projects/tor/ticket/1329 + try: + self._closedEx = exc + cb = self._queue.get(timeout=0) + if cb != "CLOSE": + cb("EXCEPTION") + except Queue.Empty: pass + return except Exception,e: if not self._closed: diff --git a/TorUtil.py b/TorUtil.py index 60d2b2f..29c44bf 100644 --- a/TorUtil.py +++ b/TorUtil.py @@ -208,6 +208,7 @@ class BufSock: s = self._s.recv(128) except socket.timeout: if not self._isDone: continue + except: s = None if not s: return None # XXX: This really does need an exception

1 0

[pytorctl/master] Handling for connections to non-control sockets
by mikeperry＠torproject.org 18 Jun '11

18 Jun '11

commit e76418e507319887626b9d3921bdc0ac44b28e76 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jun 13 09:50:13 2011 -0700 Handling for connections to non-control sockets When we attempt to initiate a control connection to a non-control port (for instance tor's socks port instead)... - the socket shutdown issues an error since it's not connected yet - checking the authentication type fails with a TorCtlClosed rather than error reply For more information see bug 2580. --- TorCtl.py | 5 +++-- TorUtil.py | 7 ++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/TorCtl.py b/TorCtl.py index 056a65c..44a612b 100755 --- a/TorCtl.py +++ b/TorCtl.py @@ -590,8 +590,9 @@ class Connection: # check PROTOCOLINFO for authentication type try: authInfo = self.sendAndRecv("PROTOCOLINFO\r\n")[1][1] - except ErrorReply, exc: - raise IOError("Unable to query PROTOCOLINFO for the authentication type: %s" % exc) + except Exception, exc: + excMsg = ": %s" % exc if exc.message else "" + raise IOError("Unable to query PROTOCOLINFO for the authentication type%s" % excMsg) authType, cookiePath = None, None if authInfo.startswith("AUTH METHODS=NULL"): diff --git a/TorUtil.py b/TorUtil.py index 29c44bf..fcbef70 100644 --- a/TorUtil.py +++ b/TorUtil.py @@ -231,7 +231,12 @@ class BufSock: def close(self): self._isDone = True - self._s.shutdown(socket.SHUT_RDWR) + + # if we haven't yet established a connection then this raises an error + # socket.error: [Errno 107] Transport endpoint is not connected + try: self._s.shutdown(socket.SHUT_RDWR) + except socket.error: pass + self._s.close() # SocketServer.TCPServer is nuts..

1 0

[pytorctl/master] Option to get get_network_status as an iterator
by mikeperry＠torproject.org 18 Jun '11

18 Jun '11

commit 45d7753b7c947b141210017d1b987673c51d97c5 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Jun 15 19:27:12 2011 -0700 Option to get get_network_status as an iterator Setting the 'getIterator' argument drops the memory usage of calling get_network_status by 71% (from 3.5 MB to 1 MB). This is still higher than what I was expecting from a generator, though certainly much better. Unfortunately this didn't have an impact on the ConsensusTracker. The memory usage from its constructor dwarfs anything else I've looked at (18.8 MB) and didn't drop like I'd expect when consensus_only was false. :( --- TorCtl.py | 54 +++++++++++++++++++++++++++++++++++------------------- 1 files changed, 35 insertions(+), 19 deletions(-) diff --git a/TorCtl.py b/TorCtl.py index 44a612b..8680ee6 100755 --- a/TorCtl.py +++ b/TorCtl.py @@ -1040,10 +1040,21 @@ class Connection: TorCtl.NetworkStatus instances.""" return parse_ns_body(self.sendAndRecv("GETINFO dir/status-vote/current/consensus\r\n")[0][2]) - def get_network_status(self, who="all"): + def get_network_status(self, who="all", getIterator=False): """Get the entire network status list. Returns a list of - TorCtl.NetworkStatus instances.""" - return parse_ns_body(self.sendAndRecv("GETINFO ns/"+who+"\r\n")[0][2]) + TorCtl.NetworkStatus instances. + + Be aware that by default this reads the whole consensus into memory at + once which can be fairly sizable (as of writing 3.5 MB), and even if + freed it may remain allocated to the interpretor: + http://effbot.org/pyfaq/why-doesnt-python-release-the-memory-when-i-delete-… + + To avoid this use the iterator instead. + """ + + nsData = self.sendAndRecv("GETINFO ns/"+who+"\r\n")[0][2] + if getIterator: return ns_body_iter(nsData) + else: return parse_ns_body(nsData) def get_address_mappings(self, type="all"): # TODO: Also parse errors and GMTExpiry @@ -1232,21 +1243,26 @@ class Connection: def parse_ns_body(data): """Parse the body of an NS event or command into a list of NetworkStatus instances""" - if not data: return [] - nsgroups = re.compile(r"^r ", re.M).split(data) - nsgroups.pop(0) - nslist = [] - for nsline in nsgroups: - m = re.search(r"^s((?:[ ]\S*)+)", nsline, re.M) - flags = m.groups() - flags = flags[0].strip().split(" ") - m = re.match(r"(\S+)\s(\S+)\s(\S+)\s(\S+\s\S+)\s(\S+)\s(\d+)\s(\d+)", nsline) - w = re.search(r"^w Bandwidth=(\d+)", nsline, re.M) - if w: - nslist.append(NetworkStatus(*(m.groups()+(flags,)+(int(w.group(1))*1000,)))) - else: - nslist.append(NetworkStatus(*(m.groups() + (flags,)))) - return nslist + return list(ns_body_iter(data)) + +def ns_body_iter(data): + """Generator for NetworkStatus instances of an NS event""" + if data: + nsgroups = re.compile(r"^r ", re.M).split(data) + nsgroups.pop(0) + + while nsgroups: + nsline = nsgroups.pop(0) + m = re.search(r"^s((?:[ ]\S*)+)", nsline, re.M) + flags = m.groups() + flags = flags[0].strip().split(" ") + m = re.match(r"(\S+)\s(\S+)\s(\S+)\s(\S+\s\S+)\s(\S+)\s(\d+)\s(\d+)", nsline) + w = re.search(r"^w Bandwidth=(\d+)", nsline, re.M) + + if w: + yield NetworkStatus(*(m.groups()+(flags,)+(int(w.group(1))*1000,))) + else: + yield NetworkStatus(*(m.groups() + (flags,))) class EventSink: def heartbeat_event(self, event): pass @@ -1687,7 +1703,7 @@ class ConsensusTracker(EventHandler): if self.consensus_only: self._update_consensus(self.c.get_consensus()) else: - self._update_consensus(self.c.get_network_status()) + self._update_consensus(self.c.get_network_status(getIterator=True)) self._read_routers(self.ns_map.values()) def new_consensus_event(self, n):

1 0

[pytorctl/master] TorCtl connect method inflexible
by mikeperry＠torproject.org 18 Jun '11

18 Jun '11

commit 565e18a6ff6189dd11d8a43ca77eed287743b479 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Jun 16 07:50:58 2011 -0700 TorCtl connect method inflexible The connect method is a nice helper for cli initiation, but lacks sufficient call visibility to help in more general use cases. In particular it... - sends feedback to stdout - prompts for the controller password on stdin - suppresses exceptions To get around this I'm adding a more generic 'connectionComp' that does the icky bits of the connection negotiation I was trying to hide while still giving the caller what it needs to handle the connection process however it'd like. Tested by exercising the connect functionality with cookies, password, and connection components to handle my TBB cookie auth renegotiation fix. --- TorCtl.py | 62 +++++++++++++++++++++++++++++++++++++++++------------------- 1 files changed, 42 insertions(+), 20 deletions(-) diff --git a/TorCtl.py b/TorCtl.py index 8680ee6..cb3b53f 100755 --- a/TorCtl.py +++ b/TorCtl.py @@ -108,36 +108,20 @@ def connect(controlAddr="127.0.0.1", controlPort=9051, passphrase=None): passphrase - authentication passphrase (if defined this is used rather than prompting the user) """ - - conn = None + try: - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.connect((controlAddr, controlPort)) - conn = Connection(s) - authType, authValue = conn.get_auth_type(), "" - + conn, authType, authValue = connectionComp(controlAddr, controlPort) + if authType == AUTH_TYPE.PASSWORD: # password authentication, promting for the password if it wasn't provided if passphrase: authValue = passphrase else: try: authValue = getpass.getpass() except KeyboardInterrupt: return None - elif authType == AUTH_TYPE.COOKIE: - authValue = conn.get_auth_cookie_path() - + conn.authenticate(authValue) return conn - except socket.error, exc: - if "Connection refused" in exc.args: - # most common case - tor control port isn't available - print "Connection refused. Is the ControlPort enabled?" - else: print "Failed to establish socket: %s" % exc - - if conn: conn.close() - return None except Exception, exc: - if conn: conn.close() - if passphrase and str(exc) == "Unable to authenticate: password incorrect": # provide a warning that the provided password didn't work, then try # again prompting for the user to enter it @@ -147,6 +131,44 @@ def connect(controlAddr="127.0.0.1", controlPort=9051, passphrase=None): print exc return None +def connectionComp(controlAddr="127.0.0.1", controlPort=9051): + """ + Provides an uninitiated torctl connection components for the control port, + returning a tuple of the form... + (torctl connection, authType, authValue) + + The authValue corresponds to the cookie path if using an authentication + cookie, otherwise this is the empty string. This raises an IOError in case + of failure. + + Arguments: + controlAddr - ip address belonging to the controller + controlPort - port belonging to the controller + """ + + conn = None + try: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.connect((controlAddr, controlPort)) + conn = Connection(s) + authType, authValue = conn.get_auth_type(), "" + + if authType == AUTH_TYPE.COOKIE: + authValue = conn.get_auth_cookie_path() + + return (conn, authType, authValue) + except socket.error, exc: + if conn: conn.close() + + if "Connection refused" in exc.args: + # most common case - tor control port isn't available + raise IOError("Connection refused. Is the ControlPort enabled?") + + raise IOError("Failed to establish socket: %s" % exc) + except Exception, exc: + if conn: conn.close() + raise IOError(exc) + class TorCtlError(Exception): "Generic error raised by TorControl code." pass

1 0

[pytorctl/master] Closing connection in case of auth failure
by mikeperry＠torproject.org 18 Jun '11

18 Jun '11

commit 40f7cbc50d460d8e3e11669827108bfb912ab3bf Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Jun 16 18:15:05 2011 -0700 Closing connection in case of auth failure When authentication failed in TorCtl.connect() the abandoned connection wasn't being closed. --- TorCtl.py | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diff --git a/TorCtl.py b/TorCtl.py index cb3b53f..2eddabc 100755 --- a/TorCtl.py +++ b/TorCtl.py @@ -109,6 +109,7 @@ def connect(controlAddr="127.0.0.1", controlPort=9051, passphrase=None): than prompting the user) """ + conn = None try: conn, authType, authValue = connectionComp(controlAddr, controlPort) @@ -122,6 +123,8 @@ def connect(controlAddr="127.0.0.1", controlPort=9051, passphrase=None): conn.authenticate(authValue) return conn except Exception, exc: + if conn: conn.close() + if passphrase and str(exc) == "Unable to authenticate: password incorrect": # provide a warning that the provided password didn't work, then try # again prompting for the user to enter it

1 0

[pytorctl/master] Merge commit '40f7cbc' into bug2812
by mikeperry＠torproject.org 18 Jun '11

18 Jun '11

commit 3c6a3268326d928474b4a1093cf74a2c32898208 Merge: 42b5f0e 40f7cbc Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Jun 16 19:15:38 2011 -0700 Merge commit '40f7cbc' into bug2812 Conflicts: TorUtil.py TorCtl.py | 186 ++++++++++++++++++++++++++++++------------------------------ TorUtil.py | 9 ++- example.py | 34 +++++++++++ 3 files changed, 134 insertions(+), 95 deletions(-) diff --cc TorUtil.py index 82c2a12,fcbef70..b8d6425 --- a/TorUtil.py +++ b/TorUtil.py @@@ -200,7 -204,11 +200,8 @@@ class BufSock return result while 1: - s = self._s.recv(128) - try: - s = self._s.recv(128) - except socket.timeout: - if not self._isDone: continue ++ try: s = self._s.recv(128) + except: s = None if not s: return None # XXX: This really does need an exception @@@ -222,7 -230,13 +223,11 @@@ self._s.send(s) def close(self): - self._s.shutdown(socket.SHUT_RDWR) - self._isDone = True - + # if we haven't yet established a connection then this raises an error + # socket.error: [Errno 107] Transport endpoint is not connected + try: self._s.shutdown(socket.SHUT_RDWR) + except socket.error: pass + self._s.close() # SocketServer.TCPServer is nuts..

1 0

[pytorctl/master] Merge remote branch 'atagar/bug2812'
by mikeperry＠torproject.org 18 Jun '11

18 Jun '11

commit cb62cf7c70982b83c2d480e55843e098d42ed697 Merge: 8554340 3c6a326 Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Fri Jun 17 16:10:45 2011 -0700 Merge remote branch 'atagar/bug2812' TorCtl.py | 130 ++++++++++++++++++++++++++++++++++++++++++------------------ TorUtil.py | 9 +++- 2 files changed, 98 insertions(+), 41 deletions(-)

1 0

[pytorctl/master] Add iterator support to get_consensus().
by mikeperry＠torproject.org 18 Jun '11

18 Jun '11

commit 5b73e36159cab89d76f481047f8524e8e9938ed5 Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Fri Jun 17 16:24:50 2011 -0700 Add iterator support to get_consensus(). Also rename getIterator to get_iterator. --- TorCtl.py | 23 +++++++++++++++++------ 1 files changed, 17 insertions(+), 6 deletions(-) diff --git a/TorCtl.py b/TorCtl.py index 2eddabc..b1387a0 100755 --- a/TorCtl.py +++ b/TorCtl.py @@ -1060,12 +1060,23 @@ class Connection: """ return self.sendAndRecv("RESETCONF %s\r\n"%(" ".join(keylist))) - def get_consensus(self): + def get_consensus(self, get_iterator=False): """Get the pristine Tor Consensus. Returns a list of - TorCtl.NetworkStatus instances.""" - return parse_ns_body(self.sendAndRecv("GETINFO dir/status-vote/current/consensus\r\n")[0][2]) + TorCtl.NetworkStatus instances. + + Be aware that by default this reads the whole consensus into memory at + once which can be fairly sizable (as of writing 3.5 MB), and even if + freed it may remain allocated to the interpretor: + http://effbot.org/pyfaq/why-doesnt-python-release-the-memory-when-i-delete-… + + To avoid this use the iterator instead. + """ + + nsData = self.sendAndRecv("GETINFO dir/status-vote/current/consensus\r\n")[0][2] + if get_iterator: return ns_body_iter(nsData) + else: return parse_ns_body(nsData) - def get_network_status(self, who="all", getIterator=False): + def get_network_status(self, who="all", get_iterator=False): """Get the entire network status list. Returns a list of TorCtl.NetworkStatus instances. @@ -1078,7 +1089,7 @@ class Connection: """ nsData = self.sendAndRecv("GETINFO ns/"+who+"\r\n")[0][2] - if getIterator: return ns_body_iter(nsData) + if get_iterator: return ns_body_iter(nsData) else: return parse_ns_body(nsData) def get_address_mappings(self, type="all"): @@ -1728,7 +1739,7 @@ class ConsensusTracker(EventHandler): if self.consensus_only: self._update_consensus(self.c.get_consensus()) else: - self._update_consensus(self.c.get_network_status(getIterator=True)) + self._update_consensus(self.c.get_network_status(get_iterator=True)) self._read_routers(self.ns_map.values()) def new_consensus_event(self, n):

1 0