- tor-commits - lists.torproject.org

[pytorctl/master] Option to get get_network_status as an iterator
by mikeperry＠torproject.org 17 Jun '11

17 Jun '11

commit 45d7753b7c947b141210017d1b987673c51d97c5 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Jun 15 19:27:12 2011 -0700 Option to get get_network_status as an iterator Setting the 'getIterator' argument drops the memory usage of calling get_network_status by 71% (from 3.5 MB to 1 MB). This is still higher than what I was expecting from a generator, though certainly much better. Unfortunately this didn't have an impact on the ConsensusTracker. The memory usage from its constructor dwarfs anything else I've looked at (18.8 MB) and didn't drop like I'd expect when consensus_only was false. :( --- TorCtl.py | 54 +++++++++++++++++++++++++++++++++++------------------- 1 files changed, 35 insertions(+), 19 deletions(-) diff --git a/TorCtl.py b/TorCtl.py index 44a612b..8680ee6 100755 --- a/TorCtl.py +++ b/TorCtl.py @@ -1040,10 +1040,21 @@ class Connection: TorCtl.NetworkStatus instances.""" return parse_ns_body(self.sendAndRecv("GETINFO dir/status-vote/current/consensus\r\n")[0][2]) - def get_network_status(self, who="all"): + def get_network_status(self, who="all", getIterator=False): """Get the entire network status list. Returns a list of - TorCtl.NetworkStatus instances.""" - return parse_ns_body(self.sendAndRecv("GETINFO ns/"+who+"\r\n")[0][2]) + TorCtl.NetworkStatus instances. + + Be aware that by default this reads the whole consensus into memory at + once which can be fairly sizable (as of writing 3.5 MB), and even if + freed it may remain allocated to the interpretor: + http://effbot.org/pyfaq/why-doesnt-python-release-the-memory-when-i-delete-… + + To avoid this use the iterator instead. + """ + + nsData = self.sendAndRecv("GETINFO ns/"+who+"\r\n")[0][2] + if getIterator: return ns_body_iter(nsData) + else: return parse_ns_body(nsData) def get_address_mappings(self, type="all"): # TODO: Also parse errors and GMTExpiry @@ -1232,21 +1243,26 @@ class Connection: def parse_ns_body(data): """Parse the body of an NS event or command into a list of NetworkStatus instances""" - if not data: return [] - nsgroups = re.compile(r"^r ", re.M).split(data) - nsgroups.pop(0) - nslist = [] - for nsline in nsgroups: - m = re.search(r"^s((?:[ ]\S*)+)", nsline, re.M) - flags = m.groups() - flags = flags[0].strip().split(" ") - m = re.match(r"(\S+)\s(\S+)\s(\S+)\s(\S+\s\S+)\s(\S+)\s(\d+)\s(\d+)", nsline) - w = re.search(r"^w Bandwidth=(\d+)", nsline, re.M) - if w: - nslist.append(NetworkStatus(*(m.groups()+(flags,)+(int(w.group(1))*1000,)))) - else: - nslist.append(NetworkStatus(*(m.groups() + (flags,)))) - return nslist + return list(ns_body_iter(data)) + +def ns_body_iter(data): + """Generator for NetworkStatus instances of an NS event""" + if data: + nsgroups = re.compile(r"^r ", re.M).split(data) + nsgroups.pop(0) + + while nsgroups: + nsline = nsgroups.pop(0) + m = re.search(r"^s((?:[ ]\S*)+)", nsline, re.M) + flags = m.groups() + flags = flags[0].strip().split(" ") + m = re.match(r"(\S+)\s(\S+)\s(\S+)\s(\S+\s\S+)\s(\S+)\s(\d+)\s(\d+)", nsline) + w = re.search(r"^w Bandwidth=(\d+)", nsline, re.M) + + if w: + yield NetworkStatus(*(m.groups()+(flags,)+(int(w.group(1))*1000,))) + else: + yield NetworkStatus(*(m.groups() + (flags,))) class EventSink: def heartbeat_event(self, event): pass @@ -1687,7 +1703,7 @@ class ConsensusTracker(EventHandler): if self.consensus_only: self._update_consensus(self.c.get_consensus()) else: - self._update_consensus(self.c.get_network_status()) + self._update_consensus(self.c.get_network_status(getIterator=True)) self._read_routers(self.ns_map.values()) def new_consensus_event(self, n):

1 0

[pytorctl/master] TorCtl connect method inflexible
by mikeperry＠torproject.org 17 Jun '11

17 Jun '11

commit 565e18a6ff6189dd11d8a43ca77eed287743b479 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Jun 16 07:50:58 2011 -0700 TorCtl connect method inflexible The connect method is a nice helper for cli initiation, but lacks sufficient call visibility to help in more general use cases. In particular it... - sends feedback to stdout - prompts for the controller password on stdin - suppresses exceptions To get around this I'm adding a more generic 'connectionComp' that does the icky bits of the connection negotiation I was trying to hide while still giving the caller what it needs to handle the connection process however it'd like. Tested by exercising the connect functionality with cookies, password, and connection components to handle my TBB cookie auth renegotiation fix. --- TorCtl.py | 62 +++++++++++++++++++++++++++++++++++++++++------------------- 1 files changed, 42 insertions(+), 20 deletions(-) diff --git a/TorCtl.py b/TorCtl.py index 8680ee6..cb3b53f 100755 --- a/TorCtl.py +++ b/TorCtl.py @@ -108,36 +108,20 @@ def connect(controlAddr="127.0.0.1", controlPort=9051, passphrase=None): passphrase - authentication passphrase (if defined this is used rather than prompting the user) """ - - conn = None + try: - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.connect((controlAddr, controlPort)) - conn = Connection(s) - authType, authValue = conn.get_auth_type(), "" - + conn, authType, authValue = connectionComp(controlAddr, controlPort) + if authType == AUTH_TYPE.PASSWORD: # password authentication, promting for the password if it wasn't provided if passphrase: authValue = passphrase else: try: authValue = getpass.getpass() except KeyboardInterrupt: return None - elif authType == AUTH_TYPE.COOKIE: - authValue = conn.get_auth_cookie_path() - + conn.authenticate(authValue) return conn - except socket.error, exc: - if "Connection refused" in exc.args: - # most common case - tor control port isn't available - print "Connection refused. Is the ControlPort enabled?" - else: print "Failed to establish socket: %s" % exc - - if conn: conn.close() - return None except Exception, exc: - if conn: conn.close() - if passphrase and str(exc) == "Unable to authenticate: password incorrect": # provide a warning that the provided password didn't work, then try # again prompting for the user to enter it @@ -147,6 +131,44 @@ def connect(controlAddr="127.0.0.1", controlPort=9051, passphrase=None): print exc return None +def connectionComp(controlAddr="127.0.0.1", controlPort=9051): + """ + Provides an uninitiated torctl connection components for the control port, + returning a tuple of the form... + (torctl connection, authType, authValue) + + The authValue corresponds to the cookie path if using an authentication + cookie, otherwise this is the empty string. This raises an IOError in case + of failure. + + Arguments: + controlAddr - ip address belonging to the controller + controlPort - port belonging to the controller + """ + + conn = None + try: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.connect((controlAddr, controlPort)) + conn = Connection(s) + authType, authValue = conn.get_auth_type(), "" + + if authType == AUTH_TYPE.COOKIE: + authValue = conn.get_auth_cookie_path() + + return (conn, authType, authValue) + except socket.error, exc: + if conn: conn.close() + + if "Connection refused" in exc.args: + # most common case - tor control port isn't available + raise IOError("Connection refused. Is the ControlPort enabled?") + + raise IOError("Failed to establish socket: %s" % exc) + except Exception, exc: + if conn: conn.close() + raise IOError(exc) + class TorCtlError(Exception): "Generic error raised by TorControl code." pass

1 0

[pytorctl/master] Closing connection in case of auth failure
by mikeperry＠torproject.org 17 Jun '11

17 Jun '11

commit 40f7cbc50d460d8e3e11669827108bfb912ab3bf Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Jun 16 18:15:05 2011 -0700 Closing connection in case of auth failure When authentication failed in TorCtl.connect() the abandoned connection wasn't being closed. --- TorCtl.py | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diff --git a/TorCtl.py b/TorCtl.py index cb3b53f..2eddabc 100755 --- a/TorCtl.py +++ b/TorCtl.py @@ -109,6 +109,7 @@ def connect(controlAddr="127.0.0.1", controlPort=9051, passphrase=None): than prompting the user) """ + conn = None try: conn, authType, authValue = connectionComp(controlAddr, controlPort) @@ -122,6 +123,8 @@ def connect(controlAddr="127.0.0.1", controlPort=9051, passphrase=None): conn.authenticate(authValue) return conn except Exception, exc: + if conn: conn.close() + if passphrase and str(exc) == "Unable to authenticate: password incorrect": # provide a warning that the provided password didn't work, then try # again prompting for the user to enter it

1 0

[pytorctl/master] Merge commit '40f7cbc' into bug2812
by mikeperry＠torproject.org 17 Jun '11

17 Jun '11

commit 3c6a3268326d928474b4a1093cf74a2c32898208 Merge: 42b5f0e 40f7cbc Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Jun 16 19:15:38 2011 -0700 Merge commit '40f7cbc' into bug2812 Conflicts: TorUtil.py TorCtl.py | 186 ++++++++++++++++++++++++++++++------------------------------ TorUtil.py | 9 ++- example.py | 34 +++++++++++ 3 files changed, 134 insertions(+), 95 deletions(-) diff --cc TorUtil.py index 82c2a12,fcbef70..b8d6425 --- a/TorUtil.py +++ b/TorUtil.py @@@ -200,7 -204,11 +200,8 @@@ class BufSock return result while 1: - s = self._s.recv(128) - try: - s = self._s.recv(128) - except socket.timeout: - if not self._isDone: continue ++ try: s = self._s.recv(128) + except: s = None if not s: return None # XXX: This really does need an exception @@@ -222,7 -230,13 +223,11 @@@ self._s.send(s) def close(self): - self._s.shutdown(socket.SHUT_RDWR) - self._isDone = True - + # if we haven't yet established a connection then this raises an error + # socket.error: [Errno 107] Transport endpoint is not connected + try: self._s.shutdown(socket.SHUT_RDWR) + except socket.error: pass + self._s.close() # SocketServer.TCPServer is nuts..

1 0

[pytorctl/master] Merge remote branch 'atagar/bug2812'
by mikeperry＠torproject.org 17 Jun '11

17 Jun '11

commit cb62cf7c70982b83c2d480e55843e098d42ed697 Merge: 8554340 3c6a326 Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Fri Jun 17 16:10:45 2011 -0700 Merge remote branch 'atagar/bug2812' TorCtl.py | 130 ++++++++++++++++++++++++++++++++++++++++++------------------ TorUtil.py | 9 +++- 2 files changed, 98 insertions(+), 41 deletions(-)

1 0

[pytorctl/master] Add iterator support to get_consensus().
by mikeperry＠torproject.org 17 Jun '11

17 Jun '11

commit 5b73e36159cab89d76f481047f8524e8e9938ed5 Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Fri Jun 17 16:24:50 2011 -0700 Add iterator support to get_consensus(). Also rename getIterator to get_iterator. --- TorCtl.py | 23 +++++++++++++++++------ 1 files changed, 17 insertions(+), 6 deletions(-) diff --git a/TorCtl.py b/TorCtl.py index 2eddabc..b1387a0 100755 --- a/TorCtl.py +++ b/TorCtl.py @@ -1060,12 +1060,23 @@ class Connection: """ return self.sendAndRecv("RESETCONF %s\r\n"%(" ".join(keylist))) - def get_consensus(self): + def get_consensus(self, get_iterator=False): """Get the pristine Tor Consensus. Returns a list of - TorCtl.NetworkStatus instances.""" - return parse_ns_body(self.sendAndRecv("GETINFO dir/status-vote/current/consensus\r\n")[0][2]) + TorCtl.NetworkStatus instances. + + Be aware that by default this reads the whole consensus into memory at + once which can be fairly sizable (as of writing 3.5 MB), and even if + freed it may remain allocated to the interpretor: + http://effbot.org/pyfaq/why-doesnt-python-release-the-memory-when-i-delete-… + + To avoid this use the iterator instead. + """ + + nsData = self.sendAndRecv("GETINFO dir/status-vote/current/consensus\r\n")[0][2] + if get_iterator: return ns_body_iter(nsData) + else: return parse_ns_body(nsData) - def get_network_status(self, who="all", getIterator=False): + def get_network_status(self, who="all", get_iterator=False): """Get the entire network status list. Returns a list of TorCtl.NetworkStatus instances. @@ -1078,7 +1089,7 @@ class Connection: """ nsData = self.sendAndRecv("GETINFO ns/"+who+"\r\n")[0][2] - if getIterator: return ns_body_iter(nsData) + if get_iterator: return ns_body_iter(nsData) else: return parse_ns_body(nsData) def get_address_mappings(self, type="all"): @@ -1728,7 +1739,7 @@ class ConsensusTracker(EventHandler): if self.consensus_only: self._update_consensus(self.c.get_consensus()) else: - self._update_consensus(self.c.get_network_status(getIterator=True)) + self._update_consensus(self.c.get_network_status(get_iterator=True)) self._read_routers(self.ns_map.values()) def new_consensus_event(self, n):

1 0

[flashproxy/master] Be consistent about prefixing prompt indicators in README.
by dcf＠torproject.org 17 Jun '11

17 Jun '11

commit 4d2318d5b93d445d0d6e27bb4464ffe14336f0ce Author: David Fifield <david(a)bamsoftware.com> Date: Fri Jun 17 12:45:54 2011 -0700 Be consistent about prefixing prompt indicators in README. --- README | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README b/README index ef7e826..ef2cc65 100644 --- a/README +++ b/README @@ -39,12 +39,12 @@ Copy the resulting swfcat.swf file to a web server. === Connecting On the Tor relay, run - sudo ./crossdomaind.py + # ./crossdomaind.py In general, any computer that a proxy connects to needs to serve a crossdomain policy, but the connector and facilitator have it built in. On the facilitator, run - ./facilitator.py -r <relay-ip> + $ ./facilitator.py -r <relay-ip> The facilitator runs on port 9002 by default. In a browser somewhere, open swfcat.swf and pass it a parameters telling it @@ -58,7 +58,7 @@ coordinates connections using another server (which may be the same as the facilitator) and works when the client is behind a NAT. To connect with TCP, run - ./connector.py -f <facilitator-ip> + $ ./connector.py -f <facilitator-ip> (If you are running the facilitator locally, be sure to use an external IP address, not 127.0.0.1.) The connector informs the facilitator that it wants a connection, then listens on 0.0.0.0:9000 and 127.0.0.1:9001. @@ -66,7 +66,7 @@ The proxy will connect on port 9000 and the local Tor will connect on 9001. To connect with RTMFP, run - ./connector.py 127.0.0.1:9001 127.0.0.1:9002 + $ ./connector.py 127.0.0.1:9001 127.0.0.1:9002 and open in a browser http://www.example.com/swfcat.swf?client=1&facilitator=<facilitator-ip>:9002 In this case, it is swfcat running in client mode that registers with

1 0

[tor/master] Abandon rendezvous circuits on SIGNAL NEWNYM
by nickm＠torproject.org 17 Jun '11

17 Jun '11

commit 010b8dd4f6e8e3c3d2e44ff589ff61cbf64b952a Author: Robert Ransom <rransom.8774(a)gmail.com> Date: Thu Jun 9 19:56:40 2011 -0700 Abandon rendezvous circuits on SIGNAL NEWNYM --- changes/abandon-rend-circs-on-newnym | 8 ++++++++ src/or/circuituse.c | 3 ++- 2 files changed, 10 insertions(+), 1 deletions(-) diff --git a/changes/abandon-rend-circs-on-newnym b/changes/abandon-rend-circs-on-newnym new file mode 100644 index 0000000..67cb2dc --- /dev/null +++ b/changes/abandon-rend-circs-on-newnym @@ -0,0 +1,8 @@ + o Security fixes: + - Don't attach new streams to old rendezvous circuits after SIGNAL + NEWNYM. Previously, we would keep using an existing rendezvous + circuit if it remained open (i.e. if it were kept open by a + long-lived stream or if a new stream were attached to it before + Tor could notice that it was old and no longer in use and close + it). Bugfix on 0.1.1.15-rc; fixes bug 3375. + diff --git a/src/or/circuituse.c b/src/or/circuituse.c index 41c1899..48fc198 100644 --- a/src/or/circuituse.c +++ b/src/or/circuituse.c @@ -59,7 +59,8 @@ circuit_is_acceptable(circuit_t *circ, edge_connection_t *conn, return 0; } - if (purpose == CIRCUIT_PURPOSE_C_GENERAL) + if (purpose == CIRCUIT_PURPOSE_C_GENERAL || + purpose == CIRCUIT_PURPOSE_C_REND_JOINED) if (circ->timestamp_dirty && circ->timestamp_dirty+get_options()->MaxCircuitDirtiness <= now) return 0;

1 0

[tor/master] Merge remote-tracking branch 'origin/maint-0.2.1' into maint-0.2.2
by nickm＠torproject.org 17 Jun '11

17 Jun '11

commit 85d4c290d79264d334baa9e5c8fdf3b06d0b3d79 Merge: 209229f 010b8dd Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Jun 17 15:24:23 2011 -0400 Merge remote-tracking branch 'origin/maint-0.2.1' into maint-0.2.2 changes/abandon-rend-circs-on-newnym | 8 ++++++++ src/or/circuituse.c | 3 ++- 2 files changed, 10 insertions(+), 1 deletions(-)

1 0

[tor/master] Merge remote-tracking branch 'origin/maint-0.2.2'
by nickm＠torproject.org 17 Jun '11

17 Jun '11

commit 42d6f3459053aaafbed2ab215e9e85ae5d594b5a Merge: 31b9b1a 85d4c29 Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Jun 17 15:25:18 2011 -0400 Merge remote-tracking branch 'origin/maint-0.2.2' changes/abandon-rend-circs-on-newnym | 8 ++++++++ src/or/circuituse.c | 3 ++- 2 files changed, 10 insertions(+), 1 deletions(-)

1 0