[tor-commits] [tor/master] New UserspaceIOCPBuffers option to set SO_{SND, RCV}BUF to zero

[nickm at torproject.org]

commit 6e6a6612966d7ea9ad3d89bb5073ef400539b2a7
Author: Nick Mathewson <nickm at torproject.org>
Date:   Fri Nov 18 17:43:03 2011 -0500

    New UserspaceIOCPBuffers option to set SO_{SND,RCV}BUF to zero
    
    When running with IOCP, we are in theory able to use userspace-
    allocated buffers to avoid filling up the stingy amount of kernel
    space allocated for sockets buffers.
    
    The bufferevent_async implementation in Libevent provides this
    ability, in theory.  (There are likely to be remaining bugs).  This
    patch adds a new option that, when using IOCP bufferevents, sets
    each socket's send and receive buffers to 0, so that we should use
    this ability.
    
    When all the bugs are worked out here, if we are right about bug 98,
    this might solve or mitigate bug 98.
    
    This option is experimental and will likely require lots of testing
    and debugging.
---
 changes/UserspaceIOCPBuffers |    7 +++++++
 src/or/config.c              |    1 +
 src/or/main.c                |   28 ++++++++++++++++++++++++++++
 src/or/or.h                  |    5 +++++
 4 files changed, 41 insertions(+), 0 deletions(-)

diff --git a/changes/UserspaceIOCPBuffers b/changes/UserspaceIOCPBuffers
new file mode 100644
index 0000000..7115b5f
--- /dev/null
+++ b/changes/UserspaceIOCPBuffers
@@ -0,0 +1,7 @@
+  o Minor features:
+    - Experimental support for running on Windows with IOCP and no
+      kernel-space socket buffers. This feature is controlled by a new
+      UserspaceIOCPBuffers feature (off by default), which has no
+      effect unless Tor has been built with support for bufferevents,
+      is running on Windows, and has enabled IOCP.  This may, in the
+      long run, help solve or mitigate bug 98.
diff --git a/src/or/config.c b/src/or/config.c
index 1b9f9fb..4710b55 100644
--- a/src/or/config.c
+++ b/src/or/config.c
@@ -403,6 +403,7 @@ static config_var_t _option_vars[] = {
   V(UseEntryGuards,              BOOL,     "1"),
   V(UseMicrodescriptors,         AUTOBOOL, "auto"),
   V(User,                        STRING,   NULL),
+  V(UserspaceIOCPBuffers,        BOOL,     "0"),
   VAR("V1AuthoritativeDirectory",BOOL, V1AuthoritativeDir,   "0"),
   VAR("V2AuthoritativeDirectory",BOOL, V2AuthoritativeDir,   "0"),
   VAR("V3AuthoritativeDirectory",BOOL, V3AuthoritativeDir,   "0"),
diff --git a/src/or/main.c b/src/or/main.c
index c1a7015..9f6d307 100644
--- a/src/or/main.c
+++ b/src/or/main.c
@@ -196,6 +196,26 @@ free_old_inbuf(connection_t *conn)
 }
 #endif
 
+#ifdef MS_WINDOWS
+/** Remove the kernel-space send and receive buffers for <b>s</b>. For use
+ * with IOCP only. */
+static int
+set_buffer_lengths_to_zero(tor_socket_t s)
+{
+  int zero = 0;
+  int r = 0;
+  if (setsockopt(s, SOL_SOCKET, SO_SNDBUF, (void*)&zero, sizeof(zero))) {
+    log_warn(LD_NET, "Unable to clear SO_SNDBUF");
+    r = -1;
+  }
+  if (setsockopt(s, SOL_SOCKET, SO_RCVBUF, (void*)&zero, sizeof(zero))) {
+    log_warn(LD_NET, "Unable to clear SO_RCVBUF");
+    r = -1;
+  }
+  return r;
+}
+#endif
+
 /** Add <b>conn</b> to the array of connections that we can poll on.  The
  * connection's socket must be set; the connection starts out
  * non-reading and non-writing.
@@ -216,6 +236,14 @@ connection_add_impl(connection_t *conn, int is_connecting)
 #ifdef USE_BUFFEREVENTS
   if (connection_type_uses_bufferevent(conn)) {
     if (SOCKET_OK(conn->s) && !conn->linked) {
+
+#ifdef MS_WINDOWS
+      if (tor_libevent_using_iocp_bufferevents() &&
+          get_options()->UserspaceIOCPBuffers) {
+        set_buffer_lengths_to_zero(conn->s);
+      }
+#endif
+
       conn->bufev = bufferevent_socket_new(
                          tor_libevent_get_base(),
                          conn->s,
diff --git a/src/or/or.h b/src/or/or.h
index e4f9b9b..7fb7e9c 100644
--- a/src/or/or.h
+++ b/src/or/or.h
@@ -3424,6 +3424,11 @@ typedef struct {
    * never use it.  If -1, we do what the consensus says. */
   int OptimisticData;
 
+  /** If 1, and we are using IOCP, we set the kernel socket SNDBUF and RCVBUF
+   * to 0 to try to save kernel memory and avoid the dread "Out of buffers"
+   * issue. */
+  int UserspaceIOCPBuffers;
+
 } or_options_t;
 
 /** Persistent state for an onion router, as saved to disk. */