[or-cvs] Flush more changes from sandbox
Nick Mathewson
nickm at seul.org
Wed Sep 8 06:52:36 UTC 2004
Update of /home/or/cvsroot/src/or
In directory moria.mit.edu:/tmp/cvs-serv28436/src/or
Modified Files:
config.c connection.c directory.c dirserv.c main.c or.h
routerlist.c
Log Message:
Flush more changes from sandbox
- make clients cache directories and use them to seed their router lists
at startup. This means clients have a datadir again.
- Introduce a global_write_bucket. We need to respond better to exhausting
it.
- Remove the last vestiges of LinkPadding and TrafficShaping.
- Configuration infrastructure support for warning on obsolete options.
- Refactor directory header parsing to use smartlist_split_string.
- Respond to content-encoding headers by trying to uncompress as appropriate.
- Reply with a deflated directory when a client asks for "dir.z".
(We could use allow-encodings instead, but allow-encodings isn't
specified in HTTP 1.0.)
Index: config.c
===================================================================
RCS file: /home/or/cvsroot/src/or/config.c,v
retrieving revision 1.154
retrieving revision 1.155
diff -u -d -r1.154 -r1.155
--- config.c 2 Sep 2004 22:08:36 -0000 1.154
+++ config.c 8 Sep 2004 06:52:33 -0000 1.155
@@ -23,6 +23,7 @@
CONFIG_TYPE_CSV, /**< A list of strings, separated by commas and optional
* whitespace. */
CONFIG_TYPE_LINELIST, /**< Uninterpreted config lines */
+ CONFIG_TYPE_OBSOLETE, /**< Obsolete (ignored) option. */
} config_type_t;
/** Largest allowed config line */
@@ -170,6 +171,9 @@
*(struct config_line_t**)arg =
config_line_prepend(*(struct config_line_t**)arg, c->key, c->value);
break;
+ case CONFIG_TYPE_OBSOLETE:
+ log_fn(LOG_WARN, "Skipping obsolete configuration option '%s'", c->key);
+ break;
}
return 1;
}
@@ -225,7 +229,7 @@
config_compare(list, "LogLevel", CONFIG_TYPE_LINELIST, &options->LogOptions) ||
config_compare(list, "LogFile", CONFIG_TYPE_LINELIST, &options->LogOptions) ||
- config_compare(list, "LinkPadding", CONFIG_TYPE_BOOL, &options->LinkPadding) ||
+ config_compare(list, "LinkPadding", CONFIG_TYPE_OBSOLETE, NULL) ||
config_compare(list, "MaxConn", CONFIG_TYPE_INT, &options->MaxConn) ||
config_compare(list, "MaxOnionsPending",CONFIG_TYPE_INT, &options->MaxOnionsPending) ||
@@ -252,7 +256,7 @@
config_compare(list, "SocksBindAddress",CONFIG_TYPE_LINELIST,&options->SocksBindAddress) ||
config_compare(list, "SocksPolicy", CONFIG_TYPE_LINELIST,&options->SocksPolicy) ||
- config_compare(list, "TrafficShaping", CONFIG_TYPE_BOOL, &options->TrafficShaping) ||
+ config_compare(list, "TrafficShaping", CONFIG_TYPE_OBSOLETE, NULL) ||
config_compare(list, "User", CONFIG_TYPE_STRING, &options->User)
@@ -1024,7 +1028,7 @@
const char *d;
if (options->DataDirectory)
d = options->DataDirectory;
- else if (server_mode()) {
+ else {
#ifdef MS_WINDOWS
char *p;
p = tor_malloc(MAX_PATH);
@@ -1037,10 +1041,7 @@
#else
d = "~/.tor";
#endif
- } else
- d = NULL; /* XXX008 don't create datadir until we have something
- we'll be putting in it */
-
+ }
if (d && strncmp(d,"~/",2)==0) {
char *fn = expand_filename(d);
tor_free(options->DataDirectory);
Index: connection.c
===================================================================
RCS file: /home/or/cvsroot/src/or/connection.c,v
retrieving revision 1.251
retrieving revision 1.252
diff -u -d -r1.251 -r1.252
--- connection.c 2 Sep 2004 23:25:23 -0000 1.251
+++ connection.c 8 Sep 2004 06:52:33 -0000 1.252
@@ -592,26 +592,22 @@
return 0;
}
-extern int global_read_bucket;
+extern int global_read_bucket, global_write_bucket;
/** How many bytes at most can we read onto this connection? */
int connection_bucket_read_limit(connection_t *conn) {
int at_most;
- if(options.LinkPadding) {
- at_most = global_read_bucket;
+ /* do a rudimentary round-robin so one circuit can't hog a connection */
+ if(connection_speaks_cells(conn)) {
+ at_most = 32*(CELL_NETWORK_SIZE);
} else {
- /* do a rudimentary round-robin so one circuit can't hog a connection */
- if(connection_speaks_cells(conn)) {
- at_most = 32*(CELL_NETWORK_SIZE);
- } else {
- at_most = 32*(RELAY_PAYLOAD_SIZE);
- }
-
- if(at_most > global_read_bucket)
- at_most = global_read_bucket;
+ at_most = 32*(RELAY_PAYLOAD_SIZE);
}
+ if(at_most > global_read_bucket)
+ at_most = global_read_bucket;
+
if(connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN)
if(at_most > conn->receiver_bucket)
at_most = conn->receiver_bucket;
@@ -620,7 +616,7 @@
}
/** We just read num_read onto conn. Decrement buckets appropriately. */
-void connection_bucket_decrement(connection_t *conn, int num_read) {
+static void connection_read_bucket_decrement(connection_t *conn, int num_read) {
global_read_bucket -= num_read; tor_assert(global_read_bucket >= 0);
if(connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN) {
conn->receiver_bucket -= num_read; tor_assert(conn->receiver_bucket >= 0);
@@ -648,6 +644,7 @@
void connection_bucket_init(void) {
tor_gettimeofday(¤t_time);
global_read_bucket = options.BandwidthBurst; /* start it at max traffic */
+ global_write_bucket = options.BandwidthBurst; /* start it at max traffic */
}
/** Some time has passed; increment buckets appropriately. */
@@ -662,11 +659,15 @@
current_time.tv_sec = now->tv_sec; /* update current_time */
/* (ignore usecs for now) */
- /* refill the global bucket */
+ /* refill the global buckets */
if(global_read_bucket < options.BandwidthBurst) {
global_read_bucket += options.BandwidthRate;
log_fn(LOG_DEBUG,"global_read_bucket now %d.", global_read_bucket);
}
+ if(global_write_bucket < options.BandwidthBurst) {
+ global_write_bucket += options.BandwidthRate;
+ log_fn(LOG_DEBUG,"global_write_bucket now %d.", global_write_bucket);
+ }
/* refill the per-connection buckets */
get_connection_array(&carray,&n);
@@ -680,6 +681,8 @@
if(conn->wants_to_read == 1 /* it's marked to turn reading back on now */
&& global_read_bucket > 0 /* and we're allowed to read */
+ && global_write_bucket > 0 /* and we're allowed to write (XXXX,
+ * not the best place to check this.) */
&& (!connection_speaks_cells(conn) ||
conn->state != OR_CONN_STATE_OPEN ||
conn->receiver_bucket > 0)) {
@@ -815,7 +818,7 @@
rep_hist_note_bytes_read(result, time(NULL));
}
- connection_bucket_decrement(conn, result);
+ connection_read_bucket_decrement(conn, result);
return 0;
}
@@ -947,6 +950,8 @@
rep_hist_note_bytes_written(result, now);
}
+ global_write_bucket -= result;
+
if(!connection_wants_to_flush(conn)) { /* it's done flushing */
if(connection_finished_flushing(conn) < 0) {
/* already marked */
Index: directory.c
===================================================================
RCS file: /home/or/cvsroot/src/or/directory.c,v
retrieving revision 1.125
retrieving revision 1.126
diff -u -d -r1.125 -r1.126
--- directory.c 2 Sep 2004 18:57:09 -0000 1.125
+++ directory.c 8 Sep 2004 06:52:33 -0000 1.126
@@ -210,6 +210,7 @@
static void directory_send_command(connection_t *conn, int purpose,
const char *payload, int payload_len) {
char fetchwholedir[] = "GET / HTTP/1.0\r\n\r\n";
+ char fetchwholedir_z[] = "GET /dir.z HTTP/1.0\r\n\r\n";
char fetchrunninglist[] = "GET /running-routers HTTP/1.0\r\n\r\n";
char tmp[8192];
@@ -288,11 +289,12 @@
* Otherwise, return -1.
*/
static int
-parse_http_response(char *headers, int *code, char **message, time_t *date)
+parse_http_response(char *headers, int *code, char **message, time_t *date,
+ int *compression)
{
int n1, n2;
- const char *cp;
char datestr[RFC1123_TIME_LEN+1];
+ smartlist_t *parsed_headers;
tor_assert(headers && code);
while(isspace((int)*headers)) headers++; /* tolerate leading whitespace */
@@ -307,22 +309,40 @@
if(message) {
/* XXX should set *message correctly */
}
+ parsed_headers = smartlist_create();
+ smartlist_split_string(parsed_headers, headers, "\n",
+ SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
if (date) {
- cp = headers;
*date = 0;
- while (cp && (cp = strchr(cp, '\n'))) {
- ++cp;
- strlcpy(datestr, cp, 7);
- if (strcmpstart(cp, "Date: ") == 0) {
- strlcpy(datestr, cp+6, sizeof(datestr));
+ SMARTLIST_FOREACH(parsed_headers, const char *, s,
+ if (!strcmpstart(s, "Date: ")) {
+ strlcpy(datestr, s+6, sizeof(datestr));
/* This will do nothing on failure, so we don't need to check
the result. We shouldn't warn, since there are many other valid
date formats besides the one we use. */
parse_rfc1123_time(datestr, date);
break;
- }
+ });
+ }
+ if (compression) {
+ const char *enc = NULL;
+ SMARTLIST_FOREACH(parsed_headers, const char *, s,
+ if (!strcmpstart(s, "Content-Encoding: ")) {
+ enc = s+16; break;
+ });
+ if (!enc || strcmp(enc, "identity")) {
+ *compression = 0;
+ } else if (!strcmp(enc, "deflate") || !strcmp(enc, "x-deflate")) {
+ *compression = ZLIB_METHOD;
+ } else if (!strcmp(enc, "gzip") || !strcmp(enc, "x-gzip")) {
+ *compression = GZIP_METHOD;
+ } else {
+ log_fn(LOG_WARN, "Unrecognized content encoding: '%s'", enc);
+ *compression = 0;
}
}
+ SMARTLIST_FOREACH(parsed_headers, char *, s, tor_free(s));
+ smartlist_free(parsed_headers);
return 0;
}
@@ -342,6 +362,7 @@
int status_code;
time_t now, date_header=0;
int delta;
+ int compression;
switch(fetch_from_buf_http(conn->inbuf,
&headers, MAX_HEADERS_SIZE,
@@ -355,7 +376,8 @@
/* case 1, fall through */
}
- if(parse_http_response(headers, &status_code, NULL, &date_header) < 0) {
+ if(parse_http_response(headers, &status_code, NULL, &date_header,
+ &compression) < 0) {
log_fn(LOG_WARN,"Unparseable headers. Closing.");
free(body); free(headers);
return -1;
@@ -372,6 +394,19 @@
}
}
+ if (compression != 0) {
+ char *new_body;
+ size_t new_len;
+ if (tor_gzip_uncompress(&new_body, &new_len, body, body_len, compression)) {
+ log_fn(LOG_WARN, "Unable to decompress HTTP body.");
+ tor_free(body); tor_free(headers);
+ return -1;
+ }
+ tor_free(body);
+ body = new_body;
+ body_len = (int)new_len;
+ }
+
if(conn->purpose == DIR_PURPOSE_FETCH_DIR) {
/* fetch/process the directory to learn about new routers. */
log_fn(LOG_INFO,"Received directory (size %d):\n%s", body_len, body);
@@ -545,8 +580,8 @@
return 0;
}
- if(!strcmp(url,"/")) { /* directory fetch */
- dlen = dirserv_get_directory(&cp, 0);
+ if(!strcmp(url,"/") || !strcmp(url,"/dir.z")) { /* directory fetch */
+ dlen = dirserv_get_directory(&cp, !strcmp(url,"/dir.z"));
if(dlen == 0) {
log_fn(LOG_WARN,"My directory is empty. Closing.");
@@ -556,9 +591,10 @@
log_fn(LOG_DEBUG,"Dumping directory to client.");
format_rfc1123_time(date, time(NULL));
- snprintf(tmp, sizeof(tmp), "HTTP/1.0 200 OK\r\nDate: %s\r\nContent-Length: %d\r\nContent-Type: text/plain\r\n\r\n",
+ snprintf(tmp, sizeof(tmp), "HTTP/1.0 200 OK\r\nDate: %s\r\nContent-Length: %d\r\nContent-Type: text/plain\r\nContent-Encoding: %s\r\n\r\n",
date,
- (int)dlen);
+ (int)dlen,
+ strcmp(url,"/dir.z")?"identity":"deflate");
connection_write_to_buf(tmp, strlen(tmp), conn);
connection_write_to_buf(cp, strlen(cp), conn);
return 0;
Index: dirserv.c
===================================================================
RCS file: /home/or/cvsroot/src/or/dirserv.c,v
retrieving revision 1.76
retrieving revision 1.77
diff -u -d -r1.76 -r1.77
--- dirserv.c 2 Sep 2004 18:57:09 -0000 1.76
+++ dirserv.c 8 Sep 2004 06:52:33 -0000 1.77
@@ -630,10 +630,16 @@
{
time_t now;
size_t z_len;
+ char filename[512];
tor_assert(!options.AuthoritativeDir);
now = time(NULL);
- if (when>cached_directory_published &&
- when<now+ROUTER_ALLOW_SKEW) {
+ if (when<=cached_directory_published) {
+ log_fn(LOG_INFO, "Ignoring old directory; not caching.");
+ } else if (when>=now+ROUTER_ALLOW_SKEW) {
+ log_fn(LOG_INFO, "Ignoring future directory; not caching.");
+ } if (when>cached_directory_published &&
+ when<now+ROUTER_ALLOW_SKEW) {
+ log_fn(LOG_DEBUG, "Caching directory.");
tor_free(cached_directory);
cached_directory = tor_strdup(directory);
cached_directory_len = strlen(cached_directory);
@@ -644,6 +650,12 @@
log_fn(LOG_WARN,"Error compressing cached directory");
}
cached_directory_published = when;
+ if(get_data_directory(&options)) {
+ sprintf(filename,"%s/cached-directory", get_data_directory(&options));
+ if(write_str_to_file(filename,cached_directory) < 0) {
+ log_fn(LOG_WARN, "Couldn't write cached directory to disk. Ignoring.");
+ }
+ }
}
}
Index: main.c
===================================================================
RCS file: /home/or/cvsroot/src/or/main.c,v
retrieving revision 1.323
retrieving revision 1.324
diff -u -d -r1.323 -r1.324
--- main.c 2 Sep 2004 22:08:36 -0000 1.323
+++ main.c 8 Sep 2004 06:52:33 -0000 1.324
@@ -21,12 +21,17 @@
or_options_t options; /**< Command-line and config-file options. */
int global_read_bucket; /**< Max number of bytes I can read this second. */
+int global_write_bucket; /**< Max number of bytes I can write this second. */
/** What was the read bucket before the last call to prepare_for_pool?
* (used to determine how many bytes we've read). */
static int stats_prev_global_read_bucket;
-/** How many bytes have we read since we started the process? */
+/** What was the write bucket before the last call to prepare_for_pool?
+ * (used to determine how many bytes we've written). */
+static int stats_prev_global_write_bucket;
+/** How many bytes have we read/written since we started the process? */
static uint64_t stats_n_bytes_read = 0;
+static uint64_t stats_n_bytes_written = 0;
/** How many seconds have we been running? */
long stats_n_seconds_uptime = 0;
@@ -632,8 +637,10 @@
/* Check how much bandwidth we've consumed, and increment the token
* buckets. */
stats_n_bytes_read += stats_prev_global_read_bucket - global_read_bucket;
+ stats_n_bytes_written += stats_prev_global_write_bucket - global_write_bucket;
connection_bucket_refill(&now);
stats_prev_global_read_bucket = global_read_bucket;
+ stats_prev_global_write_bucket = global_write_bucket;
if(now.tv_sec > current_second) { /* the second has rolled over. check more stuff. */
@@ -698,6 +705,7 @@
/* Set up our buckets */
connection_bucket_init();
stats_prev_global_read_bucket = global_read_bucket;
+ stats_prev_global_write_bucket = global_write_bucket;
/* Finish backgrounding the process */
if(options.RunAsDaemon) {
@@ -782,15 +790,8 @@
}
/* load the routers file, or assign the defaults. */
- if(options.RouterFile) {
- routerlist_clear_trusted_directories();
- if (router_load_routerlist_from_file(options.RouterFile, 1) < 0) {
- log_fn(LOG_ERR,"Error loading router list.");
- return -1;
- }
- } else {
- if(config_assign_default_dirservers() < 0)
- return -1;
+ if(router_reload_router_list()) {
+ return -1;
}
if(authdir_mode()) {
Index: or.h
===================================================================
RCS file: /home/or/cvsroot/src/or/or.h,v
retrieving revision 1.416
retrieving revision 1.417
diff -u -d -r1.416 -r1.417
--- or.h 2 Sep 2004 18:57:09 -0000 1.416
+++ or.h 8 Sep 2004 06:52:33 -0000 1.417
@@ -867,8 +867,6 @@
int AuthoritativeDir; /**< Boolean: is this an authoritative directory? */
int ClientOnly; /**< Boolean: should we never evolve into a server role? */
int MaxConn; /**< Maximum number of simultaneous connections. */
- int TrafficShaping; /**< Unused. */
- int LinkPadding; /**< Unused. */
int IgnoreVersion; /**< If true, run no matter what versions of Tor the
* directory recommends. */
int RunAsDaemon; /**< If true, run in the background. (Unix only) */
@@ -1392,6 +1390,7 @@
/********************************* routerlist.c ***************************/
+int router_reload_router_list(void);
routerinfo_t *router_pick_directory_server(int requireauth, int requireothers);
int all_directory_servers_down(void);
struct smartlist_t;
Index: routerlist.c
===================================================================
RCS file: /home/or/cvsroot/src/or/routerlist.c,v
retrieving revision 1.134
retrieving revision 1.135
diff -u -d -r1.134 -r1.135
--- routerlist.c 2 Sep 2004 18:39:59 -0000 1.134
+++ routerlist.c 8 Sep 2004 06:52:33 -0000 1.135
@@ -40,6 +40,37 @@
extern int has_fetched_directory; /**< from main.c */
+/**
+ * Reload the original list of trusted dirservers, and the most recent
+ * cached directory (if present).
+ */
+int router_reload_router_list(void)
+{
+ char filename[512];
+ routerlist_clear_trusted_directories();
+ if (options.RouterFile) {
+ if (router_load_routerlist_from_file(options.RouterFile, 1) < 0) {
+ log_fn(LOG_ERR,"Error loading router list.");
+ return -1;
+ }
+ } else {
+ if (config_assign_default_dirservers() < 0)
+ return -1;
+ }
+ if (get_data_directory(&options)) {
+ char *s;
+ sprintf(filename,"%s/cached-directory", get_data_directory(&options));
+ s = read_file_to_str(filename);
+ if (s) {
+ log_fn(LOG_INFO, "Loading cached directory from %s", filename);
+ if (router_load_routerlist_from_string(s, 0) < 0) {
+ log_fn(LOG_WARN, "Cached directory was unparseable; ignoring.");
+ }
+ }
+ }
+ return 0;
+}
+
/** Try to find a running dirserver. If there are no running dirservers
* in our routerlist, set all the authoritative ones as running again,
* and pick one. If there are no dirservers at all in our routerlist,
@@ -63,12 +94,8 @@
options.FascistFirewall ? "reachable" : "known");
has_fetched_directory=0; /* reset it */
routerlist_clear_trusted_directories();
- if(options.RouterFile) {
- if(router_load_routerlist_from_file(options.RouterFile, 1) < 0)
- return NULL;
- } else {
- if(config_assign_default_dirservers() < 0)
- return NULL;
+ if(router_reload_router_list()) {
+ return NULL;
}
/* give it one last try */
choice = router_pick_directory_server_impl(requireauth, requireothers, 0);
More information about the tor-commits
mailing list