- tor-commits - lists.torproject.org

[tor/maint-0.2.2] Clarify some documentation and comments wrt resetting OR token buckets
by nickm＠torproject.org 07 Apr '11

07 Apr '11

commit d38030381b04263f76ce7f8ed2d6ceefc4ce363f Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Mar 25 17:21:16 2011 -0400 Clarify some documentation and comments wrt resetting OR token buckets --- src/or/connection_or.c | 6 +++++- src/or/networkstatus.c | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/or/connection_or.c b/src/or/connection_or.c index ff863fd..4b932ec 100644 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@ -353,6 +353,9 @@ connection_or_digest_is_known_relay(const char *id_digest) * per-conn limits that are big enough they'll never matter. But if it's * not a known relay, first check if we set PerConnBwRate/Burst, then * check if the consensus sets them, else default to 'big enough'. + * + * If <b>reset</b> is true, set the bucket to be full. Otherwise, just + * clip the bucket if it happens to be <em>too</em> full. */ static void connection_or_update_token_buckets_helper(or_connection_t *conn, int reset, @@ -392,7 +395,8 @@ connection_or_update_token_buckets_helper(or_connection_t *conn, int reset, } /** Either our set of relays or our per-conn rate limits have changed. - * Go through all the OR connections and update their token buckets. */ + * Go through all the OR connections and update their token buckets to make + * sure they don't exceed their maximum values. */ void connection_or_update_token_buckets(smartlist_t *conns, or_options_t *options) { diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 18cb477..4f6fe15 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -1747,7 +1747,8 @@ networkstatus_set_current_consensus(const char *consensus, routerstatus_list_update_named_server_map(); cell_ewma_set_scale_factor(options, current_consensus); - /* XXX022 where is the right place to put this call? */ + /* XXXX023 this call might be unnecessary here: can changing the + * current consensus really alter our view of any OR's rate limits? */ connection_or_update_token_buckets(get_connection_array(), options); circuit_build_times_new_consensus_params(&circ_times, current_consensus);

1 0

[tor/maint-0.2.2] Add XXX023s for our timestamp_dirty abuse.
by nickm＠torproject.org 07 Apr '11

07 Apr '11

commit f3b89c114112987b74bb0632cfe47a3a17859adb Author: Nick Mathewson <nickm(a)torproject.org> Date: Fri Mar 25 17:49:44 2011 -0400 Add XXX023s for our timestamp_dirty abuse. --- src/or/circuitlist.c | 3 +++ src/or/connection_edge.c | 2 ++ src/or/or.h | 10 ++++++++-- src/or/relay.c | 2 ++ 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index b4f5f45..521c21e 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -1023,6 +1023,7 @@ circuit_mark_all_unused_circs(void) * This is useful for letting the user change pseudonyms, so new * streams will not be linkable to old streams. */ +/* XXX023 this is a bad name for what this function does */ void circuit_expire_all_dirty_circs(void) { @@ -1033,6 +1034,8 @@ circuit_expire_all_dirty_circs(void) if (CIRCUIT_IS_ORIGIN(circ) && !circ->marked_for_close && circ->timestamp_dirty) + /* XXXX023 This is a screwed-up way to say "This is too dirty + * for new circuits. */ circ->timestamp_dirty -= options->MaxCircuitDirtiness; } } diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index c75c06b..af0cfbe 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -517,6 +517,7 @@ connection_ap_expire_beginning(void) /* kludge to make us not try this circuit again, yet to allow * current streams on it to survive if they can: make it * unattractive to use for new streams */ + /* XXXX023 this is a kludgy way to do this. */ tor_assert(circ->timestamp_dirty); circ->timestamp_dirty -= options->MaxCircuitDirtiness; /* give our stream another 'cutoff' seconds to try */ @@ -2164,6 +2165,7 @@ connection_ap_handshake_send_begin(edge_connection_t *ap_conn) ap_conn->stream_id = get_unique_stream_id_by_circ(circ); if (ap_conn->stream_id==0) { connection_mark_unattached_ap(ap_conn, END_STREAM_REASON_INTERNAL); + /*XXXX022 _close_ the circuit because it's full? That sounds dumb. */ circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_RESOURCELIMIT); return -1; } diff --git a/src/or/or.h b/src/or/or.h index 124892c..e44c626 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -2127,8 +2127,14 @@ typedef struct circuit_t { char *n_conn_onionskin; time_t timestamp_created; /**< When was this circuit created? */ - time_t timestamp_dirty; /**< When the circuit was first used, or 0 if the - * circuit is clean. */ + /** When the circuit was first used, or 0 if the circuit is clean. + * + * XXXX023 Note that some code will artifically adjust this value backward + * in time in order to indicate that a circuit shouldn't be used for new + * streams, but that it can stay alive as long as it has streams on it. + * That's a kludge we should fix. + */ + time_t timestamp_dirty; struct timeval highres_created; /**< When exactly was the circuit created? */ uint16_t marked_for_close; /**< Should we close this circuit at the end of diff --git a/src/or/relay.c b/src/or/relay.c index 076b46b..807cb3d 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -791,6 +791,8 @@ connection_ap_process_end_not_open( < MAX_RESOLVE_FAILURES) { /* We haven't retried too many times; reattach the connection. */ circuit_log_path(LOG_INFO,LD_APP,circ); + /* Mark this circuit "unusable for new streams". */ + /* XXXX023 this is a kludgy way to do this. */ tor_assert(circ->_base.timestamp_dirty); circ->_base.timestamp_dirty -= get_options()->MaxCircuitDirtiness;

1 0

[tor/master] Allow controllers a more up-to-date view of bridge usage.
by nickm＠torproject.org 07 Apr '11

07 Apr '11

commit 118d8ffdcb74137a36d22928ce6f46897809391e Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Mar 15 22:39:22 2011 -0400 Allow controllers a more up-to-date view of bridge usage. Instead of answering GETINFO requests about our geoip usage only after running for 24 hours, this patch makes us answer GETINFO requests immediately. We still round and quantize as before. Implements bug2711. Also, refactor the heck out of the bridge usage formatting code. No longer should we need to do a generate-parse-and-regenerate cycle to get the controller string, and that lets us simplify the code a lot. --- changes/feature2711 | 4 + src/or/control.c | 4 +- src/or/geoip.c | 172 ++++++++++++++++++++++++++++----------------------- src/or/geoip.h | 2 +- 4 files changed, 101 insertions(+), 81 deletions(-) diff --git a/changes/feature2711 b/changes/feature2711 new file mode 100644 index 0000000..7cdcfbf --- /dev/null +++ b/changes/feature2711 @@ -0,0 +1,4 @@ + o Minor features + - Export GeoIP information on usage to bridge controller even if we have + not yet been running for 24 hours. + diff --git a/src/or/control.c b/src/or/control.c index fb9c1ae..8f3af0b 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -1785,12 +1785,12 @@ getinfo_helper_events(control_connection_t *control_conn, "information", question); } } else if (!strcmp(question, "status/clients-seen")) { - const char *bridge_stats = geoip_get_bridge_stats_controller(time(NULL)); + char *bridge_stats = geoip_get_bridge_stats_controller(time(NULL)); if (!bridge_stats) { *errmsg = "No bridge-client stats available"; return -1; } - *answer = tor_strdup(bridge_stats); + *answer = bridge_stats; } else { return 0; } diff --git a/src/or/geoip.c b/src/or/geoip.c index 654241c..28f570a 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -1083,14 +1083,14 @@ geoip_bridge_stats_term(void) start_of_bridge_stats_interval = 0; } -/** Parse the bridge statistics as they are written to extra-info - * descriptors for being returned to controller clients. Return the - * controller string if successful, or NULL otherwise. */ -static char * -parse_bridge_stats_controller(const char *stats_str, time_t now) +/** Validate a bridge statistics string as it would be written to a + * current extra-info descriptor. Return 1 if the string is valid and + * recent enough, or 0 otherwise. */ +static int +validate_bridge_stats(const char *stats_str, time_t now) { char stats_end_str[ISO_TIME_LEN+1], stats_start_str[ISO_TIME_LEN+1], - *controller_str, *eos, *eol, *summary; + *eos; const char *BRIDGE_STATS_END = "bridge-stats-end "; const char *BRIDGE_IPS = "bridge-ips "; @@ -1104,63 +1104,90 @@ parse_bridge_stats_controller(const char *stats_str, time_t now) "bridge-stats-end YYYY-MM-DD HH:MM:SS (N s)" */ tmp = find_str_at_start_of_line(stats_str, BRIDGE_STATS_END); if (!tmp) - return NULL; + return 0; tmp += strlen(BRIDGE_STATS_END); if (strlen(tmp) < ISO_TIME_LEN + 6) - return NULL; + return 0; strlcpy(stats_end_str, tmp, sizeof(stats_end_str)); if (parse_iso_time(stats_end_str, &stats_end_time) < 0) - return NULL; + return 0; if (stats_end_time < now - (25*60*60) || stats_end_time > now + (1*60*60)) - return NULL; + return 0; seconds = (int)strtol(tmp + ISO_TIME_LEN + 2, &eos, 10); if (!eos || seconds < 23*60*60) - return NULL; + return 0; format_iso_time(stats_start_str, stats_end_time - seconds); /* Parse: "bridge-ips CC=N,CC=N,..." */ tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS); - if (tmp) { - tmp += strlen(BRIDGE_IPS); - tmp = eat_whitespace_no_nl(tmp); - eol = strchr(tmp, '\n'); - if (eol) - summary = tor_strndup(tmp, eol-tmp); - else - summary = tor_strdup(tmp); - } else { + if (!tmp) { /* Look if there is an empty "bridge-ips" line */ tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS_EMPTY_LINE); if (!tmp) - return NULL; - summary = tor_strdup(""); + return 0; } - tor_asprintf(&controller_str, - "TimeStarted=\"%s\" CountrySummary=%s", - stats_start_str, summary); - tor_free(summary); - return controller_str; + return 1; } /** Most recent bridge statistics formatted to be written to extra-info * descriptors. */ static char *bridge_stats_extrainfo = NULL; -/** Most recent bridge statistics formatted to be returned to controller - * clients. */ -static char *bridge_stats_controller = NULL; +/** Return a newly allocated string holding our bridge usage stats by country + * in a format suitable for inclusion in an extrainfo document. Return NULL on + * failure. */ +static char * +format_bridge_stats_extrainfo(time_t now) +{ + char *out = NULL, *data = NULL; + long duration = now - start_of_bridge_stats_interval; + char written[ISO_TIME_LEN+1]; + + if (duration < 0) + return NULL; + + format_iso_time(written, now); + data = geoip_get_client_history(GEOIP_CLIENT_CONNECT); + + tor_asprintf(&out, + "bridge-stats-end %s (%ld s)\n" + "bridge-ips %s\n", + written, duration, + data ? data : ""); + tor_free(data); + + return out; +} + +/** Return a newly allocated string holding our bridge usage stats by country + * in a format suitable for the answer to a controller request. Return NULL on + * failure. */ +static char * +format_bridge_stats_controller(time_t now) +{ + char *out = NULL, *data = NULL; + char started[ISO_TIME_LEN+1]; + (void) now; + + format_iso_time(started, start_of_bridge_stats_interval); + data = geoip_get_client_history(GEOIP_CLIENT_CONNECT); + + tor_asprintf(&out, + "TimeStarted=\"%s\" CountrySummary=%s", + started, data ? data : ""); + tor_free(data); + return out; +} /** Write bridge statistics to $DATADIR/stats/bridge-stats and return * when we should next try to write statistics. */ time_t geoip_bridge_stats_write(time_t now) { - char *statsdir = NULL, *filename = NULL, *data = NULL, - written[ISO_TIME_LEN+1], *out = NULL, *controller_str; - size_t len; + char *filename = NULL, *val = NULL, *statsdir = NULL; /* Check if 24 hours have passed since starting measurements. */ if (now < start_of_bridge_stats_interval + WRITE_STATS_INTERVAL) @@ -1169,64 +1196,54 @@ geoip_bridge_stats_write(time_t now) /* Discard all items in the client history that are too old. */ geoip_remove_old_clients(start_of_bridge_stats_interval); + /* Generate formatted string */ + val = format_bridge_stats_extrainfo(now); + if (val == NULL) + goto done; + + /* Update the stored value. */ + tor_free(bridge_stats_extrainfo); + bridge_stats_extrainfo = val; + start_of_bridge_stats_interval = now; + + /* Write it to disk. */ statsdir = get_datadir_fname("stats"); if (check_private_dir(statsdir, CPD_CREATE) < 0) goto done; filename = get_datadir_fname2("stats", "bridge-stats"); - data = geoip_get_client_history(GEOIP_CLIENT_CONNECT); - format_iso_time(written, now); - len = strlen("bridge-stats-end (999999 s)\nbridge-ips \n") + - ISO_TIME_LEN + (data ? strlen(data) : 0) + 42; - out = tor_malloc(len); - if (tor_snprintf(out, len, "bridge-stats-end %s (%u s)\nbridge-ips %s\n", - written, (unsigned) (now - start_of_bridge_stats_interval), - data ? data : "") < 0) - goto done; - write_str_to_file(filename, out, 0); - controller_str = parse_bridge_stats_controller(out, now); - if (!controller_str) - goto done; - start_of_bridge_stats_interval = now; - tor_free(bridge_stats_extrainfo); - tor_free(bridge_stats_controller); - bridge_stats_extrainfo = out; - out = NULL; - bridge_stats_controller = controller_str; - control_event_clients_seen(controller_str); + + write_str_to_file(filename, bridge_stats_extrainfo, 0); + + /* Tell the controller, "hey, there are clients!" */ + { + char *controller_str = format_bridge_stats_controller(now); + if (controller_str) + control_event_clients_seen(controller_str); + tor_free(controller_str); + } done: tor_free(filename); tor_free(statsdir); - tor_free(data); - tor_free(out); - return start_of_bridge_stats_interval + - WRITE_STATS_INTERVAL; + + return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL; } /** Try to load the most recent bridge statistics from disk, unless we - * have finished a measurement interval lately. */ + * have finished a measurement interval lately, and check whether they + * are still recent enough. */ static void load_bridge_stats(time_t now) { - char *statsdir, *fname=NULL, *contents, *controller_str; + char *fname, *contents; if (bridge_stats_extrainfo) return; - statsdir = get_datadir_fname("stats"); - if (check_private_dir(statsdir, CPD_CREATE) < 0) - goto done; + fname = get_datadir_fname2("stats", "bridge-stats"); contents = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL); - if (contents) { - controller_str = parse_bridge_stats_controller(contents, now); - if (controller_str) { - bridge_stats_extrainfo = contents; - bridge_stats_controller = controller_str; - } else { - tor_free(contents); - } - } - done: + if (contents && validate_bridge_stats(contents, now)) + bridge_stats_extrainfo = contents; + tor_free(fname); - tor_free(statsdir); } /** Return most recent bridge statistics for inclusion in extra-info @@ -1238,13 +1255,12 @@ geoip_get_bridge_stats_extrainfo(time_t now) return bridge_stats_extrainfo; } -/** Return most recent bridge statistics to be returned to controller - * clients, or NULL if we don't have recent bridge statistics. */ -const char * +/** Return a new string containing the recent bridge statistics to be returned + * to controller clients, or NULL if we don't have any bridge statistics. */ +char * geoip_get_bridge_stats_controller(time_t now) { - load_bridge_stats(now); - return bridge_stats_controller; + return format_bridge_stats_controller(now); } /** Start time of entry stats or 0 if we're not collecting entry diff --git a/src/or/geoip.h b/src/or/geoip.h index bafbeea..24f7c5b 100644 --- a/src/or/geoip.h +++ b/src/or/geoip.h @@ -51,7 +51,7 @@ void geoip_bridge_stats_init(time_t now); time_t geoip_bridge_stats_write(time_t now); void geoip_bridge_stats_term(void); const char *geoip_get_bridge_stats_extrainfo(time_t); -const char *geoip_get_bridge_stats_controller(time_t); +char *geoip_get_bridge_stats_controller(time_t); #endif

1 0

[tor/master] Merge remote-tracking branch 'origin/maint-0.2.2'
by nickm＠torproject.org 07 Apr '11

07 Apr '11

commit 07ab483e62457ee56187037304f25ec50cb0850a Merge: 09759a6 118d8ff Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Apr 7 11:55:17 2011 -0400 Merge remote-tracking branch 'origin/maint-0.2.2' changes/feature2711 | 4 + src/or/control.c | 4 +- src/or/geoip.c | 172 ++++++++++++++++++++++++++++----------------------- src/or/geoip.h | 2 +- 4 files changed, 101 insertions(+), 81 deletions(-)

1 0

[tor/maint-0.2.2] Allow controllers a more up-to-date view of bridge usage.
by nickm＠torproject.org 07 Apr '11

07 Apr '11

commit 118d8ffdcb74137a36d22928ce6f46897809391e Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Mar 15 22:39:22 2011 -0400 Allow controllers a more up-to-date view of bridge usage. Instead of answering GETINFO requests about our geoip usage only after running for 24 hours, this patch makes us answer GETINFO requests immediately. We still round and quantize as before. Implements bug2711. Also, refactor the heck out of the bridge usage formatting code. No longer should we need to do a generate-parse-and-regenerate cycle to get the controller string, and that lets us simplify the code a lot. --- changes/feature2711 | 4 + src/or/control.c | 4 +- src/or/geoip.c | 172 ++++++++++++++++++++++++++++----------------------- src/or/geoip.h | 2 +- 4 files changed, 101 insertions(+), 81 deletions(-) diff --git a/changes/feature2711 b/changes/feature2711 new file mode 100644 index 0000000..7cdcfbf --- /dev/null +++ b/changes/feature2711 @@ -0,0 +1,4 @@ + o Minor features + - Export GeoIP information on usage to bridge controller even if we have + not yet been running for 24 hours. + diff --git a/src/or/control.c b/src/or/control.c index fb9c1ae..8f3af0b 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -1785,12 +1785,12 @@ getinfo_helper_events(control_connection_t *control_conn, "information", question); } } else if (!strcmp(question, "status/clients-seen")) { - const char *bridge_stats = geoip_get_bridge_stats_controller(time(NULL)); + char *bridge_stats = geoip_get_bridge_stats_controller(time(NULL)); if (!bridge_stats) { *errmsg = "No bridge-client stats available"; return -1; } - *answer = tor_strdup(bridge_stats); + *answer = bridge_stats; } else { return 0; } diff --git a/src/or/geoip.c b/src/or/geoip.c index 654241c..28f570a 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -1083,14 +1083,14 @@ geoip_bridge_stats_term(void) start_of_bridge_stats_interval = 0; } -/** Parse the bridge statistics as they are written to extra-info - * descriptors for being returned to controller clients. Return the - * controller string if successful, or NULL otherwise. */ -static char * -parse_bridge_stats_controller(const char *stats_str, time_t now) +/** Validate a bridge statistics string as it would be written to a + * current extra-info descriptor. Return 1 if the string is valid and + * recent enough, or 0 otherwise. */ +static int +validate_bridge_stats(const char *stats_str, time_t now) { char stats_end_str[ISO_TIME_LEN+1], stats_start_str[ISO_TIME_LEN+1], - *controller_str, *eos, *eol, *summary; + *eos; const char *BRIDGE_STATS_END = "bridge-stats-end "; const char *BRIDGE_IPS = "bridge-ips "; @@ -1104,63 +1104,90 @@ parse_bridge_stats_controller(const char *stats_str, time_t now) "bridge-stats-end YYYY-MM-DD HH:MM:SS (N s)" */ tmp = find_str_at_start_of_line(stats_str, BRIDGE_STATS_END); if (!tmp) - return NULL; + return 0; tmp += strlen(BRIDGE_STATS_END); if (strlen(tmp) < ISO_TIME_LEN + 6) - return NULL; + return 0; strlcpy(stats_end_str, tmp, sizeof(stats_end_str)); if (parse_iso_time(stats_end_str, &stats_end_time) < 0) - return NULL; + return 0; if (stats_end_time < now - (25*60*60) || stats_end_time > now + (1*60*60)) - return NULL; + return 0; seconds = (int)strtol(tmp + ISO_TIME_LEN + 2, &eos, 10); if (!eos || seconds < 23*60*60) - return NULL; + return 0; format_iso_time(stats_start_str, stats_end_time - seconds); /* Parse: "bridge-ips CC=N,CC=N,..." */ tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS); - if (tmp) { - tmp += strlen(BRIDGE_IPS); - tmp = eat_whitespace_no_nl(tmp); - eol = strchr(tmp, '\n'); - if (eol) - summary = tor_strndup(tmp, eol-tmp); - else - summary = tor_strdup(tmp); - } else { + if (!tmp) { /* Look if there is an empty "bridge-ips" line */ tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS_EMPTY_LINE); if (!tmp) - return NULL; - summary = tor_strdup(""); + return 0; } - tor_asprintf(&controller_str, - "TimeStarted=\"%s\" CountrySummary=%s", - stats_start_str, summary); - tor_free(summary); - return controller_str; + return 1; } /** Most recent bridge statistics formatted to be written to extra-info * descriptors. */ static char *bridge_stats_extrainfo = NULL; -/** Most recent bridge statistics formatted to be returned to controller - * clients. */ -static char *bridge_stats_controller = NULL; +/** Return a newly allocated string holding our bridge usage stats by country + * in a format suitable for inclusion in an extrainfo document. Return NULL on + * failure. */ +static char * +format_bridge_stats_extrainfo(time_t now) +{ + char *out = NULL, *data = NULL; + long duration = now - start_of_bridge_stats_interval; + char written[ISO_TIME_LEN+1]; + + if (duration < 0) + return NULL; + + format_iso_time(written, now); + data = geoip_get_client_history(GEOIP_CLIENT_CONNECT); + + tor_asprintf(&out, + "bridge-stats-end %s (%ld s)\n" + "bridge-ips %s\n", + written, duration, + data ? data : ""); + tor_free(data); + + return out; +} + +/** Return a newly allocated string holding our bridge usage stats by country + * in a format suitable for the answer to a controller request. Return NULL on + * failure. */ +static char * +format_bridge_stats_controller(time_t now) +{ + char *out = NULL, *data = NULL; + char started[ISO_TIME_LEN+1]; + (void) now; + + format_iso_time(started, start_of_bridge_stats_interval); + data = geoip_get_client_history(GEOIP_CLIENT_CONNECT); + + tor_asprintf(&out, + "TimeStarted=\"%s\" CountrySummary=%s", + started, data ? data : ""); + tor_free(data); + return out; +} /** Write bridge statistics to $DATADIR/stats/bridge-stats and return * when we should next try to write statistics. */ time_t geoip_bridge_stats_write(time_t now) { - char *statsdir = NULL, *filename = NULL, *data = NULL, - written[ISO_TIME_LEN+1], *out = NULL, *controller_str; - size_t len; + char *filename = NULL, *val = NULL, *statsdir = NULL; /* Check if 24 hours have passed since starting measurements. */ if (now < start_of_bridge_stats_interval + WRITE_STATS_INTERVAL) @@ -1169,64 +1196,54 @@ geoip_bridge_stats_write(time_t now) /* Discard all items in the client history that are too old. */ geoip_remove_old_clients(start_of_bridge_stats_interval); + /* Generate formatted string */ + val = format_bridge_stats_extrainfo(now); + if (val == NULL) + goto done; + + /* Update the stored value. */ + tor_free(bridge_stats_extrainfo); + bridge_stats_extrainfo = val; + start_of_bridge_stats_interval = now; + + /* Write it to disk. */ statsdir = get_datadir_fname("stats"); if (check_private_dir(statsdir, CPD_CREATE) < 0) goto done; filename = get_datadir_fname2("stats", "bridge-stats"); - data = geoip_get_client_history(GEOIP_CLIENT_CONNECT); - format_iso_time(written, now); - len = strlen("bridge-stats-end (999999 s)\nbridge-ips \n") + - ISO_TIME_LEN + (data ? strlen(data) : 0) + 42; - out = tor_malloc(len); - if (tor_snprintf(out, len, "bridge-stats-end %s (%u s)\nbridge-ips %s\n", - written, (unsigned) (now - start_of_bridge_stats_interval), - data ? data : "") < 0) - goto done; - write_str_to_file(filename, out, 0); - controller_str = parse_bridge_stats_controller(out, now); - if (!controller_str) - goto done; - start_of_bridge_stats_interval = now; - tor_free(bridge_stats_extrainfo); - tor_free(bridge_stats_controller); - bridge_stats_extrainfo = out; - out = NULL; - bridge_stats_controller = controller_str; - control_event_clients_seen(controller_str); + + write_str_to_file(filename, bridge_stats_extrainfo, 0); + + /* Tell the controller, "hey, there are clients!" */ + { + char *controller_str = format_bridge_stats_controller(now); + if (controller_str) + control_event_clients_seen(controller_str); + tor_free(controller_str); + } done: tor_free(filename); tor_free(statsdir); - tor_free(data); - tor_free(out); - return start_of_bridge_stats_interval + - WRITE_STATS_INTERVAL; + + return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL; } /** Try to load the most recent bridge statistics from disk, unless we - * have finished a measurement interval lately. */ + * have finished a measurement interval lately, and check whether they + * are still recent enough. */ static void load_bridge_stats(time_t now) { - char *statsdir, *fname=NULL, *contents, *controller_str; + char *fname, *contents; if (bridge_stats_extrainfo) return; - statsdir = get_datadir_fname("stats"); - if (check_private_dir(statsdir, CPD_CREATE) < 0) - goto done; + fname = get_datadir_fname2("stats", "bridge-stats"); contents = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL); - if (contents) { - controller_str = parse_bridge_stats_controller(contents, now); - if (controller_str) { - bridge_stats_extrainfo = contents; - bridge_stats_controller = controller_str; - } else { - tor_free(contents); - } - } - done: + if (contents && validate_bridge_stats(contents, now)) + bridge_stats_extrainfo = contents; + tor_free(fname); - tor_free(statsdir); } /** Return most recent bridge statistics for inclusion in extra-info @@ -1238,13 +1255,12 @@ geoip_get_bridge_stats_extrainfo(time_t now) return bridge_stats_extrainfo; } -/** Return most recent bridge statistics to be returned to controller - * clients, or NULL if we don't have recent bridge statistics. */ -const char * +/** Return a new string containing the recent bridge statistics to be returned + * to controller clients, or NULL if we don't have any bridge statistics. */ +char * geoip_get_bridge_stats_controller(time_t now) { - load_bridge_stats(now); - return bridge_stats_controller; + return format_bridge_stats_controller(now); } /** Start time of entry stats or 0 if we're not collecting entry diff --git a/src/or/geoip.h b/src/or/geoip.h index bafbeea..24f7c5b 100644 --- a/src/or/geoip.h +++ b/src/or/geoip.h @@ -51,7 +51,7 @@ void geoip_bridge_stats_init(time_t now); time_t geoip_bridge_stats_write(time_t now); void geoip_bridge_stats_term(void); const char *geoip_get_bridge_stats_extrainfo(time_t); -const char *geoip_get_bridge_stats_controller(time_t); +char *geoip_get_bridge_stats_controller(time_t); #endif

1 0

[tor/master] Fix a compile warning on OS X 10.6
by nickm＠torproject.org 07 Apr '11

07 Apr '11

commit 286d44402e829488abe802efd1607eabbc98f1d1 Author: Sebastian Hahn <sebastian(a)torproject.org> Date: Thu Apr 7 05:30:49 2011 +0200 Fix a compile warning on OS X 10.6 --- src/or/circuituse.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/src/or/circuituse.c b/src/or/circuituse.c index 447ec84..3d5db98 100644 --- a/src/or/circuituse.c +++ b/src/or/circuituse.c @@ -291,7 +291,7 @@ circuit_expire_building(void) long ms = tor_lround(msec); \ struct timeval diff; \ diff.tv_sec = ms / 1000; \ - diff.tv_usec = (ms % 1000) * 1000; \ + diff.tv_usec = (int)((ms % 1000) * 1000); \ timersub(&now, &diff, &target); \ } while (0)

1 0

[tor/master] Merge remote-tracking branch 'origin/maint-0.2.2'
by nickm＠torproject.org 07 Apr '11

07 Apr '11

commit 09759a66eae044e640f477ebe96b1c3edbb98c94 Merge: 53d02a6 286d444 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Apr 7 11:43:36 2011 -0400 Merge remote-tracking branch 'origin/maint-0.2.2' src/or/circuituse.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-)

1 0

[tor/maint-0.2.2] Fix a compile warning on OS X 10.6
by nickm＠torproject.org 07 Apr '11

07 Apr '11

commit 286d44402e829488abe802efd1607eabbc98f1d1 Author: Sebastian Hahn <sebastian(a)torproject.org> Date: Thu Apr 7 05:30:49 2011 +0200 Fix a compile warning on OS X 10.6 --- src/or/circuituse.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/src/or/circuituse.c b/src/or/circuituse.c index 447ec84..3d5db98 100644 --- a/src/or/circuituse.c +++ b/src/or/circuituse.c @@ -291,7 +291,7 @@ circuit_expire_building(void) long ms = tor_lround(msec); \ struct timeval diff; \ diff.tv_sec = ms / 1000; \ - diff.tv_usec = (ms % 1000) * 1000; \ + diff.tv_usec = (int)((ms % 1000) * 1000); \ timersub(&now, &diff, &target); \ } while (0)

1 0

r24572: {arm} Fixing man page typo (caught by tagnaq) (arm/trunk)
by Damian Johnson 07 Apr '11

07 Apr '11

Author: atagar Date: 2011-04-07 14:20:26 +0000 (Thu, 07 Apr 2011) New Revision: 24572 Modified: arm/trunk/arm.1 Log: Fixing man page typo (caught by tagnaq) Modified: arm/trunk/arm.1 =================================================================== --- arm/trunk/arm.1 2011-04-07 01:30:11 UTC (rev 24571) +++ arm/trunk/arm.1 2011-04-07 14:20:26 UTC (rev 24572) @@ -53,7 +53,7 @@ 67890 torctl runlevel+ U Unknown Events .TP -\fB\-v\fR, \fB\-\-verion\fR +\fB\-v\fR, \fB\-\-version\fR provides version information .TP

1 0

[metrics-web/master] Add GeoIP database to generate fancy relay visualizations.
by karsten＠torproject.org 07 Apr '11

07 Apr '11

commit 4c34249760470099475e9830a3b8255b2aa5f3a3 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Apr 4 21:09:40 2011 +0200 Add GeoIP database to generate fancy relay visualizations. --- build.xml | 9 ++ db/tordir.sql | 50 ++++++++++ rserve/csv.R | 12 +++ .../ernie/cron/GeoipDatabaseImporter.java | 95 ++++++++++++++++++++ src/org/torproject/ernie/web/CsvServlet.java | 1 + 5 files changed, 167 insertions(+), 0 deletions(-) diff --git a/build.xml b/build.xml index bd06daf..4b4a7e9 100644 --- a/build.xml +++ b/build.xml @@ -42,6 +42,15 @@ </javac> </target> +  + <target name="geoipdb" depends="compile"> + <java fork="true" + maxmemory="1024m" + classname="org.torproject.ernie.cron.GeoipDatabaseImporter"> + <classpath refid="classpath"/> + </java> + </target> +  <target name="run" depends="compile"> <java fork="true" diff --git a/db/tordir.sql b/db/tordir.sql index c47d19a..d5238ea 100644 --- a/db/tordir.sql +++ b/db/tordir.sql @@ -239,6 +239,56 @@ CREATE TABLE updates ( date DATE ); +-- GeoIP database that helps resolve IP addresses to country codes, +-- latitudes, and longitudes. +CREATE TABLE geoipdb ( + id SERIAL, + ipstart INET, + ipend INET, + country CHARACTER(2) NOT NULL, + latitude NUMERIC(7, 4) NOT NULL, + longitude NUMERIC(7, 4) NOT NULL +); + +-- Indexes to speed up looking up IP addresses in the GeoIP database. +CREATE INDEX geoip_ipstart ON geoipdb (ipstart); +CREATE INDEX geoip_ipend ON geoipdb (ipend); + +-- Result type for GeoIP lookups that encapsulates the country code, +-- latitude, and longitude. +CREATE TYPE geoip_result AS (country CHARACTER(2), + latitude NUMERIC(7, 4), longitude NUMERIC(7, 4)); + +-- GeoIP database lookup function. +CREATE OR REPLACE FUNCTION geoip_lookup (CHARACTER VARYING) +RETURNS geoip_result AS $$ + SELECT country, latitude, longitude FROM geoipdb + WHERE INET($1) + BETWEEN geoipdb.ipstart AND geoipdb.ipend LIMIT 1; +$$ LANGUAGE SQL; + +-- View that contains the relays of the first known consensuses of all +-- months in the database. +CREATE OR REPLACE VIEW relays_monthly_snapshots AS +SELECT status.validafter, status.fingerprint, status.nickname, + status.address, (status.geoip).country, (status.geoip).latitude, + (status.geoip).longitude, status.isexit, status.isfast, + status.isguard, status.isstable, status.version, status.ports, + descriptor.bandwidthavg, descriptor.bandwidthburst, + descriptor.bandwidthobserved +FROM ( + SELECT *, geoip_lookup(address) AS geoip + FROM statusentry + WHERE validafter IN ( + SELECT MIN(validafter) + FROM consensus + GROUP BY DATE_TRUNC('month', validafter) + ) +) AS status +LEFT JOIN descriptor +ON status.descriptor = descriptor.descriptor +ORDER BY validafter, fingerprint; + CREATE LANGUAGE plpgsql; -- FUNCTION refresh_relay_statuses_per_day() diff --git a/rserve/csv.R b/rserve/csv.R index 44138ac..e68971c 100644 --- a/rserve/csv.R +++ b/rserve/csv.R @@ -267,3 +267,15 @@ export_connbidirect <- function(path) { quote = FALSE, row.names = FALSE) } +export_relays_monthly_snapshots <- function(path) { + drv <- dbDriver("PostgreSQL") + con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) + q <- paste("SELECT * FROM relays_monthly_snapshots") + rs <- dbSendQuery(con, q) + c <- fetch(rs, n = -1) + dbDisconnect(con) + dbUnloadDriver(drv) + write.csv(format(c, trim = TRUE, scientific = FALSE), path, + quote = FALSE, row.names = FALSE) +} + diff --git a/src/org/torproject/ernie/cron/GeoipDatabaseImporter.java b/src/org/torproject/ernie/cron/GeoipDatabaseImporter.java new file mode 100644 index 0000000..afe5245 --- /dev/null +++ b/src/org/torproject/ernie/cron/GeoipDatabaseImporter.java @@ -0,0 +1,95 @@ +/* Copyright 2011 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.ernie.cron; + +import java.io.*; +import java.sql.*; +import java.util.*; +import java.util.logging.*; + +/** + * Import a Maxmind GeoLite City database to resolve resolve IP addresses + * to country codes, latitudes, and longitudes. + */ +public class GeoipDatabaseImporter { + public static void main(String[] args) throws IOException, + SQLException { + + /* Check if the GeoIP database files are in place. */ + File locationsFile = new File("GeoLiteCity-Location.csv"), + blocksFile = new File("GeoLiteCity-Blocks.csv"); + if (!locationsFile.exists() || !blocksFile.exists()) { + System.out.println("Could not find GeoLiteCity-Location.csv and/or " + + "GeoLiteCity-Blocks.csv in the working directory! Exiting!"); + System.exit(1); + } + + /* Initialize configuration to learn JDBC string. */ + Configuration config = new Configuration(); + String jdbcString = config.getRelayDescriptorDatabaseJDBC(); + + /* Connect to database. */ + Connection c = DriverManager.getConnection(jdbcString); + + /* Start by reading location information to memory. */ + BufferedReader br = new BufferedReader(new FileReader(locationsFile)); + String line; + Map<Integer, String> locations = new HashMap<Integer, String>(); + while ((line = br.readLine()) != null) { + if (line.startsWith("Copyright") || line.startsWith("locId")) { + continue; + } + String[] parts = line.split(","); + int locId = Integer.parseInt(parts[0]); + String country = parts[1].replaceAll("\"", ""); + String latitude = parts[5]; + String longitude = parts[6]; + locations.put(locId, country + "," + latitude + "," + longitude); + } + br.close(); + + /* Parse block information and add it to the database together with + * the location information. */ + PreparedStatement ps = c.prepareStatement("INSERT INTO geoipdb " + + "(ipstart, ipend, country, latitude, longitude) VALUES " + + "(?, ?, ?, ?, ?)"); + Statement s = c.createStatement(); + s.execute("DELETE FROM geoipdb"); + /* TODO The import takes 30+ minutes. Perform the import in a single + * transaction, or requests will return strange results in these 30+ + * minutes. */ + br = new BufferedReader(new FileReader(blocksFile)); + while ((line = br.readLine()) != null) { + if (line.startsWith("Copyright") || + line.startsWith("startIpNum")) { + continue; + } + String[] parts = line.replaceAll("\"", "").split(","); + long startIpNum = Long.parseLong(parts[0]); + String startIp = "" + startIpNum / 256 / 256 / 256 + "." + + startIpNum / 256 / 256 % 256 + "." + startIpNum / 256 % 256 + + "." + startIpNum % 256; + long endIpNum = Long.parseLong(parts[1]); + String endIp = "" + endIpNum / 256 / 256 / 256 + "." + + endIpNum / 256 / 256 % 256 + "." + endIpNum / 256 % 256 + "." + + endIpNum % 256; + int locId = Integer.parseInt(parts[2]); + if (!locations.containsKey(locId)) { + System.out.println("Cannot find locId=" + locId + + " in locations file!"); + continue; + } + String[] locationParts = locations.get(locId).split(","); + String country = locationParts[0]; + double latitude = Double.parseDouble(locationParts[1]); + double longitude = Double.parseDouble(locationParts[2]); + ps.setObject(1, startIp, Types.OTHER); + ps.setObject(2, endIp, Types.OTHER); + ps.setString(3, country); + ps.setDouble(4, latitude); + ps.setDouble(5, longitude); + ps.execute(); + } + } +} + diff --git a/src/org/torproject/ernie/web/CsvServlet.java b/src/org/torproject/ernie/web/CsvServlet.java index 33c098f..c68a2d0 100644 --- a/src/org/torproject/ernie/web/CsvServlet.java +++ b/src/org/torproject/ernie/web/CsvServlet.java @@ -40,6 +40,7 @@ public class CsvServlet extends HttpServlet { this.availableCsvFiles.add("monthly-users-peak"); this.availableCsvFiles.add("networksize"); this.availableCsvFiles.add("platforms"); + this.availableCsvFiles.add("relays-monthly-snapshots"); this.availableCsvFiles.add("relayflags"); this.availableCsvFiles.add("relayflags-hour"); this.availableCsvFiles.add("torperf");

1 0