September 2013 - tor-commits - lists.torproject.org

[metrics-web/master] Extend censorship detector to new user estimates.
by karsten＠torproject.org 16 Sep '13

16 Sep '13

commit 41565da93ef5cc3e562f21b8695dbf251b55459a Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Sep 16 19:17:27 2013 +0200 Extend censorship detector to new user estimates. --- detector/detector.py | 30 ++++++----- detector/detector.sh | 1 + rserve/csv.R | 16 ++++++ rserve/graphs.R | 57 +++++++++++++++----- rserve/tables.R | 38 +++++++++++-- .../ernie/web/graphs/GraphsSubpagesServlet.java | 2 +- .../ernie/web/graphs/RObjectGenerator.java | 5 +- web/WEB-INF/users.jsp | 45 ++++++++++++++++ 8 files changed, 160 insertions(+), 34 deletions(-) diff --git a/detector/detector.py b/detector/detector.py index 7f924db..2c51b5b 100644 --- a/detector/detector.py +++ b/detector/detector.py @@ -348,8 +348,8 @@ def plot_all(tss, minx, maxx, INTERV, DAYS=None, rdir="img"): summary_file.close() """Write a CSV report on the minimum/maximum users of each country per date.""" -def write_all(tss, minc, maxc, INTERVAL=7): - ranges_file = file("direct-users-ranges.csv", "w") +def write_all(tss, minc, maxc, RANGES_FILE, INTERVAL=7): + ranges_file = file(RANGES_FILE, "w") ranges_file.write("date,country,minusers,maxusers\n") exclude = set(["all", "??", "date"]) for c in tss.country_codes: @@ -415,23 +415,25 @@ def write_ml_report(tss, minx, maxx, INTERV, DAYS, notification_period=None): report_file.close() -def main(): - # Change these to customize script - CSV_FILE = "direct-users.csv" - GRAPH_DIR = "img" - # Time interval to model connection rates. - INTERV = 7 - # Consider maximum DAYS days back. - DAYS= 6 * 31 - +# INTERV is the time interval to model connection rates; +# consider maximum DAYS days back. +def detect(CSV_FILE = "userstats-detector.csv", + RANGES_FILE = "userstats-ranges.csv", GRAPH_DIR = "img", + INTERV = 7, DAYS = 6 * 31, REPORT = True): tss = torstatstore(CSV_FILE) l = tss.get_largest_locations(50) minx, maxx = make_tendencies_minmax(l, INTERV) #plot_all(tss, minx, maxx, INTERV, DAYS, rdir=GRAPH_DIR) - write_all(tss, minx, maxx, INTERV) + write_all(tss, minx, maxx, RANGES_FILE, INTERV) - # Make our short report; only consider events of the last day - write_ml_report(tss, minx, maxx, INTERV, DAYS, 1) + if REPORT: + # Make our short report; only consider events of the last day + write_ml_report(tss, minx, maxx, INTERV, DAYS, 1) + +def main(): + detect(CSV_FILE = "direct-users.csv", + RANGES_FILE = "direct-users-ranges.csv") + detect(REPORT = False) if __name__ == "__main__": main() diff --git a/detector/detector.sh b/detector/detector.sh index 8e2ea47..56f6886 100755 --- a/detector/detector.sh +++ b/detector/detector.sh @@ -1,5 +1,6 @@ #!/bin/bash wget -qO direct-users.csv --no-check-certificate https://metrics.torproject.org/csv/direct-users.csv +wget -qO userstats-detector.csv --no-check-certificate https://metrics.torproject.org/csv/userstats-detector.csv python detector.py cat short_censorship_report.txt | mail -E -s 'Possible censorship events' tor-censorship-events(a)lists.torproject.org diff --git a/rserve/csv.R b/rserve/csv.R index 531e73f..34a3f91 100644 --- a/rserve/csv.R +++ b/rserve/csv.R @@ -328,3 +328,19 @@ export_monthly_userstats_average <- function(path) { help_export_monthly_userstats(path, mean) } +export_userstats_detector <- function(path) { + u <- read.csv(paste("/srv/metrics.torproject.org/task-8462-graphs/", + "task-8462/userstats.csv", sep = ""), + stringsAsFactors = FALSE) + u <- u[u$country != '' & u$transport == '' & u$version == '' & + u$node == 'relay', c("country", "date", "users")] + u <- rbind(u, data.frame(country = "zy", + aggregate(list(users = u$users), + by = list(date = u$date), sum))) + u <- data.frame(date = u$date, country = u$country, + users = floor(u$users)) + u <- cast(u, date ~ country, value = "users") + names(u)[names(u) == "zy"] <- "all" + write.csv(u, path, quote = FALSE, row.names = FALSE) +} + diff --git a/rserve/graphs.R b/rserve/graphs.R index 69f9aa9..da39327 100644 --- a/rserve/graphs.R +++ b/rserve/graphs.R @@ -690,8 +690,8 @@ plot_direct_users <- function(start, end, country, events, path) { if (length(r$maxusers) > 0) max_y <- max(max_y, max(r$maxusers, na.rm = TRUE)) plot <- plot + - geom_ribbon(data = r, aes(ymin = minusers, ymax = maxusers), - fill = "gray") + geom_ribbon(data = r, aes(ymin = max(0, minusers), + ymax = maxusers), fill = "gray") } if (length(upturns$date) > 0) plot <- plot + @@ -1022,7 +1022,8 @@ plot_bandwidth_flags <- function(start, end, path) { ggsave(filename = path, width = 8, height = 5, dpi = 72) } -plot_userstats <- function(start, end, node, variable, value, path) { +plot_userstats <- function(start, end, node, variable, value, events, + path) { end <- min(end, as.character(Sys.Date())) u <- read.csv(paste("/srv/metrics.torproject.org/task-8462-graphs/", "task-8462/userstats.csv", sep = ""), @@ -1052,7 +1053,7 @@ plot_userstats <- function(start, end, node, variable, value, path) { " (BETA)\n", sep = "") } else { u <- u[u$country == '' & u$transport == '' & u$version == '' & - u$node == 'bridge', ] + u$node == 'bridge', ] title <- "Bridge users (BETA)\n" } } @@ -1068,32 +1069,62 @@ plot_userstats <- function(start, end, node, variable, value, path) { formatter <- function(x, ...) { format(x, scientific = FALSE, ...) } date_breaks <- date_breaks( as.numeric(max(u$date) - min(u$date))) - ggplot(u, aes(x = date, y = users)) + + max_y <- ifelse(length(na.omit(u$users)) == 0, 0, + max(u$users, na.rm = TRUE)) + plot <- ggplot(u, aes(x = date, y = users)) + if (length(na.omit(u$users)) > 0 & events != "off" & + variable == 'country' & value != "all") { + r <- read.csv( + "/srv/metrics.torproject.org/web/detector/userstats-ranges.csv", + stringsAsFactors = FALSE) + r <- r[r$date >= start & r$date <= end & r$country == value, + c("date", "minusers", "maxusers")] + r <- cast(rbind(melt(u, id.vars = "date"), melt(r, id.vars = "date"))) + upturns <- r[r$users > r$maxusers, 1:2] + downturns <- r[r$users < r$minusers, 1:2] + if (events == "on") { + if (length(r$maxusers) > 0) + max_y <- max(max_y, max(r$maxusers, na.rm = TRUE)) + plot <- plot + + geom_ribbon(data = r, aes(ymin = max(0, minusers), + ymax = maxusers), fill = "gray") + } + if (length(upturns$date) > 0) + plot <- plot + + geom_point(data = upturns, aes(x = date, y = users), size = 5, + colour = "dodgerblue2") + if (length(downturns$date) > 0) + plot <- plot + + geom_point(data = downturns, aes(x = date, y = users), size = 5, + colour = "firebrick2") + } + plot <- plot + geom_line(size = 1) + scale_x_date(name = paste("\nThe Tor Project - ", "https://metrics.torproject.org/", sep = ""), format = date_breaks$format, major = date_breaks$major, minor = date_breaks$minor) + - scale_y_continuous(name = "", limits = c(0, - ifelse(length(na.omit(u$users)) == 0, 0, - max(u$users, na.rm = TRUE))), formatter = formatter) + + scale_y_continuous(name = "", limits = c(0, max_y), + formatter = formatter) opts(title = title) ggsave(filename = path, width = 8, height = 5, dpi = 72) } -plot_userstats_relay_country <- function(start, end, country, path) { - plot_userstats(start, end, 'relay', 'country', country, path) +plot_userstats_relay_country <- function(start, end, country, events, + path) { + plot_userstats(start, end, 'relay', 'country', country, events, path) } plot_userstats_bridge_country <- function(start, end, country, path) { - plot_userstats(start, end, 'bridge', 'country', country, path) + plot_userstats(start, end, 'bridge', 'country', country, 'off', path) } plot_userstats_bridge_transport <- function(start, end, transport, path) { - plot_userstats(start, end, 'bridge', 'transport', transport, path) + plot_userstats(start, end, 'bridge', 'transport', transport, 'off', + path) } plot_userstats_bridge_version <- function(start, end, version, path) { - plot_userstats(start, end, 'bridge', 'version', version, path) + plot_userstats(start, end, 'bridge', 'version', version, 'off', path) } diff --git a/rserve/tables.R b/rserve/tables.R index e0dc1e4..59593d6 100644 --- a/rserve/tables.R +++ b/rserve/tables.R @@ -42,9 +42,6 @@ write_censorship_events <- function(start, end, path) { u <- data.frame(date = u$date, country = u$country, users = u$r * (u$bwp * u$brn / u$bwn - u$brp) / (u$bwr * u$brn / u$bwn - u$brr) / 10) - dates <- seq(from = as.Date(start, "%Y-%m-%d"), - to = as.Date(end, "%Y-%m-%d"), by="1 day") - missing <- setdiff(dates, u$date) r <- read.csv( "/srv/metrics.torproject.org/web/detector/direct-users-ranges.csv", stringsAsFactors = FALSE) @@ -60,10 +57,10 @@ write_censorship_events <- function(start, end, path) { by = list(country = r$country), sum) r <- r[!(r$country %in% c("zy", "??", "a1", "a2", "o1", "ap", "eu")), ] r <- r[order(r$downturn, r$upturn, decreasing = TRUE), ] - r <- r[1:10, ] + r <- r[1:10, ] r <- data.frame(cc = r$country, country = sub('the ', '', countrynames(as.character(r$country))), - downturns = r$downturn, + downturns = r$downturn, upturns = r$upturn) write.csv(r, path, quote = FALSE, row.names = FALSE) } @@ -122,3 +119,34 @@ write_userstats_bridge <- function(start, end, path) { write_userstats(start, end, 'bridge', path) } +write_userstats_censorship_events <- function(start, end, path) { + end <- min(end, as.character(Sys.Date())) + u <- read.csv(paste("/srv/metrics.torproject.org/task-8462-graphs/", + "task-8462/userstats.csv", sep = ""), + stringsAsFactors = FALSE) + u <- u[u$date >= start & u$date <= end & u$country != '' & + u$transport == '' & u$version == '' & u$node == 'relay', + c("date", "country", "users")] + r <- read.csv( + "/srv/metrics.torproject.org/web/detector/userstats-ranges.csv", + stringsAsFactors = FALSE) + r <- r[r$date >= start & r$date <= end, + c("date", "country", "minusers", "maxusers")] + r <- cast(rbind(melt(u, id.vars = c("date", "country")), + melt(r, id.vars = c("date", "country")))) + r <- na.omit(r[r$users < r$minusers | r$users > r$maxusers, ]) + r <- data.frame(date = r$date, country = r$country, + upturn = ifelse(r$users > r$maxusers, 1, 0), + downturn = ifelse(r$users < r$minusers, 1, 0)) + r <- aggregate(r[, c("upturn", "downturn")], + by = list(country = r$country), sum) + r <- r[!(r$country %in% c("zy", "??", "a1", "a2", "o1", "ap", "eu")), ] + r <- r[order(r$downturn, r$upturn, decreasing = TRUE), ] + r <- r[1:10, ] + r <- data.frame(cc = r$country, + country = sub('the ', '', countrynames(as.character(r$country))), + downturns = r$downturn, + upturns = r$upturn) + write.csv(r, path, quote = FALSE, row.names = FALSE) +} + diff --git a/src/org/torproject/ernie/web/graphs/GraphsSubpagesServlet.java b/src/org/torproject/ernie/web/graphs/GraphsSubpagesServlet.java index 3ac99bb..c79f1e3 100644 --- a/src/org/torproject/ernie/web/graphs/GraphsSubpagesServlet.java +++ b/src/org/torproject/ernie/web/graphs/GraphsSubpagesServlet.java @@ -52,7 +52,7 @@ public class GraphsSubpagesServlet extends HttpServlet { this.availableGraphsSubpageTables.put("users.html", new HashSet<String>(Arrays.asList(( "direct-users,censorship-events,bridge-users,userstats-relay," - + "userstats-bridge").split(",")))); + + "userstats-censorship-events,userstats-bridge").split(",")))); this.knownCountries = Countries.getInstance().getCountryList(); } diff --git a/src/org/torproject/ernie/web/graphs/RObjectGenerator.java b/src/org/torproject/ernie/web/graphs/RObjectGenerator.java index 84d61c6..2fa0cc6 100644 --- a/src/org/torproject/ernie/web/graphs/RObjectGenerator.java +++ b/src/org/torproject/ernie/web/graphs/RObjectGenerator.java @@ -76,6 +76,7 @@ public class RObjectGenerator implements ServletContextListener { this.availableCsvFiles.add("torperf"); this.availableCsvFiles.add("torperf-failures"); this.availableCsvFiles.add("userstats"); + this.availableCsvFiles.add("userstats-detector"); this.availableCsvFiles.add("versions"); this.availableTables = new HashMap<String, String>(); @@ -84,6 +85,8 @@ public class RObjectGenerator implements ServletContextListener { this.availableTables.put("bridge-users", "start,end,filename"); this.availableTables.put("userstats-relay", "start,end,filename"); this.availableTables.put("userstats-bridge", "start,end,filename"); + this.availableTables.put("userstats-censorship-events", + "start,end,filename"); TableParameterChecker.getInstance().setAvailableTables( availableTables); @@ -111,7 +114,7 @@ public class RObjectGenerator implements ServletContextListener { this.availableGraphs.put("fast-exits", "start,end,filename"); this.availableGraphs.put("almost-fast-exits", "start,end,filename"); this.availableGraphs.put("userstats-relay-country", - "start,end,country,filename"); + "start,end,country,events,filename"); this.availableGraphs.put("userstats-bridge-country", "start,end,country,filename"); this.availableGraphs.put("userstats-bridge-transport", diff --git a/web/WEB-INF/users.jsp b/web/WEB-INF/users.jsp index 2a7bede..456aaab 100644 --- a/web/WEB-INF/users.jsp +++ b/web/WEB-INF/users.jsp @@ -208,6 +208,9 @@ Tor users (direct and bridge) per month by country.</p> daily Tor users (direct and bridge) per month by country.</p> <br> +<hr> +<hr> + <a name="userstats"></a> <h3><a href="#userstats" class="anchor">New approach to estimating daily Tor users (BETA)</a></h3> @@ -257,6 +260,14 @@ It's yet to be decided which approach is more correct.</font> </c:forEach> </select> </p><p> + Show possible censorship events if available (<a + href="http://research.torproject.org/techreports/detector-2011-09-09.pdf">BETA</a>) + <select name="events"> + <option value="off">Off</option> + <option value="on"<c:if test="${userstats_relay_country_events[0] eq 'on'}"> selected</c:if>>On: both points and expected range</option> + <option value="points"<c:if test="${userstats_relay_country_events[0] eq 'points'}"> selected</c:if>>On: points only, no expected range</option> + </select> + </p><p> <input class="submit" type="submit" value="Update graph"> </p> </div> @@ -296,6 +307,40 @@ It's yet to be decided which approach is more correct.</font> </c:forEach> </table> <hr> +<a name="userstats-censorship-events"></a> +<p><b>Top-10 countries by possible censorship events (<a + href="http://research.torproject.org/techreports/detector-2011-09-09.pdf">BETA</a>):</b></p> +<form action="users.html#userstats-censorship-events"> + <div class="formrow"> + <input type="hidden" name="table" value="userstats-censorship-events"> + <p> + <label>Start date (yyyy-mm-dd):</label> + <input type="text" name="start" size="10" + value="<c:choose><c:when test="${fn:length(userstats_censorship_events_start) == 0}">${default_start_date}</c:when><c:otherwise>${userstats_censorship_events_start[0]}</c:otherwise></c:choose>"> + <label>End date (yyyy-mm-dd):</label> + <input type="text" name="end" size="10" + value="<c:choose><c:when test="${fn:length(userstats_censorship_events_end) == 0}">${default_end_date}</c:when><c:otherwise>${userstats_censorship_events_end[0]}</c:otherwise></c:choose>"> + </p><p> + <input class="submit" type="submit" value="Update table"> + </p> + </div> +</form> +<br> +<table> + <tr> + <th>Country</th> + <th>Downturns</th> + <th>Upturns</th> + </tr> + <c:forEach var="row" items="${userstats_censorship_events_tabledata}"> + <tr> + <td><a href="users.html?graph=direct-users&country=${row['cc']}&events=on#direct-users">${row['country']}</a>&emsp;</td> + <td>${row['downturns']}</td> + <td>${row['upturns']}</td> + </tr> + </c:forEach> +</table> +<hr> <a name="userstats-bridge-country"></a> <p><b>Bridge users by country (BETA):</b></p>

1 0

[metrics-web/master] Add George Danezis' censorship detector.
by karsten＠torproject.org 16 Sep '13

16 Sep '13

commit ee9b5da66a11d5289749f7a18703e381702d116d Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Sep 16 18:06:28 2013 +0200 Add George Danezis' censorship detector. This is a slightly tweaked version of metrics-tasks.git/task-2718/. --- detector/.gitignore | 2 + detector/country_info.py | 251 ++++++++++++++++++++++++++ detector/detector.py | 437 ++++++++++++++++++++++++++++++++++++++++++++++ detector/detector.sh | 5 + 4 files changed, 695 insertions(+) diff --git a/detector/.gitignore b/detector/.gitignore new file mode 100644 index 0000000..29a7166 --- /dev/null +++ b/detector/.gitignore @@ -0,0 +1,2 @@ +*.csv + diff --git a/detector/country_info.py b/detector/country_info.py new file mode 100644 index 0000000..9dbdeb5 --- /dev/null +++ b/detector/country_info.py @@ -0,0 +1,251 @@ +# -*- coding: utf-8 -*- + +countries = { + "ad" : "Andorra", + "ae" : "the United Arab Emirates", + "af" : "Afghanistan", + "ag" : "Antigua and Barbuda", + "ai" : "Anguilla", + "al" : "Albania", + "am" : "Armenia", + "an" : "the Netherlands Antilles", + "ao" : "Angola", + "aq" : "Antarctica", + "ar" : "Argentina", + "as" : "American Samoa", + "at" : "Austria", + "au" : "Australia", + "aw" : "Aruba", + "ax" : "the Aland Islands", + "az" : "Azerbaijan", + "ba" : "Bosnia and Herzegovina", + "bb" : "Barbados", + "bd" : "Bangladesh", + "be" : "Belgium", + "bf" : "Burkina Faso", + "bg" : "Bulgaria", + "bh" : "Bahrain", + "bi" : "Burundi", + "bj" : "Benin", + "bl" : "Saint Bartelemey", + "bm" : "Bermuda", + "bn" : "Brunei", + "bo" : "Bolivia", + "br" : "Brazil", + "bs" : "the Bahamas", + "bt" : "Bhutan", + "bv" : "the Bouvet Island", + "bw" : "Botswana", + "by" : "Belarus", + "bz" : "Belize", + "ca" : "Canada", + "cc" : "the Cocos (Keeling) Islands", + "cd" : "the Democratic Republic of the Congo", + "cf" : "Central African Republic", + "cg" : "Congo", + "ch" : "Switzerland", + "ci" : u"Côte d'Ivoire", + "ck" : "the Cook Islands", + "cl" : "Chile", + "cm" : "Cameroon", + "cn" : "China", + "co" : "Colombia", + "cr" : "Costa Rica", + "cu" : "Cuba", + "cv" : "Cape Verde", + "cx" : "the Christmas Island", + "cy" : "Cyprus", + "cz" : "the Czech Republic", + "de" : "Germany", + "dj" : "Djibouti", + "dk" : "Denmark", + "dm" : "Dominica", + "do" : "the Dominican Republic", + "dz" : "Algeria", + "ec" : "Ecuador", + "ee" : "Estonia", + "eg" : "Egypt", + "eh" : "the Western Sahara", + "er" : "Eritrea", + "es" : "Spain", + "et" : "Ethiopia", + "fi" : "Finland", + "fj" : "Fiji", + "fk" : "the Falkland Islands (Malvinas)", + "fm" : "the Federated States of Micronesia", + "fo" : "the Faroe Islands", + "fr" : "France", + "fx" : "Metropolitan France", + "ga" : "Gabon", + "gb" : "the United Kingdom", + "gd" : "Grenada", + "ge" : "Georgia", + "gf" : "French Guiana", + "gg" : "Guernsey", + "gh" : "Ghana", + "gi" : "Gibraltar", + "gl" : "Greenland", + "gm" : "Gambia", + "gn" : "Guinea", + "gp" : "Guadeloupe", + "gq" : "Equatorial Guinea", + "gr" : "Greece", + "gs" : "South Georgia and the South Sandwich Islands", + "gt" : "Guatemala", + "gu" : "Guam", + "gw" : "Guinea-Bissau", + "gy" : "Guyana", + "hk" : "Hong Kong", + "hm" : "Heard Island and McDonald Islands", + "hn" : "Honduras", + "hr" : "Croatia", + "ht" : "Haiti", + "hu" : "Hungary", + "id" : "Indonesia", + "ie" : "Ireland", + "il" : "Israel", + "im" : "the Isle of Man", + "in" : "India", + "io" : "the British Indian Ocean Territory", + "iq" : "Iraq", + "ir" : "Iran", + "is" : "Iceland", + "it" : "Italy", + "je" : "Jersey", + "jm" : "Jamaica", + "jo" : "Jordan", + "jp" : "Japan", + "ke" : "Kenya", + "kg" : "Kyrgyzstan", + "kh" : "Cambodia", + "ki" : "Kiribati", + "km" : "Comoros", + "kn" : "Saint Kitts and Nevis", + "kp" : "North Korea", + "kr" : "the Republic of Korea", + "kw" : "Kuwait", + "ky" : "the Cayman Islands", + "kz" : "Kazakhstan", + "la" : "Laos", + "lb" : "Lebanon", + "lc" : "Saint Lucia", + "li" : "Liechtenstein", + "lk" : "Sri Lanka", + "lr" : "Liberia", + "ls" : "Lesotho", + "lt" : "Lithuania", + "lu" : "Luxembourg", + "lv" : "Latvia", + "ly" : "Libya", + "ma" : "Morocco", + "mc" : "Monaco", + "md" : "the Republic of Moldova", + "me" : "Montenegro", + "mf" : "Saint Martin", + "mg" : "Madagascar", + "mh" : "the Marshall Islands", + "mk" : "Macedonia", + "ml" : "Mali", + "mm" : "Burma", + "mn" : "Mongolia", + "mo" : "Macau", + "mp" : "the Northern Mariana Islands", + "mq" : "Martinique", + "mr" : "Mauritania", + "ms" : "Montserrat", + "mt" : "Malta", + "mu" : "Mauritius", + "mv" : "the Maldives", + "mw" : "Malawi", + "mx" : "Mexico", + "my" : "Malaysia", + "mz" : "Mozambique", + "na" : "Namibia", + "nc" : "New Caledonia", + "ne" : "Niger", + "nf" : "Norfolk Island", + "ng" : "Nigeria", + "ni" : "Nicaragua", + "nl" : "the Netherlands", + "no" : "Norway", + "np" : "Nepal", + "nr" : "Nauru", + "nu" : "Niue", + "nz" : "New Zealand", + "om" : "Oman", + "pa" : "Panama", + "pe" : "Peru", + "pf" : "French Polynesia", + "pg" : "Papua New Guinea", + "ph" : "the Philippines", + "pk" : "Pakistan", + "pl" : "Poland", + "pm" : "Saint Pierre and Miquelon", + "pn" : "the Pitcairn Islands", + "pr" : "Puerto Rico", + "ps" : "the Palestinian Territory", + "pt" : "Portugal", + "pw" : "Palau", + "py" : "Paraguay", + "qa" : "Qatar", + "re" : "Reunion", + "ro" : "Romania", + "rs" : "Serbia", + "ru" : "Russia", + "rw" : "Rwanda", + "sa" : "Saudi Arabia", + "sb" : "the Solomon Islands", + "sc" : "the Seychelles", + "sd" : "Sudan", + "se" : "Sweden", + "sg" : "Singapore", + "sh" : "Saint Helena", + "si" : "Slovenia", + "sj" : "Svalbard and Jan Mayen", + "sk" : "Slovakia", + "sl" : "Sierra Leone", + "sm" : "San Marino", + "sn" : "Senegal", + "so" : "Somalia", + "sr" : "Suriname", + "st" : u"São Tomé and Príncipe", + "sv" : "El Salvador", + "sy" : "the Syrian Arab Republic", + "sz" : "Swaziland", + "tc" : "Turks and Caicos Islands", + "td" : "Chad", + "tf" : "the French Southern Territories", + "tg" : "Togo", + "th" : "Thailand", + "tj" : "Tajikistan", + "tk" : "Tokelau", + "tl" : "East Timor", + "tm" : "Turkmenistan", + "tn" : "Tunisia", + "to" : "Tonga", + "tr" : "Turkey", + "tt" : "Trinidad and Tobago", + "tv" : "Tuvalu", + "tw" : "Taiwan", + "tz" : "the United Republic of Tanzania", + "ua" : "Ukraine", + "ug" : "Uganda", + "um" : "the United States Minor Outlying Islands", + "us" : "the United States", + "uy" : "Uruguay", + "uz" : "Uzbekistan", + "va" : "Vatican City", + "vc" : "Saint Vincent and the Grenadines", + "ve" : "Venezuela", + "vg" : "the British Virgin Islands", + "vi" : "the United States Virgin Islands", + "vn" : "Vietnam", + "vu" : "Vanuatu", + "wf" : "Wallis and Futuna", + "ws" : "Samoa", + "ye" : "Yemen", + "yt" : "Mayotte", + "za" : "South Africa", + "zm" : "Zambia", + "zw" : "Zimbabwe" + } diff --git a/detector/detector.py b/detector/detector.py new file mode 100644 index 0000000..7f924db --- /dev/null +++ b/detector/detector.py @@ -0,0 +1,437 @@ +## Copyright (c) 2011 George Danezis <gdane(a)microsoft.com> +## +## All rights reserved. +## +## Redistribution and use in source and binary forms, with or without +## modification, are permitted (subject to the limitations in the +## disclaimer below) provided that the following conditions are met: +## +## * Redistributions of source code must retain the above copyright +## notice, this list of conditions and the following disclaimer. +## +## * Redistributions in binary form must reproduce the above copyright +## notice, this list of conditions and the following disclaimer in the +## documentation and/or other materials provided with the +## distribution. +## +## * Neither the name of <Owner Organization> nor the names of its +## contributors may be used to endorse or promote products derived +## from this software without specific prior written permission. +## +## NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE +## GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT +## HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED +## WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +## MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +## DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +## LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +## CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +## SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +## BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +## WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +## OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +## IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +## +## (Clear BSD license: http://labs.metacarta.com/license-explanation.html#license) + +## This script reads a .csv file of the number of Tor users and finds +## anomalies that might be indicative of censorship. + +# Dep: matplotlib +from pylab import * +import matplotlib + +# Dep: numpy +import numpy + +# Dep: scipy +import scipy.stats +from scipy.stats.distributions import norm +from scipy.stats.distributions import poisson + +# Std lib +from datetime import date +from datetime import timedelta +import os.path + +# Country code -> Country names +import country_info + +# write utf8 to file +import codecs + +days = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"] + +def get_country_name_from_cc(country_code): + if (country_code.lower() in country_info.countries): + return country_info.countries[country_code.lower()] + return country_code # if we didn't find the cc in our map + +""" +Represents a .csv file containing information on the number of +connecting Tor users per country. + +'store': Dictionary with (<country code>, <counter>) as key, and the number of users as value. + <country code> can also be "date"... +'all_dates': List of the data intervals (with default timedelta: 1 day). +'country_codes': List of all relevant country codes. +'MAX_INDEX': Length of store, number of country codes etc. +'date_min': The oldest date found in the .csv. +'date_min': The latest date found in the .csv. +""" +class torstatstore: + def __init__(self, file_name): + f = file(file_name) + country_codes = f.readline() + country_codes = country_codes.strip().split(",") + + store = {} + MAX_INDEX = 0 + for i, line in enumerate(f): + MAX_INDEX += 1 + line_parsed = line.strip().split(",") + for j, (ccode, val) in enumerate(zip(country_codes,line_parsed)): + processed_val = None + if ccode == "date": + try: + year, month, day = int(val[:4]), int(val[5:7]), int(val[8:10]) + processed_val = date(year, month, day) + except Exception, e: + print "Parsing error (ignoring line %s):" % j + print "%s" % val,e + break + + elif val != "NA": + processed_val = int(val) + store[(ccode, i)] = processed_val + + # min and max + date_min = store[("date", 0)] + date_max = store[("date", i)] + + all_dates = [] + d = date_min + dt = timedelta(days=1) + while d <= date_max: + all_dates += [d] + d = d + dt + + # Save for later + self.store = store + self.all_dates = all_dates + self.country_codes = country_codes + self.MAX_INDEX = MAX_INDEX + self.date_min = date_min + self.date_max = date_max + + """Return a list representing a time series of 'ccode' with respect + to the number of connected users. + """ + def get_country_series(self, ccode): + assert ccode in self.country_codes + series = {} + for d in self.all_dates: + series[d] = None + for i in range(self.MAX_INDEX): + series[self.store[("date", i)]] = self.store[(ccode, i)] + sx = [] + for d in self.all_dates: + sx += [series[d]] + return sx + + """Return an ordered list containing tuples of the form (<number of + users>, <country code>). The list is ordered with respect to the + number of users for each country. + """ + def get_largest(self, number): + exclude = set(["all", "??", "date"]) + l = [(self.store[(c, self.MAX_INDEX-1)], c) for c in self.country_codes if c not in exclude] + l.sort() + l.reverse() + return l[:number] + + """Return a dictionary, with <country code> as key, and the time + series of the country code as the value. + """ + def get_largest_locations(self, number): + l = self.get_largest(number) + res = {} + for _, ccode in l[:number]: + res[ccode] = self.get_country_series(ccode) + return res + +"""Return a list containing lists (?) where each such list contains +the difference in users for a time delta of 'days' +""" +def n_day_rel(series, days): + rel = [] + for i, v in enumerate(series): + if series[i] is None: + rel += [None] + continue + + if i - days < 0 or series[i-days] is None or series[i-days] == 0: + rel += [None] + else: + rel += [ float(series[i]) / series[i-days]] + return rel + +# Main model: computes the expected min / max range of number of users +def make_tendencies_minmax(l, INTERVAL = 1): + lminus1 = dict([(ccode, n_day_rel(l[ccode], INTERVAL)) for ccode in l]) + c = lminus1[lminus1.keys()[0]] + dists = [] + minx = [] + maxx = [] + for i in range(len(c)): + vals = [lminus1[ccode][i] for ccode in lminus1.keys() if lminus1[ccode][i] != None] + if len(vals) < 8: + dists += [None] + minx += [None] + maxx += [None] + else: + vals.sort() + median = vals[len(vals)/2] + q1 = vals[len(vals)/4] + q2 = vals[(3*len(vals))/4] + qd = q2 - q1 + vals = [v for v in vals if median - qd*4 < v and v < median + qd*4] + if len(vals) < 8: + dists += [None] + minx += [None] + maxx += [None] + continue + mu, signma = norm.fit(vals) + dists += [(mu, signma)] + maxx += [norm.ppf(0.9999, mu, signma)] + minx += [norm.ppf(1 - 0.9999, mu, signma)] + ## print minx[-1], maxx[-1] + return minx, maxx + +# Makes pretty plots +def raw_plot(series, minc, maxc, labels, xtitle): + assert len(xtitle) == 3 + fname, stitle, slegend = xtitle + + font = {'family' : 'Bitstream Vera Sans', + 'weight' : 'normal', + 'size' : 8} + matplotlib.rc('font', **font) + + ylim( (-max(series)*0.1, max(series)*1.1) ) + plot(labels, series, linewidth=1.0, label="Users") + + wherefill = [] + for mm,mx in zip(minc, maxc): + wherefill += [not (mm == None and mx == None)] + assert mm < mx or (mm == None and mx == None) + + fill_between(labels, minc, maxc, where=wherefill, color="gray", label="Prediction") + + vdown = [] + vup = [] + for i,v in enumerate(series): + if minc[i] != None and v < minc[i]: + vdown += [v] + vup += [None] + elif maxc[i] != None and v > maxc[i]: + vdown += [None] + vup += [v] + else: + vup += [None] + vdown += [None] + + plot(labels, vdown, 'o', ms=10, lw=2, alpha=0.5, mfc='orange', label="Downturns") + plot(labels, vup, 'o', ms=10, lw=2, alpha=0.5, mfc='green', label="Upturns") + + legend(loc=2) + + xlabel('Time (days)') + ylabel('Users') + title(stitle) + grid(True) + F = gcf() + + F.set_size_inches(10,5) + F.savefig(fname, format="png", dpi = (150)) + close() + +def absolute_plot(series, minc, maxc, labels,INTERVAL, xtitle): + in_minc = [] + in_maxc = [] + for i, v in enumerate(series): + if i > 0 and i - INTERVAL >= 0 and series[i] != None and series[i-INTERVAL] != None and series[i-INTERVAL] != 0 and minc[i]!= None and maxc[i]!= None: + in_minc += [minc[i] * poisson.ppf(1-0.9999, series[i-INTERVAL])] + in_maxc += [maxc[i] * poisson.ppf(0.9999, series[i-INTERVAL])] + if not in_minc[-1] < in_maxc[-1]: + print in_minc[-1], in_maxc[-1], series[i-INTERVAL], minc[i], maxc[i] + assert in_minc[-1] < in_maxc[-1] + else: + in_minc += [None] + in_maxc += [None] + raw_plot(series, in_minc, in_maxc, labels, xtitle) + +"""Return the number of downscores and upscores of a time series +'series', given tendencies 'minc' and 'maxc' for the time interval +'INTERVAL'. + +If 'scoring_interval' is specifed we only consider upscore/downscore +that happened in the latest 'scoring_interval' days. +""" +def censor_score(series, minc, maxc, INTERVAL, scoring_interval=None): + upscore = 0 + downscore = 0 + + if scoring_interval is None: + scoring_interval = len(series) + assert(len(series) >= scoring_interval) + + for i, v in enumerate(series): + if i > 0 and i - INTERVAL >= 0 and series[i] != None and series[i-INTERVAL] != None and series[i-INTERVAL] != 0 and minc[i]!= None and maxc[i]!= None: + in_minc = minc[i] * poisson.ppf(1-0.9999, series[i-INTERVAL]) + in_maxc = maxc[i] * poisson.ppf(0.9999, series[i-INTERVAL]) + if (i >= (len(series) - scoring_interval)): + downscore += 1 if minc[i] != None and v < in_minc else 0 + upscore += 1 if maxc[i] != None and v > in_maxc else 0 + + return downscore, upscore + +def plot_target(tss, TARGET, xtitle, minx, maxx, DAYS=365, INTERV = 7): + ctarget = tss.get_country_series(TARGET) + c = n_day_rel(ctarget, INTERV) + absolute_plot(ctarget[-DAYS:], minx[-DAYS:], maxx[-DAYS:], tss.all_dates[-DAYS:],INTERV, xtitle = xtitle) + +def write_censorship_report_prologue(report_file, dates, notification_period): + if (notification_period == 1): + date_str = "%s" % (dates[-1]) # no need for date range if it's just one day + else: + date_str = "%s to %s" % (dates[-notification_period], dates[-1]) + + prologue = "=======================\n" + prologue += "Automatic Censorship Report for %s\n" % (date_str) + prologue += "=======================\n\n" + report_file.write(prologue) + +## Make a league table of censorship + nice graphs +def plot_all(tss, minx, maxx, INTERV, DAYS=None, rdir="img"): + rdir = os.path.realpath(rdir) + if not os.path.exists(rdir) or not os.path.isdir(rdir): + print "ERROR: %s does not exist or is not a directory." % rdir + return + + summary_file = file(os.path.join(rdir, "summary.txt"), "w") + + if DAYS == None: + DAYS = 6*31 + + s = tss.get_largest(200) + scores = [] + for num, li in s: + print ".", + ds,us = censor_score(tss.get_country_series(li)[-DAYS:], minx[-DAYS:], maxx[-DAYS:], INTERV) + # print ds, us + scores += [(ds,num, us, li)] + scores.sort() + scores.reverse() + s = "\n=======================\n" + s+= "Report for %s to %s\n" % (tss.all_dates[-DAYS], tss.all_dates[-1]) + s+= "=======================\n" + print s + summary_file.write(s) + for a,nx, b,c in scores: + if a > 0: + s = "%s -- down: %2d (up: %2d affected: %s)" % (c, a, b, nx) + print s + summary_file.write(s + "\n") + xtitle = (os.path.join(rdir, "%03d-%s-censor.png" % (a,c)), "Tor report for %s -- down: %2d (up: %2d affected: %s)" % (c, a, b, nx),"") + plot_target(tss, c,xtitle, minx, maxx, DAYS, INTERV) + summary_file.close() + +"""Write a CSV report on the minimum/maximum users of each country per date.""" +def write_all(tss, minc, maxc, INTERVAL=7): + ranges_file = file("direct-users-ranges.csv", "w") + ranges_file.write("date,country,minusers,maxusers\n") + exclude = set(["all", "??", "date"]) + for c in tss.country_codes: + if c in exclude: + continue + series = tss.get_country_series(c) + for i, v in enumerate(series): + if i > 0 and i - INTERVAL >= 0 and series[i] != None and series[i-INTERVAL] != None and series[i-INTERVAL] != 0 and minc[i]!= None and maxc[i]!= None: + minv = minc[i] * poisson.ppf(1-0.9999, series[i-INTERVAL]) + maxv = maxc[i] * poisson.ppf(0.9999, series[i-INTERVAL]) + if not minv < maxv: + print minv, maxv, series[i-INTERVAL], minc[i], maxc[i] + assert minv < maxv + ranges_file.write("%s,%s,%s,%s\n" % (tss.all_dates[i], c, minv, maxv)) + ranges_file.close() + +"""Return a URL that points to a graph in metrics.tpo that displays +the number of direct Tor users in country 'country_code', for a +'period'-days period. + +Let's hope that the metrics.tpo URL scheme doesn't change often. +""" +def get_tor_usage_graph_url_for_cc_and_date(country_code, dates, period): + url = "https://metrics.torproject.org/users.html?graph=direct-users&start=%s&end=%…" % \ + (dates[-period], dates[-1], country_code) + return url + +"""Write a file containing a short censorship report over the last +'notification_period' days. +""" +def write_ml_report(tss, minx, maxx, INTERV, DAYS, notification_period=None): + if notification_period is None: + notification_period = DAYS + + report_file = codecs.open('short_censorship_report.txt', 'w', 'utf-8') + file_prologue_written = False + + s = tss.get_largest(None) # no restrictions, get 'em all. + scores = [] + for num, li in s: + ds,us = censor_score(tss.get_country_series(li)[-DAYS:], minx[-DAYS:], maxx[-DAYS:], INTERV, notification_period) + scores += [(ds,num, us, li)] + scores.sort() + scores.reverse() + + for downscores,users_n,upscores,country_code in scores: + if (downscores > 0) or (upscores > 0): + if not file_prologue_written: + write_censorship_report_prologue(report_file, tss.all_dates, notification_period) + file_prologue_written = True + + if ((upscores > 0) and (downscores == 0)): + s = "We detected an unusual spike of Tor users in %s (%d upscores, %d users):\n" % \ + (get_country_name_from_cc(country_code), upscores, users_n) + else: + s = "We detected %d potential censorship events in %s (users: %d, upscores: %d):\n" % \ + (downscores, get_country_name_from_cc(country_code), users_n, upscores) + + # Also give out a link for the appropriate usage graph for a 90-days period. + s += get_tor_usage_graph_url_for_cc_and_date(country_code, tss.all_dates, 90) + + report_file.write(s + "\n") + + report_file.close() + +def main(): + # Change these to customize script + CSV_FILE = "direct-users.csv" + GRAPH_DIR = "img" + # Time interval to model connection rates. + INTERV = 7 + # Consider maximum DAYS days back. + DAYS= 6 * 31 + + tss = torstatstore(CSV_FILE) + l = tss.get_largest_locations(50) + minx, maxx = make_tendencies_minmax(l, INTERV) + #plot_all(tss, minx, maxx, INTERV, DAYS, rdir=GRAPH_DIR) + write_all(tss, minx, maxx, INTERV) + + # Make our short report; only consider events of the last day + write_ml_report(tss, minx, maxx, INTERV, DAYS, 1) + +if __name__ == "__main__": + main() diff --git a/detector/detector.sh b/detector/detector.sh new file mode 100755 index 0000000..8e2ea47 --- /dev/null +++ b/detector/detector.sh @@ -0,0 +1,5 @@ +#!/bin/bash +wget -qO direct-users.csv --no-check-certificate https://metrics.torproject.org/csv/direct-users.csv +python detector.py +cat short_censorship_report.txt | mail -E -s 'Possible censorship events' tor-censorship-events(a)lists.torproject.org +

1 0

[doctor/master] Rate limiting check for extra/missing bandwidth authorities to once per day
by atagar＠torproject.org 16 Sep '13

16 Sep '13

commit 1b288fd33e5d5ca3474988e89931681db32cf348 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Sep 16 10:58:55 2013 -0700 Rate limiting check for extra/missing bandwidth authorities to once per day The java version of doctor rate limits to 150 minutes. That seems like an odd number to me so opting for once a day instead. --- consensus_health_checker.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/consensus_health_checker.py b/consensus_health_checker.py index 299d06d..a5aadfb 100755 --- a/consensus_health_checker.py +++ b/consensus_health_checker.py @@ -395,11 +395,13 @@ def voting_bandwidth_scanners(latest_consensus, consensuses, votes): issues = [] if missing_authorities: - runlevel = Runlevel.ERROR if len(missing_authorities) > 1 else Runlevel.WARNING - issues.append(Issue.for_msg(runlevel, 'MISSING_BANDWIDTH_SCANNERS', ', '.join(missing_authorities))) + if rate_limit_notice('missing_bw_scanners.%s' % '.'.join(missing_authorities), days = 1): + runlevel = Runlevel.ERROR if len(missing_authorities) > 1 else Runlevel.WARNING + issues.append(Issue.for_msg(runlevel, 'MISSING_BANDWIDTH_SCANNERS', ', '.join(missing_authorities))) if extra_authorities: - issues.append(Issue.for_msg(Runlevel.NOTICE, 'EXTRA_BANDWIDTH_SCANNERS', ', '.join(extra_authorities))) + if rate_limit_notice('extra_bw_scanners.%s' % '.'.join(extra_authorities), days = 1): + issues.append(Issue.for_msg(Runlevel.NOTICE, 'EXTRA_BANDWIDTH_SCANNERS', ', '.join(extra_authorities))) return issues

1 0

[translation-tools/master] add two new resources
by runa＠torproject.org 16 Sep '13

16 Sep '13

commit 4a79388b955f5580a0f6ea4e72c69e8c9da151f7 Author: Runa A. Sandvik <runa.sandvik(a)gmail.com> Date: Mon Sep 16 13:20:25 2013 -0400 add two new resources --- config | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config b/config index 6da197d..b0f813b 100644 --- a/config +++ b/config @@ -23,7 +23,9 @@ tails-misc tails-misc_completed abouttor-homepage abouttor-homepage_completed torbutton-brandproperties torbutton-brandproperties_completed torbutton-branddtd torbutton-branddtd_completed torbutton-browserproperties -torbutton-browserproperties_completed" +torbutton-browserproperties_completed torbutton-torbuttonproperties +torbutton-torbuttonproperties_completed torbutton-torbuttondtd +torbutton-torbuttondtd_completed" PIDFILE=/srv/translation.torproject.org/run/update_translations.pid

1 0

[tor/master] Correctly re-process non-option cmdline args on sighup
by nickm＠torproject.org 16 Sep '13

16 Sep '13

commit 15b9a1ff10b6892e0cfab3139be765ee0ee6a72c Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Sep 16 13:05:04 2013 -0400 Correctly re-process non-option cmdline args on sighup Whenever we had an non-option commandline arguments *and* option-bearing commandline arguments on the commandline, we would save only the latter across invocations of options_init_from_torrc, but take their existence as license not to re-parse the former. Yuck! Incidentally, this fix lets us throw away the backup_arg[gv] logic. Fix for bug 9746; bugfix on d98dfb3746790448b0dcff2aa9a00e5e2602688a, not in any released Tor. Found by Damian. Thanks, Damian! --- src/or/config.c | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/src/or/config.c b/src/or/config.c index 5ce7bad..335d368 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -582,8 +582,12 @@ static or_options_t *global_default_options = NULL; static char *torrc_fname = NULL; /** Name of the most recently read torrc-defaults file.*/ static char *torrc_defaults_fname; -/** Configuration Options set by command line. */ +/** Configuration options set by command line. */ static config_line_t *global_cmdline_options = NULL; +/** Non-configuration options set by the command line */ +static config_line_t *global_cmdline_only_options = NULL; +/** Boolean: Have we parsed the command line? */ +static int have_parsed_cmdline = 0; /** Contents of most recently read DirPortFrontPage file. */ static char *global_dirfrontpagecontents = NULL; /** List of port_cfg_t for all configured ports. */ @@ -745,6 +749,9 @@ config_free_all(void) config_free_lines(global_cmdline_options); global_cmdline_options = NULL; + config_free_lines(global_cmdline_only_options); + global_cmdline_only_options = NULL; + if (configured_ports) { SMARTLIST_FOREACH(configured_ports, port_cfg_t *, p, port_cfg_free(p)); @@ -3894,30 +3901,22 @@ options_init_from_torrc(int argc, char **argv) char *cf=NULL, *cf_defaults=NULL; int command; int retval = -1; - static char **backup_argv; - static int backup_argc; char *command_arg = NULL; char *errmsg=NULL; - config_line_t *cmdline_only_options = NULL; config_line_t *p_index = NULL; - - if (argv) { /* first time we're called. save command line args */ - backup_argv = argv; - backup_argc = argc; - } else { /* we're reloading. need to clean up old options first. */ - argv = backup_argv; - argc = backup_argc; - } + config_line_t *cmdline_only_options = NULL; /* Go through command-line variables */ - if (!global_cmdline_options) { + if (! have_parsed_cmdline) { /* Or we could redo the list every time we pass this place. * It does not really matter */ if (config_parse_commandline(argc, argv, 0, &global_cmdline_options, - &cmdline_only_options) < 0) { + &global_cmdline_only_options) < 0) { goto err; } + have_parsed_cmdline = 1; } + cmdline_only_options = global_cmdline_only_options; if (config_line_find(cmdline_only_options, "-h") || config_line_find(cmdline_only_options, "--help")) { @@ -3990,7 +3989,6 @@ options_init_from_torrc(int argc, char **argv) tor_free(cf); tor_free(cf_defaults); - config_free_lines(cmdline_only_options); if (errmsg) { log_warn(LD_CONFIG,"%s", errmsg); tor_free(errmsg);

1 0

[tor/master] Merge remote-tracking branch 'origin/maint-0.2.2' into maint-0.2.3
by nickm＠torproject.org 16 Sep '13

16 Sep '13

commit b46353b7931235da3ecaec0723060501042d6ef4 Merge: 3cebc5e 13d192c Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Sep 16 12:46:50 2013 -0400 Merge remote-tracking branch 'origin/maint-0.2.2' into maint-0.2.3 changes/geoip-september2013 | 3 + src/config/geoip |12806 ++++++++++++++++++++++--------------------- 2 files changed, 6556 insertions(+), 6253 deletions(-)

1 0

[tor/master] Merge remote-tracking branch 'origin/maint-0.2.3' into maint-0.2.4
by nickm＠torproject.org 16 Sep '13

16 Sep '13

commit ace95c525c55387d4f219043400c9821e41994bc Merge: 2c877d2 b46353b Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Sep 16 12:47:05 2013 -0400 Merge remote-tracking branch 'origin/maint-0.2.3' into maint-0.2.4 changes/geoip-september2013 | 3 + src/config/geoip |12806 ++++++++++++++++++++++--------------------- 2 files changed, 6556 insertions(+), 6253 deletions(-)

1 0

[tor/master] Merge remote-tracking branch 'origin/maint-0.2.4'
by nickm＠torproject.org 16 Sep '13

16 Sep '13

commit 3892e03cfc1a17a8a82a931bdd62c5649752ee45 Merge: 147a0f3 ace95c5 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Sep 16 12:48:45 2013 -0400 Merge remote-tracking branch 'origin/maint-0.2.4' changes/geoip-september2013 | 3 + src/config/geoip |12806 ++++++++++++++++++++++--------------------- 2 files changed, 6556 insertions(+), 6253 deletions(-)

1 0

[tor/master] Merge remote-tracking branch 'karsten/geoip-manual-update-sep2013'
by nickm＠torproject.org 16 Sep '13

16 Sep '13

commit 57859980a83991d08d98f450b5e5f87762e9f3f6 Merge: 3892e03 e7361aa Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Sep 16 12:48:49 2013 -0400 Merge remote-tracking branch 'karsten/geoip-manual-update-sep2013' src/config/geoip-manual | 85 ----------------------------------------------- 1 file changed, 85 deletions(-)

1 0

[tor/master] Remove 17 now unused manual A1 substitutions.
by nickm＠torproject.org 16 Sep '13

16 Sep '13

commit e7361aa40a993d8a9b7cc037f9cfd7cd265e24ab Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Sep 10 14:56:39 2013 +0200 Remove 17 now unused manual A1 substitutions. --- src/config/geoip-manual | 85 ----------------------------------------------- 1 file changed, 85 deletions(-) diff --git a/src/config/geoip-manual b/src/config/geoip-manual index ee217fc..061769f 100644 --- a/src/config/geoip-manual +++ b/src/config/geoip-manual @@ -3,27 +3,6 @@ # directory to process this file when producing a new geoip file. See # README.geoip in the same directory for details. -# GB, because previous MaxMind entry 31.6.16.0-31.6.25.255 is GB, and RIR -# delegation files say entire range 31.6.0.0-31.6.63.255 is GB. -# -KL 2013-03-07 -"31.6.26.0","31.6.27.255","520493568","520494079","GB","United Kingdom" - -# NL, because previous MaxMind entry 31.171.128.0-31.171.133.255 is NL, -# and RIR delegation files say 31.171.128.0-31.171.135.255 is NL. -# -KL 2012-11-27 -"31.171.134.0","31.171.135.255","531334656","531335167","NL","Netherlands" - -# EU, because next MaxMind entry 37.139.64.1-37.139.64.9 is EU, because -# RIR delegation files say 37.139.64.0-37.139.71.255 is EU, and because it -# just makes more sense for the next entry to start at .0 and not .1. -# -KL 2012-11-27 -"37.139.64.0","37.139.64.0","629882880","629882880","EU","Europe" - -# US, because next MaxMind entry 38.99.145.0-38.99.149.255 is US and -# RIR delegation files say entire range 38.0.0.0-38.255.255.255 is US. -# -KL 2013-05-13 -"38.99.144.0","38.99.144.255","644059136","644059391","US","United States" - # GB, because RIR delegation files say exactly this range # 46.16.32.0-46.16.39.255 is GB, even though neither previous nor next # MaxMind range is GB. Both previous and next MaxMind ranges match RIR @@ -40,21 +19,6 @@ # -KL 2012-11-27 "46.166.128.0","46.166.128.255","782663680","782663935","GB","United Kingdom" -# GB, because previous MaxMind entry 46.166.129.0-46.166.134.255 is GB, -# and RIR delegation files say entire range 46.166.128.0-46.166.191.255 is -# GB. -KL 2013-03-07 -"46.166.135.0","46.166.139.255","782665472","782666751","GB","United Kingdom" - -# Removing, because RIR delegation files don't even have an entry for this -# single-address range, and there's no previous or next range in MaxMind. -# -KL 2013-03-07 -"64.185.237.110","64.185.237.110","1085926766","1085926766","","" - -# US, because next MaxMind entry 67.43.145.0-67.43.155.255 is US, and RIR -# delegation files say 67.43.144.0-67.43.159.255 is US. -# -KL 2012-11-27 -"67.43.144.0","67.43.144.255","1126928384","1126928639","US","United States" - # US, because previous MaxMind entry 70.159.21.51-70.232.244.255 is US, # because next MaxMind entry 70.232.245.58-70.232.245.59 is A2 ("Satellite # Provider") which is a country information about as useless as A1, and @@ -69,36 +33,11 @@ # US. -KL 2012-11-27 "70.232.245.60","70.232.245.255","1189672252","1189672447","US","United States" -# NL, because previous MaxMind entry 81.171.56.0-81.171.80.255 is NL, and -# RIR delegation files say entire range 81.171.64.0-81.171.127.255 is NL. -# -KL 2013-03-07 -"81.171.81.0","81.171.81.127","1370181888","1370182015","NL","Netherlands" - -# BE, because next MaxMind entry 86.39.147.0-86.39.148.31 is BE, and RIR -# delegation files say entire range 86.39.128.0-86.39.255.255 is BE. -# -KL 2013-04-08 -"86.39.146.0","86.39.146.255","1445433856","1445434111","BE","Belgium" - # GB, despite neither previous (GE) nor next (LV) MaxMind entry being GB, # but because RIR delegation files agree with both previous and next # MaxMind entry and say GB for 91.228.0.0-91.228.3.255. -KL 2012-11-27 "91.228.0.0","91.228.3.255","1541668864","1541669887","GB","United Kingdom" -# GB, because next MaxMind entry 91.232.125.0-91.232.125.255 is GB, and -# RIR delegation files say 91.232.124.0-91.232.125.255 is GB. -# -KL 2012-11-27 -"91.232.124.0","91.232.124.255","1541962752","1541963007","GB","United Kingdom" - -# GB, despite neither previous (RU) nor next (PL) MaxMind entry being GB, -# but because RIR delegation files agree with both previous and next -# MaxMind entry and say GB for 91.238.214.0-91.238.215.255. -# -KL 2012-11-27 -"91.238.214.0","91.238.215.255","1542379008","1542379519","GB","United Kingdom" - -# US, because ARIN says US for 198.159.0.0-198.159.255.255 and because -# previous MaxMind entry is US, too. -KL 2013-08-12 -"98.159.239.0","98.159.239.255","1654648576","1654648831","US","United States" - # NL, because next MaxMind entry 176.56.173.0-176.56.173.63 is NL, and RIR # delegation files say 176.56.160.0-176.56.191.255 is NL. -KL 2013-05-13 "176.56.172.0","176.56.172.255","2956504064","2956504319","NL","Netherlands" @@ -108,36 +47,12 @@ # 176.56.160.0-176.56.191.255 is NL. -KL 2013-05-13 "176.56.174.0","176.56.174.255","2956504576","2956504831","NL","Netherlands" -# US, because next MaxMind entry 176.67.84.0-176.67.84.79 is US, and RIR -# delegation files say 176.67.80.0-176.67.87.255 is US. -KL 2012-11-27 -"176.67.80.0","176.67.83.255","2957201408","2957202431","US","United States" - -# US, because previous MaxMind entry 176.67.84.192-176.67.85.255 is US, -# and RIR delegation files say 176.67.80.0-176.67.87.255 is US. -# -KL 2012-11-27 -"176.67.86.0","176.67.87.255","2957202944","2957203455","US","United States" - # GB, because RIR delegation files say exactly this range # 185.25.84.0-185.25.87.255 is GB, even though neither previous nor next # MaxMind range is GB. Both previous and next MaxMind ranges match RIR # delegation files, too. -KL 2013-05-13 "185.25.84.0","185.25.87.255","3105444864","3105445887","GB","United Kingdom" -# US, because ARIN says US for 192.238.16.0-192.238.23.255 and because -# next MaxMind entry is US, too. -KL 2013-08-12 -"192.238.16.0","192.238.19.255","3236827136","3236828159","US","United States" - -# EU, despite neither previous (RU) nor next (UA) MaxMind entry being EU, -# but because RIR delegation files agree with both previous and next -# MaxMind entry and say EU for 193.200.150.0-193.200.150.255. -# -KL 2012-11-27 -"193.200.150.0","193.200.150.255","3251148288","3251148543","EU","Europe" - -# US, because previous MaxMind entry 199.96.68.0-199.96.87.127 is US, and -# RIR delegation files say 199.96.80.0-199.96.87.255 is US. -# -KL 2012-11-27 -"199.96.87.128","199.96.87.255","3344979840","3344979967","US","United States" - # US, because next MaxMind entry 199.101.193.0-199.101.195.255 is US, and, # together with next entries, matches RIR delegation file entry # 199.101.192.0-199.101.199.255 which is US. -KL 2013-05-13

1 0