- tor-commits - lists.torproject.org

[tech-reports/master] Add generated and renamed PDFs to .gitignore.
by karsten＠torproject.org 25 Jul '12

25 Jul '12

commit 2d6a3bd00ccdf3e763b9e1e6d405698074386442 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Jul 25 12:34:22 2012 +0200 Add generated and renamed PDFs to .gitignore. --- 2011/bridge-blockings/.gitignore | 2 ++ 2011/bridge-stability/.gitignore | 2 ++ 2011/censorship-detector/.gitignore | 2 ++ 2011/relay-stability/.gitignore | 2 ++ 2011/statistical-data/.gitignore | 2 ++ 5 files changed, 10 insertions(+), 0 deletions(-) diff --git a/2011/bridge-blockings/.gitignore b/2011/bridge-blockings/.gitignore index 787dd0b..82a057b 100644 --- a/2011/bridge-blockings/.gitignore +++ b/2011/bridge-blockings/.gitignore @@ -1 +1,3 @@ blocking.pdf +blocking-2011-09-15.pdf + diff --git a/2011/bridge-stability/.gitignore b/2011/bridge-stability/.gitignore index d481f99..f91ef20 100644 --- a/2011/bridge-stability/.gitignore +++ b/2011/bridge-stability/.gitignore @@ -1 +1,3 @@ report.pdf +bridge-stability-2011-10-31.pdf + diff --git a/2011/censorship-detector/.gitignore b/2011/censorship-detector/.gitignore index 7046539..0a88959 100644 --- a/2011/censorship-detector/.gitignore +++ b/2011/censorship-detector/.gitignore @@ -1 +1,3 @@ detector.pdf +detector-2011-09-09.pdf + diff --git a/2011/relay-stability/.gitignore b/2011/relay-stability/.gitignore index d481f99..407d62c 100644 --- a/2011/relay-stability/.gitignore +++ b/2011/relay-stability/.gitignore @@ -1 +1,3 @@ report.pdf +relay-stability-2011-06-30.pdf + diff --git a/2011/statistical-data/.gitignore b/2011/statistical-data/.gitignore index d02299e..3f3c1f0 100644 --- a/2011/statistical-data/.gitignore +++ b/2011/statistical-data/.gitignore @@ -1 +1,3 @@ data.pdf +data-2011-03-14.pdf +

1 0

[metrics-tasks/master] Update #2911 graphing code to ggplot2 0.9.0.
by karsten＠torproject.org 25 Jul '12

25 Jul '12

commit c36a24508907287387accb6b30d69ed994c19661 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Jul 25 12:29:24 2012 +0200 Update #2911 graphing code to ggplot2 0.9.0. --- task-2911/stability.R | 33 +++++++++++++++++---------------- 1 files changed, 17 insertions(+), 16 deletions(-) diff --git a/task-2911/stability.R b/task-2911/stability.R index 5b40d67..63e1748 100644 --- a/task-2911/stability.R +++ b/task-2911/stability.R @@ -1,4 +1,5 @@ library(ggplot2) +library(scales) data <- read.csv("stability.csv", stringsAsFactors = FALSE) d <- data @@ -13,8 +14,8 @@ d <- rbind( variable = "Running + Guard")) ggplot(d, aes(x = as.Date(date), y = value, colour = variable)) + geom_line(size = 0.7) + -scale_x_date("", major = "3 months", minor = "1 month", - format = "%b %Y") + +scale_x_date("", breaks = "3 months", minor_breaks = "1 month", + labels = date_format("%b %Y")) + scale_y_continuous("Number \nof relays ", limits = c(0, max(d$value, na.rm = TRUE))) + scale_colour_manual(name = "Assigned relay flags\n", @@ -41,9 +42,9 @@ d[d$x >= '2010-01-25' & d$x <= '2010-01-26', "y"] <- NA ggplot(d, aes(x = x, y = y, colour = variable, linetype = variable)) + geom_line() + scale_y_continuous(name = "Fraction of \nRunning relays ", - formatter = "percent", limits = c(0, max(d$y, na.rm = TRUE))) + -scale_x_date(name = "", major = "3 months", minor = "1 month", - format = "%b %Y") + + labels = percent, limits = c(0, max(d$y, na.rm = TRUE))) + +scale_x_date("", breaks = "3 months", minor_breaks = "1 month", + labels = date_format("%b %Y")) + scale_colour_manual(name = "Assigned relay flags\n", values = c("Stable (observed)" = "grey50", "Stable (simulated)" = "grey50", @@ -79,8 +80,8 @@ ggplot(d, aes(x = x, y = y, linetype = variable)) + geom_line() + facet_grid(flag ~ ., scale = "free_y") + scale_y_continuous(name = "Number \nof relays ") + -scale_x_date(name = "", major = "3 months", minor = "1 month", - format = "%b %Y") + +scale_x_date("", breaks = "3 months", minor_breaks = "1 month", + labels = date_format("%b %Y")) + scale_linetype_manual(name = "", values = c(4, 3, 1)) + opts(axis.title.x = theme_text(size = 12 * 0.8, face = "bold", hjust = 0.5), @@ -124,8 +125,8 @@ geom_ribbon(alpha = 0.3) + geom_hline(data = e, aes(yintercept = yintercept), colour = "gray30", linetype = 2) + facet_wrap(~ var, scales = "free_y") + -scale_x_date(name = "", major = "3 months", minor = "1 month", - format = "%b %Y") + +scale_x_date("", breaks = "3 months", minor_breaks = "1 month", + labels = date_format("%b %Y")) + scale_y_continuous(name = "") ggsave(filename = "requirements.pdf", width = 8, height = 5, dpi = 100) @@ -157,7 +158,7 @@ scale_x_continuous(name = paste("\n10th percentile of time until next", limits = c(0, max(d$x, na.rm = TRUE))) + scale_y_continuous(name = paste("Cumulative fraction \nof", "consensuses \nfrom July to \nDecember 2010 "), - formatter = "percent", limits = c(0, 1)) + + labels = percent, limits = c(0, 1)) + scale_colour_manual(name = paste("WMTBF percentile\nfor", "assigning\nStable flag\n"), values = c("60th (simulated)" = "black", @@ -199,10 +200,10 @@ d <- rbind( ggplot(d, aes(x = x, y = y, colour = sim, linetype = sim)) + geom_line() + scale_x_continuous(name = "\n10th percentile of WFU in the future", - formatter = "percent") + + labels = percent) + scale_y_continuous(name = paste("Cumulative fraction \nof", "consensuses \nfrom July to \nDecember 2010 "), - formatter = "percent", limits = c(0, 1)) + + labels = percent, limits = c(0, 1)) + scale_colour_manual(name = paste("WFU percentile\nfor", "assigning\nGuard flag\n"), values = c("60th (simulated)" = "black", @@ -246,7 +247,7 @@ scale_x_continuous(name = paste("\n10th percentile of weighted bandwidth", "in KiB/s in the future")) + scale_y_continuous(name = paste("Cumulative fraction \nof", "consensuses \nfrom July to \nDecember 2010 "), - formatter = "percent", limits = c(0, 1)) + + labels = percent, limits = c(0, 1)) + scale_colour_manual(name = paste("Advertised\nbandwidth\npercentile\nfor", "assigning\nGuard flag\n"), values = c( @@ -279,9 +280,9 @@ d <- rbind( ggplot(d, aes(x = x, y = y, colour = variable, linetype = variable)) + geom_line() + scale_y_continuous(name = "Fraction of \nRunning relays ", - formatter = "percent", limits = c(0, max(d$y, na.rm = TRUE))) + -scale_x_date(name = "", major = "3 months", minor = "1 month", - format = "%b %Y") + + labels = percent, limits = c(0, max(d$y, na.rm = TRUE))) + +scale_x_date("", breaks = "3 months", minor_breaks = "1 month", + labels = date_format("%b %Y")) + scale_colour_manual(name = paste("Advertised\nbandwidth\npercentile\nfor", "assigning\nGuard flag\n"), values = c(

1 0

[metrics-tasks/master] Update #4030 graphing code to ggplot2 0.9.0 and switch to .pdf.
by karsten＠torproject.org 25 Jul '12

25 Jul '12

commit 3ff269f890d1476a4d9e0034b547ef586c536bec Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Jul 25 12:30:47 2012 +0200 Update #4030 graphing code to ggplot2 0.9.0 and switch to .pdf. --- task-4030/bridge-blockings.R | 21 +++++++++++---------- 1 files changed, 11 insertions(+), 10 deletions(-) diff --git a/task-4030/bridge-blockings.R b/task-4030/bridge-blockings.R index da40186..4435354 100644 --- a/task-4030/bridge-blockings.R +++ b/task-4030/bridge-blockings.R @@ -1,4 +1,5 @@ library(ggplot2) +library(scales) u <- read.csv("bridge-users.csv", stringsAsFactors = FALSE) u <- u[u$date >= "2009-07-01" & u$date < "2011-08-01", c("date", "cn")] @@ -7,12 +8,12 @@ geom_line(size = 0.75) + geom_rect(aes(NULL, NULL, xmin = as.Date("2010-01-01"), xmax = as.Date("2010-06-30"), ymin = -Inf, ymax = Inf, fill = TRUE)) + scale_fill_manual(name = "", breaks = TRUE, - values = alpha("purple", 0.2)) + -scale_x_date(name = "", major = "6 months", minor = "1 month", - format = "%b %Y") + + values = alpha("purple", 0.005)) + +scale_x_date("", breaks = "6 months", minor_breaks = "1 month", + labels = date_format("%b %Y")) + scale_y_continuous(name = "", limits = c(0, max(u$cn, na.rm = TRUE))) + opts(legend.position = "none") -ggsave("bridge-users.png", width = 8, height = 4, dpi = 150) +ggsave("bridge-users.pdf", width = 8, height = 4, dpi = 150) b <- read.csv("bridge-blockings.csv", stringsAsFactors = FALSE) b <- b[b$date >= '2010-01-01' & b$date <= '2010-06-30', ] @@ -33,10 +34,10 @@ d <- rbind(data.frame(date = u$date, value = u$cn, variable = "Users"), ggplot(d, aes(x = as.Date(date), y = value)) + geom_line(size = 0.75) + facet_grid(variable ~ ., scales = "free_y") + -scale_x_date(name = "", format = "%b %Y", major = "1 month", - minor = "months") + +scale_x_date("", breaks = "1 months", minor_breaks = "1 month", + labels = date_format("%b %Y")) + scale_y_continuous(name = "") -ggsave("bridge-users-blockings.png", width = 8, height = 4, dpi = 150) +ggsave("bridge-users-blockings.pdf", width = 8, height = 4, dpi = 150) b <- data.frame(date = as.Date(b$date), ips = b$ips, fingerprint = substr(b$fingerprint, 1, 4), @@ -50,11 +51,11 @@ geom_point(size = 0.75) + geom_hline(yintercept = 32, linetype = 2, size = 0.25) + geom_point(data = bb, aes(x = date, y = ips), colour = "red", size = 3, alpha = 0.25) + -scale_x_date(name = "", format = "%b", major = "2 months", - minor = "months") + +scale_x_date("", breaks = "2 months", minor_breaks = "1 month", + labels = date_format("%b")) + scale_y_continuous(name = "", breaks = c(0, 500, 1000)) + scale_colour_manual(breaks = c("red", "black"), values = c("red", "black")) + opts(legend.position = "none") -ggsave("bridge-blockings.png", height = 9, width = 8) +ggsave("bridge-blockings.pdf", height = 9, width = 8)

1 0

[metrics-tasks/master] Update #6232 graphing code to ASes.
by karsten＠torproject.org 25 Jul '12

25 Jul '12

commit a213be8ad0796edb4e9ffdfa470f17fe2671642e Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Jul 25 12:28:59 2012 +0200 Update #6232 graphing code to ASes. --- task-6232/plot-entropy.R | 22 ++++++++++++++-------- 1 files changed, 14 insertions(+), 8 deletions(-) diff --git a/task-6232/plot-entropy.R b/task-6232/plot-entropy.R index ab57678..0ed4bac 100644 --- a/task-6232/plot-entropy.R +++ b/task-6232/plot-entropy.R @@ -4,18 +4,21 @@ library(scales) d <- read.csv("entropy.csv", header = FALSE, col.names = c("validafter", "all", "max_all", "exit", "max_exit", - "guard", "max_guard", "country", "max_country")) + "guard", "max_guard", "country", "max_country", "as", "max_as")) e <- aggregate( list(all = d$all / d$max_all, exit = d$exit / d$max_exit, - guard = d$guard / d$max_guard, country = d$country / d$max_country), + guard = d$guard / d$max_guard, country = d$country / d$max_country, + as = d$as / d$max_as), by = list(date = as.Date(d$validafter, origin = "1970-01-01 00:00:00")), FUN = median) e <- melt(e, "date") -e <- data.frame(date = e$date, variable = ifelse(e$variable == "all", - "All relays", ifelse(e$variable == "exit", "All exits", - ifelse(e$variable == "guard", "All guards", "All countries"))), - value = e$value) +e <- data.frame(date = e$date, variable = + ifelse(e$variable == "all", "All relays", + ifelse(e$variable == "exit", "All exits", + ifelse(e$variable == "guard", "All guards", + ifelse(e$variable == "country", "All countries", + "All ASes")))), value = e$value) ggplot(e, aes(x = date, y = value)) + geom_line() + facet_wrap(~ variable) + @@ -26,7 +29,8 @@ ggsave("degree-of-anonymity.png", width = 8, height = 6, dpi = 100) f <- aggregate(list(all = d$all, max_all = d$max_all, exit = d$exit, max_exit = d$max_exit, guard = d$guard, max_guard = d$max_guard, - country = d$country, max_country = d$max_country), + country = d$country, max_country = d$max_country, as = d$as, + max_as = d$max_as), by = list(date = as.Date(d$validafter, origin = "1970-01-01 00:00:00")), FUN = median) f <- rbind( @@ -37,7 +41,9 @@ f <- rbind( data.frame(date = f$date, entropy = f$guard, max = f$max_guard, type = "All guards"), data.frame(date = f$date, entropy = f$country, max = f$max_country, - type = "All countries")) + type = "All countries"), + data.frame(date = f$date, entropy = f$as, max = f$max_as, + type = "All ASes")) f <- melt(f, c("date", "type")) ggplot(f, aes(x = date, y = value, colour = variable)) + geom_line() +

1 0

[metrics-tasks/master] Update #4255 graphing code to ggplot2 0.9.0.
by karsten＠torproject.org 25 Jul '12

25 Jul '12

commit ba24732e4d23671da527b45d0fb8e35c54470f95 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Jul 25 12:31:38 2012 +0200 Update #4255 graphing code to ggplot2 0.9.0. --- task-4255/stability.R | 35 +++++++++++++++++++---------------- 1 files changed, 19 insertions(+), 16 deletions(-) diff --git a/task-4255/stability.R b/task-4255/stability.R index c46266e..b4f162e 100644 --- a/task-4255/stability.R +++ b/task-4255/stability.R @@ -1,4 +1,7 @@ library(ggplot2) +library(scales) +library(grid) +library(reshape) stability <- read.csv("stability.csv", stringsAsFactors = FALSE) d <- stability[stability$time > '2010-07' & stability$time < '2011-07', ] @@ -18,8 +21,8 @@ d <- rbind(d, y = NA, ymin = NA, ymax = NA)) ggplot(d, aes(x = as.Date(x), y = y, ymin = ymin, ymax = ymax)) + geom_line() + -scale_x_date(name = "", major = "3 months", minor = "1 month", - format = "%b %Y") + +scale_x_date("", breaks = "3 months", minor_breaks = "1 month", + labels = date_format("%b %Y")) + scale_y_continuous(name = "Running \nbridges ", limits = c(0, max(d_mean$running, na.rm = TRUE))) + opts(axis.title.x = theme_text(size = 12 * 0.8, face = "bold", @@ -35,8 +38,8 @@ d <- stability[stability$time > '2010-07-10' & stability$time < '2010-07-31', ] a <- ggplot(d, aes(x = as.POSIXct(time), y = running)) + geom_point(size = 0.75) + -scale_x_datetime(name = "", major = "1 week", minor = "1 day", - format = "%b %d, %Y") + +scale_x_datetime(name = "", breaks = "1 week", minor_breaks = "1 day", + labels = date_format("%b %d, %Y")) + scale_y_continuous(name = "Running \nbridges ", limits = c(0, max(d$running, na.rm = TRUE))) + opts(axis.title.x = theme_text(size = 12 * 0.8, face = "bold", @@ -53,8 +56,8 @@ d <- rbind( data.frame(time = e$time, running = 687, colour = "grey")) b <- ggplot(d, aes(x = as.POSIXct(time), y = running, colour = colour)) + geom_point(size = 0.75) + -scale_x_datetime(name = "", major = "1 week", minor = "1 day", - format = "%b %d, %Y") + +scale_x_datetime(name = "", breaks = "1 week", minor_breaks = "1 day", + labels = date_format("%b %d, %Y")) + scale_y_continuous(name = "Running \nbridges ", limits = c(0, max(d$running, na.rm = TRUE))) + scale_colour_manual(values = c("black", "grey60")) + @@ -108,8 +111,8 @@ geom_line() +#colour = "grey30") + geom_hline(data = e, aes(yintercept = yintercept), colour = "gray40", linetype = 2) + facet_grid(var ~ ., scales = "free_y") + -scale_x_date(name = "", major = "3 months", minor = "1 month", - format = "%b %Y") + +scale_x_date("", breaks = "3 months", minor_breaks = "1 month", + labels = date_format("%b %Y")) + scale_y_continuous(name = "") + opts(axis.title.x = theme_text(size = 12 * 0.8, face = "bold", hjust = 0.5), @@ -132,9 +135,9 @@ d <- melt(d, id = "date") ggplot(d, aes(x = date, y = value / 10000, linetype = variable)) + geom_line() + scale_y_continuous(name = paste("10th perc. \nWFU in \n", - "the future ", sep = ""), formatter = "percent", limits = c(0, 1)) + -scale_x_date(name = "", major = "3 months", minor = "1 month", - format = "%b %Y") + + "the future ", sep = ""), labels = percent, limits = c(0, 1)) + +scale_x_date("", breaks = "3 months", minor_breaks = "1 month", + labels = date_format("%b %Y")) + scale_linetype_manual(name = paste("Requirements for\nconsidering", "a\nbridge as stable\n"), breaks = c("perc10wfu50wfu50wmtbac", "perc10wfu50wfu0wmtbac", "perc10wfu0wfu50wmtbac", @@ -166,8 +169,8 @@ scale_y_continuous(name = paste("10th perc. \ntime on \nthe same \n", breaks = seq(0, max(d$value / 86400, na.rm = TRUE), 7), minor = seq(0, max(d$value / 86400, na.rm = TRUE), 1), limits = c(0, max(d$value / 86400, na.rm = TRUE))) + -scale_x_date(name = "", major = "3 months", minor = "1 month", - format = "%b %Y") + +scale_x_date("", breaks = "3 months", minor_breaks = "1 month", + labels = date_format("%b %Y")) + scale_linetype_manual(name = paste("Requirements for\nconsidering", "a\nbridge as stable\n"), breaks = c("perc10tosa50wfu50wmtbac", "perc10tosa0wfu50wmtbac", "perc10tosa50wfu0wmtbac", @@ -207,9 +210,9 @@ d <- rbind(d, ggplot(d, aes(x = x, y = y, linetype = variable)) + geom_line() + scale_y_continuous(name = "Fraction of \nRunning \nbridges ", - formatter = "percent", limits = c(0, max(d$y, na.rm = TRUE))) + -scale_x_date(name = "", major = "3 months", minor = "1 month", - format = "%b %Y") + + labels = percent, limits = c(0, max(d$y, na.rm = TRUE))) + +scale_x_date("", breaks = "3 months", minor_breaks = "1 month", + labels = date_format("%b %Y")) + scale_linetype_manual(name = paste("\nRequirements for\nconsidering", "a\nbridge as stable\n"), values = c(3, 2, 4)) + opts(axis.title.x = theme_text(size = 12 * 0.8, face = "bold",

1 0

[metrics-web/master] Close a few readers/streams, spotted by Eclipse.
by karsten＠torproject.org 25 Jul '12

25 Jul '12

commit a12af627cdb812d0ef32fb99e5383b4993ee764e Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Jul 25 09:00:33 2012 +0200 Close a few readers/streams, spotted by Eclipse. --- .../ernie/cron/ExoneraTorDatabaseImporter.java | 1 + src/org/torproject/ernie/web/RObjectGenerator.java | 1 + .../torproject/ernie/web/ResearchDataServlet.java | 1 + 3 files changed, 3 insertions(+), 0 deletions(-) diff --git a/src/org/torproject/ernie/cron/ExoneraTorDatabaseImporter.java b/src/org/torproject/ernie/cron/ExoneraTorDatabaseImporter.java index c87236e..c23b4eb 100755 --- a/src/org/torproject/ernie/cron/ExoneraTorDatabaseImporter.java +++ b/src/org/torproject/ernie/cron/ExoneraTorDatabaseImporter.java @@ -153,6 +153,7 @@ public class ExoneraTorDatabaseImporter { + "contains a corrupt entry in line " + lineNumber + ". Ignoring parse history file entirely."); lastImportHistory.clear(); + br.close(); return; } long lastModified = Long.parseLong(parts[0]); diff --git a/src/org/torproject/ernie/web/RObjectGenerator.java b/src/org/torproject/ernie/web/RObjectGenerator.java index 993bc59..1b56f31 100644 --- a/src/org/torproject/ernie/web/RObjectGenerator.java +++ b/src/org/torproject/ernie/web/RObjectGenerator.java @@ -410,6 +410,7 @@ public class RObjectGenerator implements ServletContextListener { while ((length = bis.read(buffer)) > 0) { baos.write(buffer, 0, length); } + bis.close(); this.result = baos.toByteArray(); } catch (IOException e) { return; diff --git a/src/org/torproject/ernie/web/ResearchDataServlet.java b/src/org/torproject/ernie/web/ResearchDataServlet.java index e8ae79d..8adb06c 100644 --- a/src/org/torproject/ernie/web/ResearchDataServlet.java +++ b/src/org/torproject/ernie/web/ResearchDataServlet.java @@ -54,6 +54,7 @@ public class ResearchDataServlet extends HttpServlet { dataFileUrls.add(line); } } + br.close(); } } catch (SecurityException e) { /* We're not permitted to read the file with URLs. Ignore. */

1 0

[metrics-db/master] Close a few readers/streams, spotted by Eclipse.
by karsten＠torproject.org 25 Jul '12

25 Jul '12

commit 7d2ec97550c468b20d8c653af72509ed7bfee206 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Jul 25 08:58:29 2012 +0200 Close a few readers/streams, spotted by Eclipse. --- src/org/torproject/ernie/db/ArchiveWriter.java | 1 + .../torproject/ernie/db/BridgeSnapshotReader.java | 1 + .../ernie/db/SanitizedBridgesWriter.java | 1 + 3 files changed, 3 insertions(+), 0 deletions(-) diff --git a/src/org/torproject/ernie/db/ArchiveWriter.java b/src/org/torproject/ernie/db/ArchiveWriter.java index 1d37edc..d1b9499 100644 --- a/src/org/torproject/ernie/db/ArchiveWriter.java +++ b/src/org/torproject/ernie/db/ArchiveWriter.java @@ -317,6 +317,7 @@ public class ArchiveWriter { } } } + br.close(); sb.append(String.format("%nC, %s, %d/%d (%.1f%%), " + "%d/%d (%.1f%%), %d/%d (%.1f%%)", validAfterTime, foundVotes, allVotes, diff --git a/src/org/torproject/ernie/db/BridgeSnapshotReader.java b/src/org/torproject/ernie/db/BridgeSnapshotReader.java index 0de9d83..f21794d 100644 --- a/src/org/torproject/ernie/db/BridgeSnapshotReader.java +++ b/src/org/torproject/ernie/db/BridgeSnapshotReader.java @@ -174,6 +174,7 @@ public class BridgeSnapshotReader { descriptorImportHistory.add(fileDigest); parsedFiles++; } + bis.close(); } in.close(); diff --git a/src/org/torproject/ernie/db/SanitizedBridgesWriter.java b/src/org/torproject/ernie/db/SanitizedBridgesWriter.java index 366da00..4f72c6a 100644 --- a/src/org/torproject/ernie/db/SanitizedBridgesWriter.java +++ b/src/org/torproject/ernie/db/SanitizedBridgesWriter.java @@ -109,6 +109,7 @@ public class SanitizedBridgesWriter { byte[] secret = Hex.decodeHex(parts[1].toCharArray()); this.secretsForHashingIPAddresses.put(month, secret); } + br.close(); if (!this.persistenceProblemWithSecrets) { this.logger.fine("Read " + this.secretsForHashingIPAddresses.size() + " secrets for "

1 0

[ooni-probe/master] Add wrapper scripts to avoid PYTHONPATH madness!
by art＠torproject.org 24 Jul '12

24 Jul '12

commit 5cfd7e7bff7728557a5a628f86c2922837f2b24d Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Jul 24 22:41:45 2012 +0200 Add wrapper scripts to avoid PYTHONPATH madness! --- bin/oonib | 4 ++++ bin/ooniprobe | 4 ++++ oonib/lib/__init__.py | 6 ++++-- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/bin/oonib b/bin/oonib new file mode 100755 index 0000000..df7c62f --- /dev/null +++ b/bin/oonib @@ -0,0 +1,4 @@ +#!/bin/sh +ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )" +export PYTHONPATH=$PYTHONPATH:$ROOT +twistd -ny $ROOT/oonib/oonibackend.py diff --git a/bin/ooniprobe b/bin/ooniprobe new file mode 100755 index 0000000..505e1c2 --- /dev/null +++ b/bin/ooniprobe @@ -0,0 +1,4 @@ +#!/bin/sh +ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )" +export PYTHONPATH=$PYTHONPATH:$ROOT +python $ROOT/ooni/ooniprobe.py diff --git a/oonib/lib/__init__.py b/oonib/lib/__init__.py index 2f1b6f9..5b4380a 100644 --- a/oonib/lib/__init__.py +++ b/oonib/lib/__init__.py @@ -1,6 +1,8 @@ from ooni.utils import Storage from ooni.utils.config import Config +import os +root = os.path.normpath(os.path.join(os.path.realpath(__file__), '../../')) config = Storage() -config.main = Config('main', 'oonibackend.conf') -config.daphn3 = Config('daphn3', 'oonibackend.conf') +config.main = Config('main', os.path.join(root, 'oonibackend.conf')) +config.daphn3 = Config('daphn3', os.path.join(root, 'oonibackend.conf'))

1 0

[ooni-probe/master] Restructure directories to reflect architecture spec.
by art＠torproject.org 24 Jul '12

24 Jul '12

commit 178ab96765ddec35d0dc42e764d2bd45dd361d23 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Jul 24 15:16:04 2012 +0200 Restructure directories to reflect architecture spec. --- oonib/README.md | 7 ++++ oonib/backends/daphn3.py | 37 -------------------- oonib/backends/dns.py | 16 --------- oonib/backends/http.py | 74 ----------------------------------------- oonib/backends/ssl.py | 7 ---- oonib/common.py | 6 --- oonib/lib/__init__.py | 6 +++ oonib/lib/ssl.py | 7 ++++ oonib/oonibackend.py | 10 +++--- oonib/testhelpers/daphn3.py | 37 ++++++++++++++++++++ oonib/testhelpers/dns.py | 16 +++++++++ oonib/testhelpers/http.py | 74 +++++++++++++++++++++++++++++++++++++++++ 12 files changed, 152 insertions(+), 145 deletions(-) diff --git a/oonib/README.md b/oonib/README.md index 6823d06..2d6caba 100644 --- a/oonib/README.md +++ b/oonib/README.md @@ -1,3 +1,10 @@ +# Dependencies + +The extra dependencies necessary to run OONIB are: + +* cyclone: https://github.com/fiorix/cyclone +* + # Generate self signed certs for OONIB openssl genrsa -des3 -out private.key 4096 diff --git a/oonib/backends/__init__.py b/oonib/backends/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/oonib/backends/daphn3.py b/oonib/backends/daphn3.py deleted file mode 100644 index 59bdeea..0000000 --- a/oonib/backends/daphn3.py +++ /dev/null @@ -1,37 +0,0 @@ -from twisted.internet import protocol -from twisted.internet.error import ConnectionDone - -from oonib.common import config - -from ooni.plugoo import reports -from ooni.protocols.daphn3 import Mutator, Daphn3Protocol -from ooni.protocols.daphn3 import read_pcap - -class Daphn3Server(protocol.ServerFactory): - """ - This is the main class that deals with the daphn3 server side component. - We keep track of global state of every client here. - Every client is identified by their IP address and the state of mutation is - stored by using their IP address as a key. This may lead to some bugs if - two different clients are sharing the same IP, but hopefully the - probability of such thing is not that likely. - """ - protocol = Daphn3Protocol - mutations = {} - def buildProtocol(self, addr): - p = self.protocol() - p.factory = self - p.factory.steps = read_pcap(config.daphn3.pcap_file) - - if addr.host not in self.mutations: - self.mutations[addr.host] = Mutator(p.steps) - else: - print "Moving on to next mutation" - if not self.mutations[addr.host].next_mutation(): - self.mutations.pop(addr.host) - try: - p.mutator = self.mutations[addr.host] - except: - pass - return p - diff --git a/oonib/backends/dns.py b/oonib/backends/dns.py deleted file mode 100644 index 689d1a4..0000000 --- a/oonib/backends/dns.py +++ /dev/null @@ -1,16 +0,0 @@ -from twisted.internet.protocol import Factory, Protocol -from twisted.internet import reactor -from twisted.names import dns -from twisted.names import client, server - -class ProxyDNSServer(server.DNSServerFactory): - def __init__(self, authorities = None, - caches = None, clients = None, - verbose = 0): - resolver = client.Resolver(servers=[('8.8.8.8', 53)]) - server.DNSServerFactory.__init__(self, authorities = authorities, - caches = caches, clients = [resolver], - verbose = verbose) - def handleQuery(self, message, protocol, address): - print message, protocol, address - server.DNSServerFactory.handleQuery(self, message, protocol, address) diff --git a/oonib/backends/http.py b/oonib/backends/http.py deleted file mode 100644 index 79a487b..0000000 --- a/oonib/backends/http.py +++ /dev/null @@ -1,74 +0,0 @@ -import json -import random -import string - -from twisted.application import internet, service -from twisted.internet import protocol, reactor, defer -from twisted.protocols import basic -from twisted.web import resource, server, static -from twisted.web.microdom import escape - -server.version = "Apache" - -class HTTPRandomPage(resource.Resource): - """ - This generates a random page of arbitrary length and containing the string - selected by the user. - The format is the following: - /random/<length>/<keyword> - """ - isLeaf = True - def _gen_random_string(self, length): - return ''.join(random.choice(string.letters) for x in range(length)) - - def genRandomPage(self, length=100, keyword=None): - data = self._gen_random_string(length/2) - if keyword: - data += keyword - data += self._gen_random_string(length - length/2) - data += '\n' - return data - - def render(self, request): - length = 100 - keyword = None - path_parts = request.path.split('/') - if len(path_parts) > 2: - length = int(path_parts[2]) - if length > 100000: - length = 100000 - - if len(path_parts) > 3: - keyword = escape(path_parts[3]) - - return self.genRandomPage(length, keyword) - -class HTTPReturnHeaders(resource.Resource): - """ - This returns the headers being sent by the client in JSON format. - """ - isLeaf = True - def render(self, request): - req_headers = request.getAllHeaders() - return json.dumps(req_headers) - -class HTTPSendHeaders(resource.Resource): - """ - This sends to the client the headers that they send inside of the POST - request encoded in json. - """ - isLeaf = True - def render_POST(self, request): - headers = json.loads(request.content.read()) - for header, value in headers.items(): - request.setHeader(str(header), str(value)) - return '' - -class HTTPBackend(resource.Resource): - def __init__(self): - resource.Resource.__init__(self) - self.putChild('random', HTTPRandomPage()) - self.putChild('returnheaders', HTTPReturnHeaders()) - self.putChild('sendheaders', HTTPSendHeaders()) - - diff --git a/oonib/backends/ssl.py b/oonib/backends/ssl.py deleted file mode 100644 index 5f19686..0000000 --- a/oonib/backends/ssl.py +++ /dev/null @@ -1,7 +0,0 @@ -from twisted.internet import ssl - -class SSLContext(ssl.DefaultOpenSSLContextFactory): - def __init__(self, config): - ssl.DefaultOpenSSLContextFactory.__init__(self, config.main.ssl_private_key, - config.main.ssl_certificate) - diff --git a/oonib/common.py b/oonib/common.py deleted file mode 100644 index 2f1b6f9..0000000 --- a/oonib/common.py +++ /dev/null @@ -1,6 +0,0 @@ -from ooni.utils import Storage -from ooni.utils.config import Config - -config = Storage() -config.main = Config('main', 'oonibackend.conf') -config.daphn3 = Config('daphn3', 'oonibackend.conf') diff --git a/oonib/lib/__init__.py b/oonib/lib/__init__.py new file mode 100644 index 0000000..2f1b6f9 --- /dev/null +++ b/oonib/lib/__init__.py @@ -0,0 +1,6 @@ +from ooni.utils import Storage +from ooni.utils.config import Config + +config = Storage() +config.main = Config('main', 'oonibackend.conf') +config.daphn3 = Config('daphn3', 'oonibackend.conf') diff --git a/oonib/lib/ssl.py b/oonib/lib/ssl.py new file mode 100644 index 0000000..5f19686 --- /dev/null +++ b/oonib/lib/ssl.py @@ -0,0 +1,7 @@ +from twisted.internet import ssl + +class SSLContext(ssl.DefaultOpenSSLContextFactory): + def __init__(self, config): + ssl.DefaultOpenSSLContextFactory.__init__(self, config.main.ssl_private_key, + config.main.ssl_certificate) + diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index c5a866b..d6ba3f0 100755 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -16,11 +16,11 @@ from twisted.web import resource, server, static from twisted.web.microdom import escape from twisted.names import dns -from oonib.common import config -from oonib.backends.http import HTTPBackend -from oonib.backends.ssl import SSLContext -from oonib.backends.dns import ProxyDNSServer -from oonib.backends.daphn3 import Daphn3Server +from oonib.lib import config +from oonib.testhelpers.http import HTTPBackend +from oonib.lib.ssl import SSLContext +from oonib.testhelpers.dns import ProxyDNSServer +from oonib.testhelpers.daphn3 import Daphn3Server # This tells twisted to set the server.version = config.main.server_version diff --git a/oonib/testhelpers/__init__.py b/oonib/testhelpers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/oonib/testhelpers/daphn3.py b/oonib/testhelpers/daphn3.py new file mode 100644 index 0000000..59bdeea --- /dev/null +++ b/oonib/testhelpers/daphn3.py @@ -0,0 +1,37 @@ +from twisted.internet import protocol +from twisted.internet.error import ConnectionDone + +from oonib.common import config + +from ooni.plugoo import reports +from ooni.protocols.daphn3 import Mutator, Daphn3Protocol +from ooni.protocols.daphn3 import read_pcap + +class Daphn3Server(protocol.ServerFactory): + """ + This is the main class that deals with the daphn3 server side component. + We keep track of global state of every client here. + Every client is identified by their IP address and the state of mutation is + stored by using their IP address as a key. This may lead to some bugs if + two different clients are sharing the same IP, but hopefully the + probability of such thing is not that likely. + """ + protocol = Daphn3Protocol + mutations = {} + def buildProtocol(self, addr): + p = self.protocol() + p.factory = self + p.factory.steps = read_pcap(config.daphn3.pcap_file) + + if addr.host not in self.mutations: + self.mutations[addr.host] = Mutator(p.steps) + else: + print "Moving on to next mutation" + if not self.mutations[addr.host].next_mutation(): + self.mutations.pop(addr.host) + try: + p.mutator = self.mutations[addr.host] + except: + pass + return p + diff --git a/oonib/testhelpers/dns.py b/oonib/testhelpers/dns.py new file mode 100644 index 0000000..689d1a4 --- /dev/null +++ b/oonib/testhelpers/dns.py @@ -0,0 +1,16 @@ +from twisted.internet.protocol import Factory, Protocol +from twisted.internet import reactor +from twisted.names import dns +from twisted.names import client, server + +class ProxyDNSServer(server.DNSServerFactory): + def __init__(self, authorities = None, + caches = None, clients = None, + verbose = 0): + resolver = client.Resolver(servers=[('8.8.8.8', 53)]) + server.DNSServerFactory.__init__(self, authorities = authorities, + caches = caches, clients = [resolver], + verbose = verbose) + def handleQuery(self, message, protocol, address): + print message, protocol, address + server.DNSServerFactory.handleQuery(self, message, protocol, address) diff --git a/oonib/testhelpers/http.py b/oonib/testhelpers/http.py new file mode 100644 index 0000000..79a487b --- /dev/null +++ b/oonib/testhelpers/http.py @@ -0,0 +1,74 @@ +import json +import random +import string + +from twisted.application import internet, service +from twisted.internet import protocol, reactor, defer +from twisted.protocols import basic +from twisted.web import resource, server, static +from twisted.web.microdom import escape + +server.version = "Apache" + +class HTTPRandomPage(resource.Resource): + """ + This generates a random page of arbitrary length and containing the string + selected by the user. + The format is the following: + /random/<length>/<keyword> + """ + isLeaf = True + def _gen_random_string(self, length): + return ''.join(random.choice(string.letters) for x in range(length)) + + def genRandomPage(self, length=100, keyword=None): + data = self._gen_random_string(length/2) + if keyword: + data += keyword + data += self._gen_random_string(length - length/2) + data += '\n' + return data + + def render(self, request): + length = 100 + keyword = None + path_parts = request.path.split('/') + if len(path_parts) > 2: + length = int(path_parts[2]) + if length > 100000: + length = 100000 + + if len(path_parts) > 3: + keyword = escape(path_parts[3]) + + return self.genRandomPage(length, keyword) + +class HTTPReturnHeaders(resource.Resource): + """ + This returns the headers being sent by the client in JSON format. + """ + isLeaf = True + def render(self, request): + req_headers = request.getAllHeaders() + return json.dumps(req_headers) + +class HTTPSendHeaders(resource.Resource): + """ + This sends to the client the headers that they send inside of the POST + request encoded in json. + """ + isLeaf = True + def render_POST(self, request): + headers = json.loads(request.content.read()) + for header, value in headers.items(): + request.setHeader(str(header), str(value)) + return '' + +class HTTPBackend(resource.Resource): + def __init__(self): + resource.Resource.__init__(self) + self.putChild('random', HTTPRandomPage()) + self.putChild('returnheaders', HTTPReturnHeaders()) + self.putChild('sendheaders', HTTPSendHeaders()) + +

1 0

[ooni-probe/master] Improve OONI(B) design and fix Daphn3
by art＠torproject.org 24 Jul '12

24 Jul '12

commit 64ed356e8cde9d6f866205f978cf7c895ed28885 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Jul 24 21:23:52 2012 +0200 Improve OONI(B) design and fix Daphn3 * Create a new kind of test functionality called tool * Add support for terminating a test via .end() * Fix some crazy off by one bugs in Daphn3 * Fix unit tests by starting reactor in the right place * Make Daphn3 into an OONI tool that generates a YAML file --- ooni/ooniprobe.py | 13 +++++-- ooni/plugins/daphn3.py | 66 +++++++++++++++++++++++++++++++---- ooni/plugoo/tests.py | 7 ++++ ooni/plugoo/work.py | 21 +++++++++--- ooni/protocols/daphn3.py | 76 +++++++++++++++++++++++++--------------- oonib/README.md | 2 +- oonib/lib/ssl.py | 3 +- oonib/oonibackend.conf.sample | 1 + oonib/testhelpers/daphn3.py | 17 +++++++-- 9 files changed, 155 insertions(+), 51 deletions(-) diff --git a/ooni/ooniprobe.py b/ooni/ooniprobe.py index d6de6a6..b75deca 100755 --- a/ooni/ooniprobe.py +++ b/ooni/ooniprobe.py @@ -79,13 +79,21 @@ def runTest(test, options, global_options, reactor=reactor): if 'resume' in options: resume = options['resume'] - wgen = work.WorkGenerator(test_class(options, global_options, report, - reactor=reactor), + test = test_class(options, global_options, report, reactor=reactor) + if test.tool: + test.runTool() + return + + if test.end: + return + + wgen = work.WorkGenerator(test, dict(options), start=resume) for x in wgen: worker.push(x) + reactor.run() class Options(usage.Options): tests = plugoo.keys() @@ -132,5 +140,4 @@ if __name__ == "__main__": sys.exit(1) runTest(config.subCommand, config.subOptions, config) - reactor.run() diff --git a/ooni/plugins/daphn3.py b/ooni/plugins/daphn3.py index 82a3331..6c1186e 100644 --- a/ooni/plugins/daphn3.py +++ b/ooni/plugins/daphn3.py @@ -8,6 +8,7 @@ from twisted.python import usage from twisted.plugin import IPlugin from twisted.internet import protocol, endpoints +from ooni.plugoo import reports from ooni.plugoo.tests import ITest, OONITest from ooni.plugoo.assets import Asset from ooni.protocols import daphn3 @@ -15,40 +16,53 @@ from ooni.utils import log class Daphn3ClientProtocol(daphn3.Daphn3Protocol): def connectionMade(self): + print "I have made a connection!" self.next_state() def connectionLost(self, reason): - print "LOST!" + pass class Daphn3ClientFactory(protocol.ClientFactory): protocol = Daphn3ClientProtocol mutator = None steps = None + test = None + report = reports.Report('daphn3', 'daphn3.yamlooni') def buildProtocol(self, addr): p = self.protocol() + p.report = self.report p.factory = self if self.steps: p.steps = self.steps if not self.mutator: self.mutator = daphn3.Mutator(p.steps) - p.mutator = self.mutator else: print "Moving on to next mutation" - self.mutator.next_mutation() + self.mutator.next() + p.mutator = self.mutator return p def clientConnectionFailed(self, reason): print "We failed connecting the the OONIB" print "Cannot perform test. Perhaps it got blocked?" print "Please report this to tor-assistants(a)torproject.org" + self.test.end(d) def clientConnectionLost(self, reason): print "Connection Lost." class daphn3Args(usage.Options): - optParameters = [['pcap', 'f', None, 'PCAP file to take as input'], + optParameters = [['pcap', 'f', None, + 'PCAP to read for generating the YAML output'], + + ['output', 'o', 'daphn3.yaml', + 'What file should be written'], + + ['yaml', 'y', None, + 'The input file to the test'], + ['host', 'h', None, 'Target Hostname'], ['port', 'p', None, 'Target port number'], ['resume', 'r', 0, 'Resume at this index']] @@ -65,30 +79,66 @@ class daphn3Test(OONITest): local_options = None steps = None + def initialize(self): if not self.local_options: + self.end() return #pass self.factory = Daphn3ClientFactory() - self.steps = daphn3.read_pcap(self.local_options['pcap']) + self.factory.test = self + + if self.local_options['pcap']: + self.tool = True + + elif self.local_options['yaml']: + self.steps = daphn3.read_yaml(self.local_options['yaml']) + + else: + log.msg("Not enough inputs specified to the test") + self.end() + + def runTool(self): + import yaml + pcap = daphn3.read_pcap(self.local_options['pcap']) + f = open(self.local_options['output'], 'w') + f.write(yaml.dump(pcap)) + f.close() def control(self, exp_res, args): - mutation = self.factory.mutator.get_mutation(0) + try: + mutation = self.factory.mutator.get(0) + except: + mutation = None return {'mutation_number': args['mutation'], 'value': mutation} + def _failure(self, *argc, **kw): + print "We failed connecting the the OONIB" + print "Cannot perform test. Perhaps it got blocked?" + print "Please report this to tor-assistants(a)torproject.org" + print "Traceback: %s %s" % (argc, kw) + self.end() + def experiment(self, args): log.msg("Doing mutation %s" % args['mutation']) self.factory.steps = self.steps host = self.local_options['host'] port = int(self.local_options['port']) log.msg("Connecting to %s:%s" % (host, port)) + + if self.ended: + return + endpoint = endpoints.TCP4ClientEndpoint(self.reactor, host, port) - return endpoint.connect(self.factory) + d = endpoint.connect(self.factory) + d.addErrback(self._failure) + return d #return endpoint.connect(Daphn3ClientFactory) def load_assets(self): if not self.steps: - print "No asset!" + print "Error: No assets!" + self.end() return {} mutations = 0 for x in self.steps: diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index 9b6ea26..482b8bc 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -25,6 +25,7 @@ class OONITest(object): # By default we set this to False, meaning that we don't block blocking = False reactor = None + tool = False def __init__(self, local_options, global_options, report, ooninet=None, reactor=None): @@ -58,6 +59,12 @@ class OONITest(object): return "<OONITest %s %s %s>" % (self.options, self.global_options, self.assets) + def end(self): + """ + State that the current test should finish. + """ + self.ended = True + def finished(self, control): """ The Test has finished running, we must now calculate the test runtime diff --git a/ooni/plugoo/work.py b/ooni/plugoo/work.py index 6106456..db88fbf 100644 --- a/ooni/plugoo/work.py +++ b/ooni/plugoo/work.py @@ -43,12 +43,22 @@ class Worker(object): @param r: the return value of a previous test. """ - self._running -= 1 + if self._running > 0: + self._running -= 1 + if self._running < self.maxconcurrent and self._queued: workunit, d = self._queued.pop(0) asset, test, idx = workunit - self._running += 1 - actuald = test.startTest(asset).addBoth(self._run) + while test.ended and workunit: + try: + workunit, d = self._queued.pop(0) + asset, test, idx = workunit + except: + workunit = None + + if not test.ended: + self._running += 1 + actuald = test.startTest(asset).addBoth(self._run) if isinstance(r, failure.Failure): # XXX probably we should be doing something to retry test running @@ -71,8 +81,9 @@ class Worker(object): """ if self._running < self.maxconcurrent: asset, test, idx = workunit - self._running += 1 - return test.startTest(asset).addBoth(self._run) + if not test.ended: + self._running += 1 + return test.startTest(asset).addBoth(self._run) d = defer.Deferred() self._queued.append((workunit, d)) diff --git a/ooni/protocols/daphn3.py b/ooni/protocols/daphn3.py index 093929b..d7471ab 100644 --- a/ooni/protocols/daphn3.py +++ b/ooni/protocols/daphn3.py @@ -1,13 +1,14 @@ +import sys +import yaml + from twisted.internet import protocol, defer from twisted.internet.error import ConnectionDone +from scapy.all import * + from ooni.utils import log from ooni.plugoo import reports -import sys -from scapy.all import * -import yaml - def read_pcap(filename): """ @param filename: Filesystem path to the pcap. @@ -65,6 +66,12 @@ def read_pcap(filename): return messages +def read_yaml(filename): + f = open(filename) + obj = yaml.load(f) + f.close() + return obj + class Mutator: idx = 0 step = 0 @@ -74,10 +81,10 @@ class Mutator: def __init__(self, steps): """ - @param steps: array of dicts containing as keys data and wait. Data is - the content of the ith packet to be sent and wait is how - much we should wait before mutating the packet of the - next step. + @param steps: array of dicts for the steps that must be gone over by + the mutator. Looks like this: + [{"sender": "client", "data": "\xde\xad\xbe\xef"}, + {"sender": "server", "data": "\xde\xad\xbe\xef"}] """ self.steps = steps @@ -105,7 +112,7 @@ class Mutator: current_state = {'idx': self.idx, 'step': self.step} return current_state - def next_mutation(self): + def next(self): """ Increases by one the mutation state. @@ -121,9 +128,9 @@ class Mutator: returns True if another mutation is available. returns False if all the possible mutations have been done. """ - if (self.step + 1) > len(self.steps): + if (self.step) == len(self.steps): # Hack to stop once we have gone through all the steps - print "[Mutator.next_mutation()] I believe I have gone over all steps" + print "[Mutator.next()] I believe I have gone over all steps" print " Stopping!" self.waiting = True return False @@ -132,35 +139,40 @@ class Mutator: current_idx = self.idx current_step = self.step current_data = self.steps[current_step]['data'] - try: - data_to_receive = len(self.steps[current_step +1 ]['data']) - print "[Mutator.next_mutation()] Managed to receive some data." - except: - print "[Mutator.next_mutation()] No more data to receive." + + if 0: + print "current_step: %s" % current_step + print "current_idx: %s" % current_idx + print "current_data: %s" % current_data + print "steps: %s" % len(self.steps) + print "waiting_step: %s" % self.waiting_step + + data_to_receive = len(self.steps[current_step]['data']) if self.waiting and self.waiting_step == data_to_receive: - print "[Mutator.next_mutation()] I am no longer waiting" + print "[Mutator.next()] I am no longer waiting" log.debug("I am no longer waiting.") self.waiting = False self.waiting_step = 0 self.idx = 0 elif self.waiting: - print "[Mutator.next_mutation()] Waiting some more." + print "[Mutator.next()] Waiting some more." log.debug("Waiting some more.") self.waiting_step += 1 elif current_idx >= len(current_data): - print "[Mutator.next_mutation()] Entering waiting mode." + print "[Mutator.next()] Entering waiting mode." log.debug("Entering waiting mode.") self.step += 1 self.idx = 0 self.waiting = True + log.debug("current index %s" % current_idx) log.debug("current data %s" % len(current_data)) return True - def get_mutation(self, step): + def get(self, step): """ Returns the current packet to be sent to the wire. If no mutation is necessary it will return the plain data. @@ -172,11 +184,11 @@ class Mutator: returns the mutated packet for the specified step. """ if step != self.step or self.waiting: - log.debug("[Mutator.get_mutation()] I am not going to do anything :)") + log.debug("[Mutator.get()] I am not going to do anything :)") return self.steps[step]['data'] data = self.steps[step]['data'] - print "Mutating %s with idx %s" % (data, self.idx) + #print "Mutating %s with idx %s" % (data, self.idx) return self._mutate(data, self.idx) class Daphn3Protocol(protocol.Protocol): @@ -205,11 +217,13 @@ class Daphn3Protocol(protocol.Protocol): print "[Daphn3Protocol.next_state] No mutator. There is no point to stay on this earth." self.transport.loseConnection() return + if self.role is self.steps[self.state]['sender']: print "[Daphn3Protocol.next_state] I am a sender" - data = self.mutator.get_mutation(self.state) + data = self.mutator.get(self.state) self.transport.write(data) self.to_receive_data = 0 + else: print "[Daphn3Protocol.next_state] I am a receiver" self.to_receive_data = len(self.steps[self.state]['data']) @@ -228,11 +242,8 @@ class Daphn3Protocol(protocol.Protocol): print "I don't have a mutator. My life means nothing." self.transport.loseConnection() return - if len(self.steps) <= self.state: - print "I have reached the end of the state machine" - print "Censorship fingerprint bruteforced!" - report = {'mutator_state': self.mutator.state()} - self.report(report) + + if len(self.steps) == self.state: self.transport.loseConnection() return @@ -256,7 +267,7 @@ class Daphn3Protocol(protocol.Protocol): print "The connection was closed because of %s" % report['reason'] print "State %s, Mutator %s" % (report['proto_state'], report['mutator_state']) - self.mutator.next_mutation() + self.mutator.next() @@ -275,6 +286,13 @@ class Daphn3Protocol(protocol.Protocol): report['trigger'] = 'did not finish state walk' self.censorship_detected(report) + else: + print "I have reached the end of the state machine" + print "Censorship fingerprint bruteforced!" + report = {'mutator_state': self.mutator.state()} + self.report(report) + return + if reason.check(ConnectionDone): print "Connection closed cleanly" else: diff --git a/oonib/README.md b/oonib/README.md index 2d6caba..8136b06 100644 --- a/oonib/README.md +++ b/oonib/README.md @@ -2,8 +2,8 @@ The extra dependencies necessary to run OONIB are: +* twisted-names * cyclone: https://github.com/fiorix/cyclone -* # Generate self signed certs for OONIB diff --git a/oonib/lib/ssl.py b/oonib/lib/ssl.py index 5f19686..094bdc9 100644 --- a/oonib/lib/ssl.py +++ b/oonib/lib/ssl.py @@ -1,7 +1,8 @@ from twisted.internet import ssl +from oonib.lib import config class SSLContext(ssl.DefaultOpenSSLContextFactory): - def __init__(self, config): + def __init__(self, *args, **kw): ssl.DefaultOpenSSLContextFactory.__init__(self, config.main.ssl_private_key, config.main.ssl_certificate) diff --git a/oonib/oonibackend.conf.sample b/oonib/oonibackend.conf.sample index a5cbbd3..021e0e2 100644 --- a/oonib/oonibackend.conf.sample +++ b/oonib/oonibackend.conf.sample @@ -8,3 +8,4 @@ ssl_private_key = /path/to/private.key ssl_certificate = /path/to/certificate.crt [daphn3] pcap_file = /path/to/server.pcap +yaml_file = /path/to/server.yaml diff --git a/oonib/testhelpers/daphn3.py b/oonib/testhelpers/daphn3.py index 59bdeea..2dd2801 100644 --- a/oonib/testhelpers/daphn3.py +++ b/oonib/testhelpers/daphn3.py @@ -1,11 +1,11 @@ from twisted.internet import protocol from twisted.internet.error import ConnectionDone -from oonib.common import config +from oonib.lib import config from ooni.plugoo import reports from ooni.protocols.daphn3 import Mutator, Daphn3Protocol -from ooni.protocols.daphn3 import read_pcap +from ooni.protocols.daphn3 import read_pcap, read_yaml class Daphn3Server(protocol.ServerFactory): """ @@ -21,13 +21,22 @@ class Daphn3Server(protocol.ServerFactory): def buildProtocol(self, addr): p = self.protocol() p.factory = self - p.factory.steps = read_pcap(config.daphn3.pcap_file) + + if config.daphn3.yaml_file: + steps = read_yaml(config.daphn3.yaml_file) + elif config.daphn3.pcap_file: + steps = read_pcap(config.daphn3.pcap_file) + else: + print "Error! No PCAP, nor YAML file provided." + steps = None + + p.factory.steps = steps if addr.host not in self.mutations: self.mutations[addr.host] = Mutator(p.steps) else: print "Moving on to next mutation" - if not self.mutations[addr.host].next_mutation(): + if not self.mutations[addr.host].next(): self.mutations.pop(addr.host) try: p.mutator = self.mutations[addr.host]

1 0