- tor-commits - lists.torproject.org

[metrics-web/master] Add bridge-blocking case study PDF.
by karsten＠torproject.org 15 Sep '11

15 Sep '11

commit 1e17bd6b35a87155494b5e8461151bd2dd867bf5 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Sep 15 20:54:58 2011 +0200 Add bridge-blocking case study PDF. --- web/WEB-INF/papers.jsp | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diff --git a/web/WEB-INF/papers.jsp b/web/WEB-INF/papers.jsp index 311944a..f6aa7bd 100644 --- a/web/WEB-INF/papers.jsp +++ b/web/WEB-INF/papers.jsp @@ -45,6 +45,10 @@ been included in the <a href="#papers">papers</a> above or in the daily updated <a href="graphs.html">graphs</a>. <ul> + <li>Case study: Learning whether a Tor bridge is blocked by + looking at its aggregate usage statistics, Part one (<a + href="papers/blocking-2011-09-15.pdf">PDF</a>, 767K from + September 15, 2011).</li> <li>An anomaly-based censorship-detection system for Tor (<a href="papers/detector-2011-09-09.pdf">PDF</a>, 106K from September 9, 2011).</li>

1 0

[metrics-tasks/master] Add sources for bridge-blocking case study (#4030).
by karsten＠torproject.org 15 Sep '11

15 Sep '11

commit 73f278eb1ab44d6691dbbe349f608a1cbb72b42b Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Sep 15 20:49:40 2011 +0200 Add sources for bridge-blocking case study (#4030). --- task-4030/.gitignore | 7 + task-4030/DetectBridgeBlockings.java | 140 ++++++++++++++++++++ task-4030/README | 9 ++ task-4030/blocking.tex | 237 ++++++++++++++++++++++++++++++++++ task-4030/bridge-blockings.R | 60 +++++++++ 5 files changed, 453 insertions(+), 0 deletions(-) diff --git a/task-4030/.gitignore b/task-4030/.gitignore new file mode 100644 index 0000000..94e7a87 --- /dev/null +++ b/task-4030/.gitignore @@ -0,0 +1,7 @@ +bridge-stats +*.csv +*.png +*.aux +*.log +*.pdf + diff --git a/task-4030/DetectBridgeBlockings.java b/task-4030/DetectBridgeBlockings.java new file mode 100644 index 0000000..ca9416a --- /dev/null +++ b/task-4030/DetectBridgeBlockings.java @@ -0,0 +1,140 @@ +import java.io.*; +import java.text.*; +import java.util.*; + +/** + * Processes previously extracted bridge usage statistics to detect + * possible bridge blockings in a given country. + * + * Before running this tool, make sure you download (sanitized) bridge + * descriptors from https://metrics.torproject.org/data.html#bridgedesc, + * extract them to a local directory, and run the following command: + * + * $ grep -Rl "^bridge-ips [a-z]" bridge-descriptors-* | \ + * xargs -I {} grep -E "^extra-info|^bridge" {} > bridge-stats + **/ + +public class DetectBridgeBlockings { + public static void main(String[] args) throws Exception { + + /* Run the analysis for the country with this country code. */ + final String COUNTRY = "cn"; + + /* Consider bridges with at most this many users as potentially + * blocked. */ + final int BLOCKED_THRESHOLD = 36; + + /* Consider bridges blocked that report no more than BLOCKED_THRESHOLD + * users for at least this number of days after having reported more + * than BLOCKED_THRESHOLD users at least once before. */ + final int BLOCKED_DAYS = 7; + + /* Only include bridges in the results that have reported at least + * this number of statistics. */ + final int MIN_REPORTS = 30; + + /* Begin of analysis interval (inclusive). */ + final String INTERVAL_BEGIN = "2010-01-01"; + + /* End of analysis interval (inclusive). */ + final String INTERVAL_END = "2010-07-31"; + + /* Check whether we have an input file. */ + File inputFile = new File("bridge-stats"); + if (!inputFile.exists()) { + System.out.println("File " + inputFile + " not found. Please see " + + "the README."); + System.exit(1); + } + + /* Read the relevant bridge statistics parts into memory. */ + BufferedReader br = new BufferedReader(new FileReader(inputFile)); + String line, fingerprint = null, date = null; + SortedMap<String, SortedMap<String, Integer>> usersPerBridgeAndDay = + new TreeMap<String, SortedMap<String, Integer>>(); + while ((line = br.readLine()) != null) { + if (line.startsWith("extra-info ")) { + fingerprint = line.split(" ")[2]; + } else if (line.startsWith("bridge-stats-end ")) { + date = line.substring("bridge-stats-end ".length(), + "bridge-stats-end YYYY-MM-DD".length()); + } else if (line.startsWith("bridge-ips ")) { + if (date.compareTo(INTERVAL_BEGIN) < 0 || + date.compareTo(INTERVAL_END) > 0) { + continue; + } + int ipsFromCountry = 0; + for (String part : line.split(" ")[1].split(",")) { + String country = part.split("=")[0]; + if (country.equals(COUNTRY)) { + ipsFromCountry = Integer.parseInt(part.split("=")[1]); + break; + } + } + if (!usersPerBridgeAndDay.containsKey(fingerprint)) { + usersPerBridgeAndDay.put(fingerprint, + new TreeMap<String, Integer>()); + } + usersPerBridgeAndDay.get(fingerprint).put(date, ipsFromCountry); + } + } + br.close(); + + /* Write processed statistics for COUNTRY to disk including a column + * for suspected blockings. */ + SimpleDateFormat dateFormat = new SimpleDateFormat( + "yyyy-MM-dd"); + dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + BufferedWriter bw = new BufferedWriter(new FileWriter( + "bridge-blockings.csv")); + bw.write("fingerprint,date,ips,blocked\n"); + for (Map.Entry<String, SortedMap<String, Integer>> e : + usersPerBridgeAndDay.entrySet()) { + fingerprint = e.getKey(); + SortedMap<String, Integer> usersPerDay = e.getValue(); + if (usersPerDay.size() < MIN_REPORTS) { + continue; + } + long lastDateMillis = 0L; + String lastExceededBlockedThreshold = null; + SortedSet<String> datesNotExceedingBlockedThreshold = + new TreeSet<String>(); + for (Map.Entry<String, Integer> e1 : usersPerDay.entrySet()) { + date = e1.getKey(); + long dateMillis = dateFormat.parse(date).getTime(); + while (lastDateMillis > 0L && + dateMillis > lastDateMillis + 24L * 60L * 60L * 1000L) { + lastDateMillis += 24L * 60L * 60L * 1000L; + bw.write(fingerprint + "," + dateFormat.format(lastDateMillis) + + ",NA,NA\n"); + } + lastDateMillis = dateMillis; + int ips = e1.getValue(); + String bwLinePart = fingerprint + "," + date + "," + ips; + if (ips > BLOCKED_THRESHOLD) { + String blocked = "FALSE"; + if (lastExceededBlockedThreshold != null && + dateFormat.parse(date).getTime() - + dateFormat.parse(lastExceededBlockedThreshold).getTime() + > BLOCKED_DAYS * 24L * 60L * 60L * 1000L) { + blocked = "TRUE"; + } + for (String buffered : datesNotExceedingBlockedThreshold) { + bw.write(buffered + "," + blocked + "\n"); + } + datesNotExceedingBlockedThreshold.clear(); + bw.write(bwLinePart + ",FALSE\n"); + lastExceededBlockedThreshold = date; + } else { + datesNotExceedingBlockedThreshold.add(bwLinePart); + } + } + for (String buffered : datesNotExceedingBlockedThreshold) { + bw.write(buffered + ",TRUE\n"); + } + datesNotExceedingBlockedThreshold.clear(); + } + bw.close(); + } +} + diff --git a/task-4030/README b/task-4030/README new file mode 100644 index 0000000..53ffb68 --- /dev/null +++ b/task-4030/README @@ -0,0 +1,9 @@ +$ grep -Rl "^bridge-ips [a-z]" bridge-descriptors-* | \ + xargs -I {} grep -E "^extra-info|^bridge" {} > bridge-stats + +$ javac DetectBridgeBlockings.java && java DetectBridgeBlockings + +$ R --slave -f bridge-blockings.R + +$ pdflatex blocking.tex + diff --git a/task-4030/blocking.tex b/task-4030/blocking.tex new file mode 100644 index 0000000..bb2ad90 --- /dev/null +++ b/task-4030/blocking.tex @@ -0,0 +1,237 @@ +\documentclass{article} +\usepackage[pdftex]{graphicx} +\usepackage{graphics} +\usepackage{color} +\usepackage{url} + +\begin{document} + +\author{Karsten Loesing\\{\tt karsten(a)torproject.org}} +\title{Case study:\\Learning whether a Tor bridge is blocked\\by looking +at its aggregate usage statistics\\-- Part one --} +\maketitle + +\section{Introduction} + +Tor bridges\footnote{\url{https://www.torproject.org/docs/bridges}} are +relays that are not listed in the main directory. +Clients which cannot access the Tor network directly can try to learn a +few bridge addresses and use these bridges to connect to the Tor network. +Bridges have been introduced to impede censoring the Tor network, but in +the past we experienced successful blocking of bridges in a few countries. + +In this report we investigate whether we can learn that a bridge is +blocked in a given country only by looking at its reported aggregate +statistics on usage by country. +By knowing that a bridge is blocked, we can, for example, avoid giving +out its address to users from that country. + +Learning whether a bridge is blocked is somewhat related to our recent +efforts to detect censorship of direct access to the Tor +network.\footnote{\url{https://metrics.torproject.org/papers/detector-2011-09-09.pdf}} +The main difference is that we want to know which bridges are blocked and +which are not, whereas we don't care which relays are accessible in the +case of blocked direct access. +It's easy to block all relays, but it should be difficult to block all +bridges. + +This report can only be seen as a first step towards researching bridge +blocking. +Even if a bridge reports that it had zero users from a country, we're +lacking the confirmation that the bridge was really blocked. +There can be other reasons for low user numbers which may be completely +unrelated. +The results of this analysis should be considered when actively scanning +bridge reachability from inside a country, both to decide how frequently a +bridge should be scanned and to evaluate how reliable an analysis of +passive usage statistics can be. + +\section{Bridge usage statistics} + +Bridges report aggregate usage statistics on the number of connecting +clients. +Bridges gather these statistics by memorizing unique IP addresses of +connecting clients over 24 hour periods and resolving IP addresses to +country codes using an internal GeoIP database. +Archives of these statistics are available for analysis from the metrics +website.\footnote{\url{https://metrics.torproject.org/data.html#bridgedesc}} +Figure~\ref{fig:bridgeextrainfo} shows an example of bridge usage +statistics. +This bridge observed 41 to 48 connecting clients from Saudi Arabia +(all numbers are rounded up to the next multiple of 8), 33 to 40 +connecting clients from the U.S.A., 25 to 32 from Germany, 25 to 32 from +Iran, and so on. +These connecting clients were observed in the 24~hours (86,400 seconds) +before December 27, 2010, 14:56:29 UTC. + +\begin{figure}[h] +\begin{quote} +\begin{verbatim} +extra-info Unnamed A5FA7F38B02A415E72FE614C64A1E5A92BA99BBD +published 2010-12-27 18:55:01 +[...] +bridge-stats-end 2010-12-27 14:56:29 (86400 s) +bridge-ips sa=48,us=40,de=32,ir=32,[...] +\end{verbatim} +\end{quote} +\caption{Example of aggregate bridge usage statistics} +\label{fig:bridgeextrainfo} +\end{figure} + +An obvious limitation of these bridge usage statistics is that we can only +learn about connecting clients from bridges with at least 24 hours uptime. +It's still unclear how many bridge users are not included in the +statistics because of this, which is left for a different analysis. + +We further decided to exclude bridges running Tor versions 0.2.2.3-alpha +or earlier. +These bridges report similar statistics as the later Tor versions that +we're considering here, but do not enforce a measurement interval of +exactly 24 hours which would have slightly complicated the analysis. +We don't expect the bridge version to have an influence on bridge usage +or on the likelihood of the bridge to be blocked in a given country. + +\section{Case study: China in the first half of 2010} + +The major limitation of this analysis is that we don't have the data +confirming that a bridge was actually blocked. +We may decide on a case-by-case basis whether a blocking is a plausible +explanation for the change in observed users from a given country. +Anything more objective requires additional data, e.g., data obtained from +active reachability scans. + +We decided to investigate bridge usage from China in the first half of +2010 as a case study. +Figure~\ref{fig:bridge-users} shows estimated daily bridge users from China +since July 2009. +The huge slope in September and October 2009 is very likely a result from +China blocking direct access to the Tor network. +It seems plausible that the drops in March and May 2010 result from +attempts to block access to bridges, too. +We're going to focus only on the interval from January to June 2010 which +promises the most interesting results. +We should be able to detect these blockings in the reported statistics of +single bridges. +Obviously, it may be hard or impossible to transfer the findings from this +case study to other countries or situations. + +\begin{figure} +\includegraphics[width=\textwidth]{bridge-users.png} +\caption{Estimated daily bridge users from China} +\label{fig:bridge-users} +\end{figure} + +\paragraph{Definition of bridge blocking} + +We have a few options to define when we consider a bridge to be blocked +from a given country on a given day. + +\begin{itemize} +\item \textbf{Absolute threshold:} +The absolute number of connecting clients from a country falls below a +fixed threshold. +\item \textbf{Relative threshold compared to other countries:} +The fraction of connecting clients from a country drops below a fixed +percent value. +\item \textbf{Estimated interval based on history:} +The absolute or relative number of connecting clients falls outside an +estimated interval based on the recent history. +\end{itemize} + +For this case study we decided to stick with the simplest solution being +an absolute threshold. +We define a somewhat arbitrary threshold of 32 users to decide whether a +bridge is potentially blocked. +A blocked bridge does not necessarily report zero users per day. +A likely explanation for reporting users from a country that blocks a +bridge is that our GeoIP is not 100~\% accurate and reports a few users +which in fact come from other countries. + +The reason against using a relative threshold was that it depends on +development in other countries. +As we can see in the example of China, bridge usage can depend on the +abilty to directly access the Tor network. +A sudden increase in country $A$ could significantly lower the relative +usage in country $B$. +We should probably consider both absolute and relative thresholds in +future investigations. +Maybe we also need to take direct usage numbers into account. + +We also didn't build our analysis upon an estimated interval based on the +recent history, because it's unclear how fast a bridge will be blocked +after being set up. +If it only takes the censor a few hours, the bridge may never see much use +from a country at all. +An estimate based on the bridge's history may not detect the censorship at +all, because it may look like a bridge with only few users from that +country. + +We plan to reconsider other options for deciding that a bridge is blocked +once we have data confirming this. + +\paragraph{Visualization of bridge blockings} + +Figure~\ref{fig:bridge-blockings} shows a subset of the raw bridge usage +statistics for clients connecting from China in the first half of 2010. +Possible blocking events are those when the bridge reports 32 or fewer +connecting clients per day. +These events are marked with red dots. + +We decided to only include bridges in the figure that report at least +100~Chinese clients on at least one day in the whole interval. +Bridges with fewer users than that have a usage pattern that makes it much +more difficult to detect blockings at all. +The figure also shows only bridges reporting statistics on at least 30 +days in the measurement interval. + +\begin{figure}[t] +\includegraphics[width=\textwidth]{bridge-blockings.png} +\caption{Subset of bridge usage statistics for Chinese clients in the +first half of 2010} +\label{fig:bridge-blockings} +\end{figure} + +The single bridge usage plots indicate how difficult it is to detect +blockings only from usage statistics. +About 10 of the displayed 27 plots have a pattern similar to the expected +pattern from Figure~\ref{fig:bridge-users}. +The best examples are probably bridges \verb+C037+ and \verb+D795+. +Interestingly, bridge \verb+A5FA+ was unaffected by the blocking in March +2010, but affected by the blocking in May 2010. + +\paragraph{Aggregating blocking events} + +As the last step of this case study we want to compare observed bridge +users to the number of blocked bridges as detected by our simple threshold +approach. +We would expect most of our bridges to exhibit blockings in March 2010 and +from May 2010 on. +Figure~\ref{fig:bridge-users-blockings} plots users and blocked bridges +over time. +The two plots indicate that our detection algorithm is at least not +totally off. + +\begin{figure}[t] +\includegraphics[width=\textwidth]{bridge-users-blockings.png} +\caption{Estimated users and assumed bridge blockings in China in the +first half of 2010} +\label{fig:bridge-users-blockings} +\end{figure} + +\section{Conclusion} + +Passively collected bridge usage statistics seem to be a useful tool to +detect whether a bridge is blocked from a country. +However, the main conclusion from this analysis is that we're lacking the +data to conduct it usefully. +One way to obtain the data we need are active scans. +When conducting such scans, passively collected statistics may help reduce +the total number and frequency of scans. +For example, when selecting a bridge to scan, the reciprocal of the last +reported number of connecting clients could be used as a probability +weight. +Once we have better data confirming bridge blocking we shall revisit the +criteria for deriving the blocking from usage statistics. + +\end{document} + diff --git a/task-4030/bridge-blockings.R b/task-4030/bridge-blockings.R new file mode 100644 index 0000000..da40186 --- /dev/null +++ b/task-4030/bridge-blockings.R @@ -0,0 +1,60 @@ +library(ggplot2) + +u <- read.csv("bridge-users.csv", stringsAsFactors = FALSE) +u <- u[u$date >= "2009-07-01" & u$date < "2011-08-01", c("date", "cn")] +ggplot(u, aes(x = as.Date(date), y = cn)) + +geom_line(size = 0.75) + +geom_rect(aes(NULL, NULL, xmin = as.Date("2010-01-01"), + xmax = as.Date("2010-06-30"), ymin = -Inf, ymax = Inf, fill = TRUE)) + +scale_fill_manual(name = "", breaks = TRUE, + values = alpha("purple", 0.2)) + +scale_x_date(name = "", major = "6 months", minor = "1 month", + format = "%b %Y") + +scale_y_continuous(name = "", limits = c(0, max(u$cn, na.rm = TRUE))) + +opts(legend.position = "none") +ggsave("bridge-users.png", width = 8, height = 4, dpi = 150) + +b <- read.csv("bridge-blockings.csv", stringsAsFactors = FALSE) +b <- b[b$date >= '2010-01-01' & b$date <= '2010-06-30', ] +fingerprints <- unique(b[b$ips >= 100, "fingerprint"]) +b <- b[b$fingerprint %in% fingerprints, ] + +d <- data.frame(date = b$date, blocked = ifelse(b$ips < 40, 1, NA)) +d <- na.omit(d) +d <- aggregate(d$blocked, by = list(date = d$date), sum) +e <- as.Date(setdiff(seq(from = as.Date("2010-01-01"), + to = as.Date("2010-06-30"), by = "1 day"), as.Date(d$date)), + origin = "1970-01-01") +u <- u[u$date >= '2010-01-01' & u$date <= '2010-06-30', ] +d <- rbind(data.frame(date = u$date, value = u$cn, variable = "Users"), + data.frame(date = d$date, value = d$x, variable = "Blocked Bridges"), + data.frame(date = as.character(e), value = 0, + variable = "Blocked Bridges")) +ggplot(d, aes(x = as.Date(date), y = value)) + +geom_line(size = 0.75) + +facet_grid(variable ~ ., scales = "free_y") + +scale_x_date(name = "", format = "%b %Y", major = "1 month", + minor = "months") + +scale_y_continuous(name = "") +ggsave("bridge-users-blockings.png", width = 8, height = 4, dpi = 150) + +b <- data.frame(date = as.Date(b$date), ips = b$ips, + fingerprint = substr(b$fingerprint, 1, 4), + blocked = ifelse(b$blocked, "red", "black")) +bb <- b +bb[!is.na(b$ips) & b$ips >= 36, "ips"] <- NA +ggplot(b, aes(x = date, y = ips)) + +facet_wrap(~ fingerprint, ncol = 4) + +geom_line(size = 0.75) + +geom_point(size = 0.75) + +geom_hline(yintercept = 32, linetype = 2, size = 0.25) + +geom_point(data = bb, aes(x = date, y = ips), colour = "red", size = 3, + alpha = 0.25) + +scale_x_date(name = "", format = "%b", major = "2 months", + minor = "months") + +scale_y_continuous(name = "", breaks = c(0, 500, 1000)) + +scale_colour_manual(breaks = c("red", "black"), + values = c("red", "black")) + +opts(legend.position = "none") +ggsave("bridge-blockings.png", height = 9, width = 8) +

1 0

[arm/master] Tab completion for interpretor and help commands
by atagar＠torproject.org 15 Sep '11

15 Sep '11

commit 48c1888a715209e88c5962f756d40325a6ae28c1 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Sep 15 08:47:31 2011 -0700 Tab completion for interpretor and help commands Minor interpretor improvements, mostly adding tab completion for the current interpretor commands and anything that will be recognized by '/help'. --- src/util/torConfig.py | 13 +++++++++++++ src/util/torInterpretor.py | 26 +++++++++++++++++++++----- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/src/util/torConfig.py b/src/util/torConfig.py index b2cf345..1d54196 100644 --- a/src/util/torConfig.py +++ b/src/util/torConfig.py @@ -321,6 +321,19 @@ def getConfigDescription(option): CONFIG_DESCRIPTIONS_LOCK.release() return returnVal +def getConfigOptions(): + """ + Provides the configuration options from the loaded man page. This is an empty + list if no man page has been loaded. + """ + + CONFIG_DESCRIPTIONS_LOCK.acquire() + + returnVal = [CONFIG_DESCRIPTIONS[opt].option for opt in CONFIG_DESCRIPTIONS] + + CONFIG_DESCRIPTIONS_LOCK.release() + return returnVal + def getConfigLocation(): """ Provides the location of the torrc, raising an IOError with the reason if the diff --git a/src/util/torInterpretor.py b/src/util/torInterpretor.py index 5733f8a..ce28e9c 100644 --- a/src/util/torInterpretor.py +++ b/src/util/torInterpretor.py @@ -49,11 +49,11 @@ BACKLOG_LIMIT = 100 CONTENT_LIMIT = 20000 GENERAL_HELP = """Interpretor commands include: - help - provides information for interpretor and tor commands/config options - write - saves backlog to a given location - info - general information for a relay - find - searches backlog for lines with the given regex - quit - shuts down the interpretor + /help - provides information for interpretor and tor commands/config options + /write - saves backlog to a given location + /info - general information for a relay + /find - searches backlog for lines with the given regex + /quit - shuts down the interpretor Tor commands include: GETINFO - queries information from tor @@ -228,6 +228,19 @@ class TorControlCompleter: self.commands.append("+LOADCONF") # TODO: another multiline... self.commands.append("TAKEOWNERSHIP") self.commands.append("QUIT") # TODO: give a confirmation when the user does this? + + # adds interpretor commands + for cmd in INTERPRETOR_HELP: + self.commands.append("/" + cmd.lower()) + + # adds help options for the previous commands + baseCmd = set([cmd.split(" ")[0].replace("+", "").replace("/", "") for cmd in self.commands]) + for cmd in baseCmd: + self.commands.append("/help " + cmd) + + # adds /help for tor configuration options + for opt in torConfig.getConfigOptions(): + self.commands.append("/help " + opt) def getMatches(self, text): """ @@ -289,6 +302,9 @@ class ControlInterpretor: arg = arg.upper() + # strip slash if someone enters an interpretor command (ex. "/help /help") + if arg.startswith("/"): arg = arg[1:] + if arg: if arg in INTERPRETOR_HELP: # Provides information for the interpretor argument. This bolds the

1 0

[arm/master] Help content for GETINFO
by atagar＠torproject.org 15 Sep '11

15 Sep '11

commit d3da20b9e311de1604192a28ba63593316a901ac Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Sep 15 09:38:59 2011 -0700 Help content for GETINFO Simple help snippet followed by the 'GETINFO info/names' descriptions. --- src/util/torInterpretor.py | 46 +++++++++++++++++++++++++++++++++++-------- 1 files changed, 37 insertions(+), 9 deletions(-) diff --git a/src/util/torInterpretor.py b/src/util/torInterpretor.py index ce28e9c..21603a1 100644 --- a/src/util/torInterpretor.py +++ b/src/util/torInterpretor.py @@ -98,12 +98,16 @@ Results are deduplicated and the matching portions bolded.""" HELP_QUIT = """Terminates the interpretor.""" -INTERPRETOR_HELP = { +HELP_GETINFO = """Queries the tor process for information. Options are... +""" + +HELP_OPTIONS = { "HELP": ("/help [OPTION]", HELP_HELP), "WRITE": ("/write [PATH]", HELP_WRITE), "INFO": ("/info [relay fingerprint, nickname, or IP address]", HELP_INFO), "FIND": ("/find PATTERN", HELP_FIND), - "QUIT": ("/quit", HELP_QUIT) + "QUIT": ("/quit", HELP_QUIT), + "GETINFO": ("GETINFO [OPTION]", HELP_GETINFO) } class InterpretorClosed(Exception): @@ -230,8 +234,9 @@ class TorControlCompleter: self.commands.append("QUIT") # TODO: give a confirmation when the user does this? # adds interpretor commands - for cmd in INTERPRETOR_HELP: - self.commands.append("/" + cmd.lower()) + for cmd in HELP_OPTIONS: + if HELP_OPTIONS[cmd][0].startswith("/"): + self.commands.append("/" + cmd.lower()) # adds help options for the previous commands baseCmd = set([cmd.split(" ")[0].replace("+", "").replace("/", "") for cmd in self.commands]) @@ -302,19 +307,42 @@ class ControlInterpretor: arg = arg.upper() + # If there's multiple arguments then just take the first. This is + # particularly likely if they're trying to query a full command (for + # instance "/help GETINFO version") + + arg = arg.split(" ")[0] + # strip slash if someone enters an interpretor command (ex. "/help /help") if arg.startswith("/"): arg = arg[1:] if arg: - if arg in INTERPRETOR_HELP: - # Provides information for the interpretor argument. This bolds the - # usage information and indents the description after it. - usage, description = INTERPRETOR_HELP[arg] + if arg in HELP_OPTIONS: + # Provides information for the tor or interpretor argument. This bolds + # the usage information and indents the description after it. + usage, description = HELP_OPTIONS[arg] outputEntry.append((usage + "\n", OUTPUT_FORMAT + (Attr.BOLD, ))) for line in description.split("\n"): outputEntry.append((" " + line + "\n", OUTPUT_FORMAT)) + + # if this is the GETINFO option then also list the valid options + if arg == "GETINFO": + infoOptions = torTools.getConn().getInfo("info/names") + + if infoOptions: + for line in infoOptions.split("\n"): + if line.startswith("config/*") or line.startswith("dir-usage"): + continue + + lineMatch = re.match("^(.+) -- (.+)$", line) + + if lineMatch: + opt, description = lineMatch.groups() + + outputEntry.append(("%-33s" % opt, OUTPUT_FORMAT + (Attr.BOLD, ))) + outputEntry.append((" - %s\n" % description, OUTPUT_FORMAT)) else: # check if this is a configuration option manEntry = torConfig.getConfigDescription(arg) @@ -336,7 +364,7 @@ class ControlInterpretor: drawPortion, line = uiTools.cropStr(line, 88, 4, 4, uiTools.Ending.HYPHEN, True) outputEntry.append((" %s\n" % drawPortion.strip(), OUTPUT_FORMAT)) else: - outputEntry.append(("No help information for '%s'..." % arg, ERROR_FORMAT)) + outputEntry.append(("No help information available for '%s'..." % arg, ERROR_FORMAT)) else: # provides the GENERAL_HELP with everything bolded except descriptions for line in GENERAL_HELP.split("\n"):

1 0

[arm/master] Help content for GETCONF
by atagar＠torproject.org 15 Sep '11

15 Sep '11

commit 73ec825798a13cc803792c0115619c422f939bf5 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Sep 15 09:58:59 2011 -0700 Help content for GETCONF Providing a simple description followed by all of the tor options. --- src/util/torInterpretor.py | 26 ++++++++++++++++++++++++-- 1 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/util/torInterpretor.py b/src/util/torInterpretor.py index 21603a1..71080c9 100644 --- a/src/util/torInterpretor.py +++ b/src/util/torInterpretor.py @@ -101,13 +101,17 @@ HELP_QUIT = """Terminates the interpretor.""" HELP_GETINFO = """Queries the tor process for information. Options are... """ +HELP_GETCONF = """Provides the current value for a given configuration value. Options include... +""" + HELP_OPTIONS = { "HELP": ("/help [OPTION]", HELP_HELP), "WRITE": ("/write [PATH]", HELP_WRITE), "INFO": ("/info [relay fingerprint, nickname, or IP address]", HELP_INFO), "FIND": ("/find PATTERN", HELP_FIND), "QUIT": ("/quit", HELP_QUIT), - "GETINFO": ("GETINFO [OPTION]", HELP_GETINFO) + "GETINFO": ("GETINFO [OPTION]", HELP_GETINFO), + "GETCONF": ("GETCONF [OPTION]", HELP_GETCONF) } class InterpretorClosed(Exception): @@ -327,8 +331,8 @@ class ControlInterpretor: for line in description.split("\n"): outputEntry.append((" " + line + "\n", OUTPUT_FORMAT)) - # if this is the GETINFO option then also list the valid options if arg == "GETINFO": + # if this is the GETINFO option then also list the valid options infoOptions = torTools.getConn().getInfo("info/names") if infoOptions: @@ -343,6 +347,24 @@ class ControlInterpretor: outputEntry.append(("%-33s" % opt, OUTPUT_FORMAT + (Attr.BOLD, ))) outputEntry.append((" - %s\n" % description, OUTPUT_FORMAT)) + elif arg == "GETCONF": + # lists all of the configuration options + + confOptions = torTools.getConn().getInfo("config/names") + if confOptions: + confEntries = [opt.split(" ", 1)[0] for opt in confOptions.split("\n")] + + # displays four columns of 30 characters + for i in range(0, len(confEntries), 2): + lineEntries = confEntries[i : i+2] + + lineContent = "" + for entry in lineEntries: + lineContent += "%-42s" % entry + + outputEntry.append((lineContent + "\n", OUTPUT_FORMAT)) + + outputEntry.append(("For more information use '/help [CONFIG OPTION]'.", OUTPUT_FORMAT + (Attr.BOLD, ))) else: # check if this is a configuration option manEntry = torConfig.getConfigDescription(arg)

1 0

r25077: {website} update the mirrors table. (website/trunk/include)
by Andrew Lewman 15 Sep '11

15 Sep '11

Author: phobos Date: 2011-09-15 16:58:04 +0000 (Thu, 15 Sep 2011) New Revision: 25077 Modified: website/trunk/include/mirrors-table.wmi Log: update the mirrors table. Modified: website/trunk/include/mirrors-table.wmi =================================================================== --- website/trunk/include/mirrors-table.wmi 2011-09-15 16:56:37 UTC (rev 25076) +++ website/trunk/include/mirrors-table.wmi 2011-09-15 16:58:04 UTC (rev 25077) @@ -18,23 +18,6 @@ <tr> - <td>DK</td> - - <td>Zentrum der Gesundheit</td> - - <td>Up to date</td> - - <td> - </td> - <td><a href="http://tor.idnr.ws/dist/">http</a></td> - <td><a href="http://tor.idnr.ws/">http</a></td> - <td> - </td> - <td> - </td> - <td> - </td> - <td> - </td> -</tr> - -<tr> - <td>EE</td> <td>CyberSIDE</td> @@ -137,6 +120,23 @@ <tr> + <td>DE</td> + + <td>CCC-Hanau</td> + + <td>Up to date</td> + + <td><a href="ftp://key-server.org/tor">ftp</a></td> + <td><a href="http://tor.key-server.org/dist">http</a></td> + <td><a href="http://tor.key-server.org">http</a></td> + <td><a href="https://tor.key-server.org/dist">https</a></td> + <td><a href="https://tor.key-server.org">https</a></td> + <td> - </td> + <td> - </td> +</tr> + +<tr> + <td>AT</td> <td>cyberarmy</td> @@ -258,7 +258,7 @@ <td>IL</td> - <td></td> + <td>Host4site</td> <td>Up to date</td> @@ -273,15 +273,15 @@ <tr> - <td>FR</td> + <td>US</td> - <td>LazyTiger</td> + <td>AskApache</td> <td>Up to date</td> <td> - </td> - <td><a href="http://tor.taiga-san.net/dist">http</a></td> - <td><a href="http://tor.taiga-san.net/">http</a></td> + <td><a href="http://tor.askapache.com/dist/">http</a></td> + <td><a href="http://tor.askapache.com/">http</a></td> <td> - </td> <td> - </td> <td> - </td> @@ -290,15 +290,15 @@ <tr> - <td>IS</td> + <td>FR</td> - <td>TheOnionRouter</td> + <td>LazyTiger</td> <td>Up to date</td> <td> - </td> - <td><a href="http://theonionrouter.com/dist/">http</a></td> - <td><a href="http://theonionrouter.com/">http</a></td> + <td><a href="http://tor.taiga-san.net/dist">http</a></td> + <td><a href="http://tor.taiga-san.net/">http</a></td> <td> - </td> <td> - </td> <td> - </td> @@ -307,15 +307,15 @@ <tr> - <td>INT</td> + <td>IS</td> - <td>CoralCDN</td> + <td>TheOnionRouter</td> <td>Up to date</td> <td> - </td> - <td><a href="http://www.torproject.org.nyud.net/dist/">http</a></td> - <td><a href="http://www.torproject.org.nyud.net/">http</a></td> + <td><a href="http://theonionrouter.com/dist/">http</a></td> + <td><a href="http://theonionrouter.com/">http</a></td> <td> - </td> <td> - </td> <td> - </td> @@ -324,23 +324,6 @@ <tr> - <td>US</td> - - <td>AskApache</td> - - <td>Up to date</td> - - <td> - </td> - <td><a href="http://tor.askapache.com/dist/">http</a></td> - <td><a href="http://tor.askapache.com/">http</a></td> - <td> - </td> - <td> - </td> - <td> - </td> - <td> - </td> -</tr> - -<tr> - <td>DE</td> <td>[[:bbs:]]</td> @@ -392,19 +375,19 @@ <tr> - <td>DE</td> + <td>DK</td> - <td></td> + <td>Zentrum der Gesundheit</td> <td>Up to date</td> <td> - </td> - <td><a href="http://tor.key-server.org/dist">http</a></td> - <td><a href="http://tor.key-server.org">http</a></td> - <td><a href="https://tor.key-server.org/dist">https</a></td> - <td><a href="https://tor.key-server.org">https</a></td> + <td><a href="http://tor.idnr.ws/dist/">http</a></td> + <td><a href="http://tor.idnr.ws/">http</a></td> <td> - </td> <td> - </td> + <td> - </td> + <td> - </td> </tr> <tr> @@ -413,7 +396,7 @@ <td></td> - <td>Up to date</td> + <td>Out of date</td> <td> - </td> <td><a href="http://tor.freie-re.de/dist">http</a></td> @@ -430,7 +413,7 @@ <td></td> - <td>Up to date</td> + <td>Out of date</td> <td> - </td> <td><a href="http://tor.vesta.nu/dist">http</a></td> @@ -447,7 +430,7 @@ <td>India Tor Fans</td> - <td>Up to date</td> + <td>Out of date</td> <td> - </td> <td><a href="http://www.torproject.org.in/dist/">http</a></td> @@ -515,7 +498,7 @@ <td></td> - <td>Up to date</td> + <td>Out of date</td> <td> - </td> <td> - </td> @@ -528,6 +511,23 @@ <tr> + <td>INT</td> + + <td>CoralCDN</td> + + <td>Unknown</td> + + <td> - </td> + <td><a href="http://www.torproject.org.nyud.net/dist/">http</a></td> + <td><a href="http://www.torproject.org.nyud.net/">http</a></td> + <td> - </td> + <td> - </td> + <td> - </td> + <td> - </td> +</tr> + +<tr> + <td>IS</td> <td>crypto.is</td>

1 0

r25076: {website} add an ftp mirror, update an org. (website/trunk)
by Andrew Lewman 15 Sep '11

15 Sep '11

Author: phobos Date: 2011-09-15 16:56:37 +0000 (Thu, 15 Sep 2011) New Revision: 25076 Modified: website/trunk/update-mirrors.pl Log: add an ftp mirror, update an org. Modified: website/trunk/update-mirrors.pl =================================================================== --- website/trunk/update-mirrors.pl 2011-09-15 00:59:43 UTC (rev 25075) +++ website/trunk/update-mirrors.pl 2011-09-15 16:56:37 UTC (rev 25076) @@ -627,7 +627,7 @@ }, mirror039 => { adminContact => "", - orgName => "", + orgName => "CCC-Hanau", isoCC => "DE", subRegion => "", region => "DE", @@ -637,7 +637,7 @@ httpWebsiteMirror => "http://tor.key-server.org", httpsWebsiteMirror => "https://tor.key-server.org", rsyncWebsiteMirror => "", - ftpWebsiteMirror => "", + ftpWebsiteMirror => "ftp://key-server.org/tor", httpDistMirror => "http://tor.key-server.org/dist", httpsDistMirror => "https://tor.key-server.org/dist", rsyncDistMirror => "",

1 0

[tor/master] Log errno on listener socket creation failure.
by nickm＠torproject.org 15 Sep '11

15 Sep '11

commit 2394c7401769c56226bd95b70f9b85928a3017de Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Sep 15 09:51:48 2011 -0400 Log errno on listener socket creation failure. This may help diagnose bug 4027. --- changes/4027-diagnose | 4 ++++ src/or/connection.c | 3 ++- 2 files changed, 6 insertions(+), 1 deletions(-) diff --git a/changes/4027-diagnose b/changes/4027-diagnose new file mode 100644 index 0000000..3fb6bd0 --- /dev/null +++ b/changes/4027-diagnose @@ -0,0 +1,4 @@ + o Minor features (diagnostics): + - When the system call to create a listener socket fails, log the + error message explaining why. This may help diagnose bug 4027. + diff --git a/src/or/connection.c b/src/or/connection.c index 790383b..0092030 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -891,7 +891,8 @@ connection_create_listener(const struct sockaddr *listensockaddr, is_tcp ? SOCK_STREAM : SOCK_DGRAM, is_tcp ? IPPROTO_TCP: IPPROTO_UDP); if (!SOCKET_OK(s)) { - log_warn(LD_NET,"Socket creation failed."); + log_warn(LD_NET,"Socket creation failed: %s", + tor_socket_strerror(tor_socket_errno(-1))); goto err; }

1 0

[torflow/master] Dust off and polish up README a bit.
by mikeperry＠torproject.org 15 Sep '11

15 Sep '11

commit adebcbf8a4dda26217a8c9cb82bcb8594857090c Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Thu Sep 15 01:01:01 2011 -0700 Dust off and polish up README a bit. --- NetworkScanners/ExitAuthority/README.ExitScanning | 34 ++++++++++++-------- 1 files changed, 20 insertions(+), 14 deletions(-) diff --git a/NetworkScanners/ExitAuthority/README.ExitScanning b/NetworkScanners/ExitAuthority/README.ExitScanning index 01b7676..8eca6ca 100644 --- a/NetworkScanners/ExitAuthority/README.ExitScanning +++ b/NetworkScanners/ExitAuthority/README.ExitScanning @@ -15,10 +15,8 @@ document. This document concerns itself only with running the scanner. II. Prerequisites Python 2.5+ -Tor 0.2.1.13 (r18556 or later) +Tor 0.2.2.x py-openssl/pyOpenSSL -sqlalchemy 5.x -Elixir 6.x Bonus: Secondary external IP address Having a second external IP address will allow your scanner to filter @@ -31,14 +29,16 @@ III. Setup A. Compiling Tor -To run SoaT you will need Tor 0.2.1.x or later. +To run SoaT you will need Tor 0.2.2.x. It is also strongly recommended that you have a custom Tor instance that is devoted only to exit scanning, and is not performing any other function (including serving as a relay or a directory authority). -B. Configuring SoaT +B. Configuring SoaT for Randomized Testing + +If you just want to run a simple static test, skip this section. To configure SoaT you will need to edit soat_config.py. @@ -91,6 +91,10 @@ positive rates. # ./soat.py --ssl --http >& ./data/soat.log & +or + +# ./soat.py --ssl --target=ip:port >& ./data/soat.log & + V. Tests and Operating Modes @@ -103,13 +107,15 @@ enable a test, simply pass SoaT its flag: --ssl, --http, --html, or By default the tests are run in search based mode, this means that the URLs to be requested during the run are gathered by querying search engines for -the terms in your ./wordlist.txt file. An alternative, and potentially less -false positive prone, operating mode is the fixed target mode. Fixed target -mode is enabled by passing SoaT one or more --target=<URL> flags. Only the -URLs referenced by the target flags will be requested. This operating mode -has several attractive features, for instance, you can reduce false positive -rates by selecting static content, and you can shorten the duration of runs -by selecting small files on highly responsive servers. +the terms in your ./wordlist.txt file. + +An alternative, and potentially less false positive prone, operating mode is +the fixed target mode. Fixed target mode is enabled by passing SoaT one or +more --target=<URL> flags. Only the URLs referenced by the target flags will +be requested. This operating mode has several attractive features, for +instance, you can reduce false positive rates by selecting static content, and +you can shorten the duration of runs by selecting small files on highly +responsive servers. It should be noted that, despite their attractive features, fixed target scans are likely to miss many of the results which search based scans @@ -125,7 +131,7 @@ make a request through such an exit which triggers its malicious behavior. VI. Monitoring and Results -A. Issues with automated search engine queries +A. Issues with automated search engine queries and Randomized Scans SoaT can use Ixquick, Google, or Yahoo to perform its search queries. The current default is Ixquick, and for most purposes this should be fine. If @@ -165,7 +171,7 @@ soat.log. If/When SoaT crashes, you should be able to resume it exactly where it left off with: -# ./soat.py --resume=-1 --ssl --html --http --dnsrebind >& soat.log & +# ./soat.py --resume=-1 [other options you used last time] >& soat.log & Keeping the same options during a --resume is a Really Good Idea.

1 0

[torflow/master] Allow SSL tests to have a port.
by mikeperry＠torproject.org 15 Sep '11

15 Sep '11

commit 3b2a461cd8a3095a6b53f44101444ba51276397a Author: Mike Perry <mikeperry-git(a)fscked.org> Date: Thu Sep 15 00:27:54 2011 -0700 Allow SSL tests to have a port. This should fix bug 2697 by allowing us to scan for failures on port 80 SSL hosts. Also use the new TorCtl.connect() method. --- NetworkScanners/ExitAuthority/soat.py | 48 ++++++++++++++++++++++----------- 1 files changed, 32 insertions(+), 16 deletions(-) diff --git a/NetworkScanners/ExitAuthority/soat.py b/NetworkScanners/ExitAuthority/soat.py index 5bcf590..7688be1 100755 --- a/NetworkScanners/ExitAuthority/soat.py +++ b/NetworkScanners/ExitAuthority/soat.py @@ -453,6 +453,14 @@ def _ssl_request(address, method='TLSv1_METHOD'): ''' initiate an ssl connection and return the server certificate ''' address=str(address) # Unicode hostnames not supported.. + if address.rfind(":") != -1: + # FIXME: %-encoding? + port = int(address[address.rfind(":")+1:]) + address_name = address[:address.rfind(":")] + else: + port = 443 + address_name = address + # specify the context ctx = SSL.Context(getattr(SSL,method)) @@ -468,7 +476,7 @@ def _ssl_request(address, method='TLSv1_METHOD'): c = SSL.Connection(ctx, s) c.set_connect_state() signal.alarm(int(read_timeout)) # raise a timeout after read_timeout - c.connect((address, 443)) # DNS OK. + c.connect((address_name, port)) # DNS OK. # XXX: A PEM encoded certificate request was a bizarre and fingerprintable # thing to send here. All we actually need to do is perform a handshake, # but it might be good to make a simple GET request to further limit @@ -643,14 +651,15 @@ class Test: if netloc.rfind(":") != -1: # FIXME: %-encoding? port = netloc[netloc.rfind(":")+1:] - try: - if int(port) != self.port: - plog("DEBUG", "Unusable port "+port+" in "+url) - return False - except: - traceback.print_exc() - plog("WARN", "Unparseable port "+port+" in "+url) - return False + # I think port restrictions are silly, come to think of it.. + #try: + # if int(port) != self.port: + # plog("DEBUG", "Unusable port "+port+" in "+url) + # return False + #except: + # traceback.print_exc() + # plog("WARN", "Unparseable port "+port+" in "+url) + # return False if valid_schemes and scheme not in valid_schemes: plog("DEBUG", "Unusable scheme "+scheme+" in "+url) return False @@ -1515,6 +1524,14 @@ class BaseSSLTest(Test): address_file = DataHandler.safeFilename(shortaddr) ssl_file_name = ssl_certs_dir + address_file + '.ssl' + if address.rfind(":") != -1: + # FIXME: %-encoding? + port = address[address.rfind(":")+1:] + address_name = address[:address.rfind(":")] + else: + port = 443 + address_name = address + # load the original cert and compare # if we don't have the original cert yet, get it try: @@ -1527,14 +1544,14 @@ class BaseSSLTest(Test): # Make 3 resolution attempts for attempt in xrange(1,4): try: - resolved = socket.getaddrinfo(address, 443, socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) + resolved = socket.getaddrinfo(address_name, 443, socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) break except socket.gaierror: plog("NOTICE", "Local resolution failure #%d for %s" % (attempt, address)) for res in resolved: if res[4][0] not in check_ips: - check_ips.append(res[4][0]) + check_ips.append(res[4][0]+":"+str(port)) if not check_ips: plog("WARN", "Local resolution failure for "+address) @@ -1697,7 +1714,7 @@ class FixedTargetSSLTest(FixedTargetTest, BaseSSLTest): def __init__(self, targets): BaseSSLTest.__init__(self) # We ask for hostnames only, please - utargets = [t for t in targets if self._is_useable_url(t, [''])] + utargets = [t for t in targets if self._is_useable_url(t)] FixedTargetTest.__init__(self, utargets) # Search Based Tests @@ -2579,11 +2596,10 @@ def cleanup(c, l, f): def setup_handler(out_dir, cookie_file, fixed_exits=[]): plog('INFO', 'Connecting to Tor at '+TorUtil.control_host+":"+str(TorUtil.control_port)) - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.connect((TorUtil.control_host,TorUtil.control_port)) - c = PathSupport.Connection(s) + + c = TorCtl.connect(TorUtil.control_host, TorUtil.control_port, ConnClass=PathSupport.Connection) c.debug(file(out_dir+"/control.log", "w", buffering=0)) - c.authenticate_cookie(file(cookie_file, "r")) + l = c.get_option("__LeaveStreamsUnattached")[0][1] h = ExitScanHandler(c, __selmgr, PathSupport.SmartSocket.StreamSelector, fixed_exits)

1 0