September 2011 - tor-commits - lists.torproject.org

[arm/master] Adding back the color prompt, with padded prompt
by atagar＠torproject.org 13 Sep '11

13 Sep '11

commit 0fb7dd674c9969d55c21fca9040f7f7bcfd09950 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Sep 13 08:58:29 2011 -0700 Adding back the color prompt, with padded prompt This is a hack to maintain a color prompt while sidestepping the 'http://bugs.python.org/issue12972' bug in *most* use cases. This pads the prompt so the original issue doesn't manifest unless the user's input exceeds around sixty characters or so. However, it introduces a new bug where tiny terminals (smaller than the padded width) get screwed up. This obviously isn't a great situation and it could be argued that a colored prompt isn't worth these issues. However, the problem use cases strike me as being fairly rare (and I much prefer the green and bolded '>>>') so going with this for now. Patches welcome if someone has a better solution. --- src/util/torInterpretor.py | 13 +++++++++++-- 1 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/util/torInterpretor.py b/src/util/torInterpretor.py index fc217f3..2c4e7fa 100644 --- a/src/util/torInterpretor.py +++ b/src/util/torInterpretor.py @@ -495,9 +495,18 @@ def prompt(): # Cycling history via the readline module with up/down is buggy with a color # prompt. For more information see: # http://bugs.python.org/issue12972 - #prompt = format(">>> ", Color.GREEN, Attr.BOLD) + # + # To work around this while keeping a color prompt I'm padding the prompt + # with extra reset encodings so its length is non-rendered higher (around + # sixty characters). There's two ways that this can go wrong... + # - if the user uses up/down to display input longer than this non-rendered + # length then the original bug will manifest (screwed up prompt) + # - if the terminal's width is smaller than the non-rendered prompt length + # then the cursor and some movement will be displaced + + prompt = format(">>> ", Color.GREEN, Attr.BOLD) + prompt += "\x1b[0m" * 10 - prompt = ">>> " input = "" # sets up tab autocompetion

1 0

[arm/master] Source flags for color prompt and hostname lookups
by atagar＠torproject.org 13 Sep '11

13 Sep '11

commit 6e88a85a7a8c45d798e8471e9ce27295aa0b122e Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Sep 13 09:34:30 2011 -0700 Source flags for color prompt and hostname lookups Adding source flags for portions of the interpretor that include substantial tradeoffs. For color prompts there's bugs in small displays/huge input, and hostname lookups add a second or two to the runtime of /info queries (otherwise the lookup is instantanious). Defaulting the hostname lookups to be disabled for now. There's plans for having cached results with... https://trac.torproject.org/projects/tor/wiki/doc/arm#CircuitDetails though I haven't a clue if it'll ever be implemented. --- src/util/torInterpretor.py | 19 +++++++++++++++---- 1 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/util/torInterpretor.py b/src/util/torInterpretor.py index 2c4e7fa..fbb33da 100644 --- a/src/util/torInterpretor.py +++ b/src/util/torInterpretor.py @@ -11,6 +11,9 @@ import version from util import connections, enum, hostnames, torTools +COLOR_PROMPT = True # provides a colored interpretor prompt +INFO_HOSTNAMES = False # hostname lookups in /info results + INIT_MSG = """Arm %s Control Interpretor Enter \"/help\" for usage information and \"/quit\" to stop. """ % version.VERSION @@ -325,7 +328,11 @@ class ControlInterpretor: if not address or not port: return locale = conn.getInfo("ip-to-country/%s" % address, "??") - hostname = hostnames.resolve(address, 10) + + if INFO_HOSTNAMES: + hostname = hostnames.resolve(address, 10) + else: + hostname = None # TODO: Most of the following is copied from the _getDetailContent method # of cli/connections/connEntry.py - useful bits should be refactored. @@ -370,8 +377,9 @@ class ControlInterpretor: outputEntry.append(("%s (%s)\n" % (nickname, fingerprint), infoAttr)) + hostnameLabel = ", %s" % hostname if hostname else "" outputEntry.append(("address: ", headingAttr)) - outputEntry.append(("%s:%s (%s, %s)\n" % (address, port, locale, hostname), infoAttr)) + outputEntry.append(("%s:%s (%s%s)\n" % (address, port, locale, hostnameLabel), infoAttr)) outputEntry.append(("published: ", headingAttr)) outputEntry.append(("%s %s" % (pubTime, pubDate) + "\n", infoAttr)) @@ -504,8 +512,11 @@ def prompt(): # - if the terminal's width is smaller than the non-rendered prompt length # then the cursor and some movement will be displaced - prompt = format(">>> ", Color.GREEN, Attr.BOLD) - prompt += "\x1b[0m" * 10 + if COLOR_PROMPT: + prompt = format(">>> ", Color.GREEN, Attr.BOLD) + prompt += "\x1b[0m" * 10 + else: + prompt = ">>> " input = ""

1 0

[metrics-web/master] Add directory listing for /csv.
by karsten＠torproject.org 13 Sep '11

13 Sep '11

commit 4f6c4bc456fd2f35588722df161549f903eeafca Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Sep 13 17:29:58 2011 +0200 Add directory listing for /csv. See https://metrics.torproject.org/csv/ . --- src/org/torproject/ernie/web/CsvServlet.java | 16 +++++++++++++--- web/WEB-INF/dir.jsp | 25 +++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/src/org/torproject/ernie/web/CsvServlet.java b/src/org/torproject/ernie/web/CsvServlet.java index 99dc63c..01cdb28 100644 --- a/src/org/torproject/ernie/web/CsvServlet.java +++ b/src/org/torproject/ernie/web/CsvServlet.java @@ -18,7 +18,7 @@ public class CsvServlet extends HttpServlet { private RObjectGenerator rObjectGenerator; /* Available CSV files. */ - private Set<String> availableCsvFiles; + private SortedSet<String> availableCsvFiles; private Logger logger; @@ -28,7 +28,7 @@ public class CsvServlet extends HttpServlet { this.logger = Logger.getLogger(CsvServlet.class.toString()); /* Initialize map of available CSV files. */ - this.availableCsvFiles = new HashSet<String>(); + this.availableCsvFiles = new TreeSet<String>(); this.availableCsvFiles.add("bandwidth"); this.availableCsvFiles.add("bridge-users"); this.availableCsvFiles.add("bwhist-flags"); @@ -60,9 +60,19 @@ public class CsvServlet extends HttpServlet { HttpServletResponse response) throws IOException, ServletException { + /* Check if the directory listing was requested. */ + String requestURI = request.getRequestURI(); + if (requestURI.equals("/ernie/csv/")) { + request.setAttribute("directory", "/csv"); + request.setAttribute("extension", ".csv"); + request.setAttribute("files", this.availableCsvFiles); + request.getRequestDispatcher("/WEB-INF/dir.jsp").forward(request, + response); + return; + } + /* Find out which CSV file was requested and make sure we know this * CSV file type. */ - String requestURI = request.getRequestURI(); String requestedCsvFile = requestURI; if (requestedCsvFile.endsWith(".csv")) { requestedCsvFile = requestedCsvFile.substring(0, diff --git a/web/WEB-INF/dir.jsp b/web/WEB-INF/dir.jsp new file mode 100644 index 0000000..1ac1819 --- /dev/null +++ b/web/WEB-INF/dir.jsp @@ -0,0 +1,25 @@ +<%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core" %> +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> +<html> +<head> + <title>Tor Metrics Portal: Index of ${directory}</title> + <meta http-equiv="content-type" content="text/html; charset=ISO-8859-1"> + <link href="/css/stylesheet-ltr.css" type="text/css" rel="stylesheet"> + <link href="/images/favicon.ico" type="image/x-icon" rel="shortcut icon"> +</head> +<body> + <div class="center"> + <%@ include file="banner.jsp"%> + <div class="main-column"> + <h2>Tor Metrics Portal: Index of ${directory}</h2> + <br> + <c:forEach var="file" items="${files}" > + <a href="${directory}/${file}${extension}">${file}${extension}</a><br> + </c:forEach> + </div> + </div> + <div class="bottom" id="bottom"> + <%@ include file="footer.jsp"%> + </div> +</body> +</html>

1 0

[metrics-web/master] Fix an IllegalStateException.
by karsten＠torproject.org 13 Sep '11

13 Sep '11

commit dac09a8ccc2a6a4166c40ccfff89751afc0e7585 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Sep 13 16:36:08 2011 +0200 Fix an IllegalStateException. We're not allowed to forward a request after we started responding to it, here by sending an error code. --- .../torproject/ernie/web/RouterDetailServlet.java | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diff --git a/src/org/torproject/ernie/web/RouterDetailServlet.java b/src/org/torproject/ernie/web/RouterDetailServlet.java index 4a25352..c8c4648 100644 --- a/src/org/torproject/ernie/web/RouterDetailServlet.java +++ b/src/org/torproject/ernie/web/RouterDetailServlet.java @@ -112,6 +112,10 @@ public class RouterDetailServlet extends HttpServlet { /* There were zero results in the set */ /* TODO Handle this case in a more user-friendly way. */ response.sendError(HttpServletResponse.SC_BAD_REQUEST); + this.logger.log(Level.WARNING, "Query with fingerprint = '" + + fingerprintParameter + "' returned zero results. " + + "Returned BAD_REQUEST."); + return; } conn.close();

1 0

[metrics-web/master] Cut off last days in the bandwith graph.
by karsten＠torproject.org 13 Sep '11

13 Sep '11

commit 46009133c21cb7da39ef113d6a3bcdda984b66d9 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Sep 13 16:10:30 2011 +0200 Cut off last days in the bandwith graph. --- rserve/graphs.R | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diff --git a/rserve/graphs.R b/rserve/graphs.R index 3c6d72a..7f417a0 100644 --- a/rserve/graphs.R +++ b/rserve/graphs.R @@ -441,7 +441,8 @@ plot_bandwidth <- function(start, end, path, dpi) { drv <- dbDriver("PostgreSQL") con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) q <- paste("SELECT date, bwadvertised FROM total_bandwidth ", - "WHERE date >= '", start, "' AND date <= '", end, "'", sep = "") + "WHERE date >= '", start, "' AND date <= '", end, "' ", + "AND date < (SELECT MAX(date) FROM total_bandwidth) - 1 ", sep = "") rs <- dbSendQuery(con, q) bw_desc <- fetch(rs, n = -1) q <- paste("SELECT date, read, written FROM total_bwhist ",

1 0

[metrics-web/master] Graph GetTor packages by language, not by bundle.
by karsten＠torproject.org 13 Sep '11

13 Sep '11

commit 32bd955ffce9c5ca514fb1cc9a997999d31fed58 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Sep 13 10:33:05 2011 +0200 Graph GetTor packages by language, not by bundle. With the recent renaming of GetTor package names it has become more difficult to decide which packages are a TBB and which are something else. tor-browser-bundle_en is now one out of linux-i386_en, linux-x86_64_en, macosx-i386_en, or windows_en. And in the future there may be a TIMBB and who knows what other packages. Time to give up the guessing game. The language code is something we can identify for certain. Let's graph that. --- rserve/graphs.R | 24 ++++++++++++---- .../ernie/web/GraphParameterChecker.java | 28 ++++++++++---------- web/WEB-INF/packages.jsp | 11 ++++--- 3 files changed, 38 insertions(+), 25 deletions(-) diff --git a/rserve/graphs.R b/rserve/graphs.R index 959fd64..3c6d72a 100644 --- a/rserve/graphs.R +++ b/rserve/graphs.R @@ -254,6 +254,18 @@ countryname <- function(country) { res } +languagelist <- list( + "en" = "English", + "fa" = "Farsi", + "zh_CN" = "Simplified Chinese") + +languagename <- function(language) { + res <- languagelist[[language]] + if (is.null(res)) + res <- "Gibberish" + res +} + date_breaks <- function(days) { length <- cut(days, c(0, 7, 12, 56, 180, 600, 5000, Inf), labels=FALSE) major <- c("days", "2 days", "weeks", "months", "3 months", "years", @@ -704,11 +716,11 @@ plot_bridge_users <- function(start, end, country, path, dpi) { ggsave(filename = path, width = 8, height = 5, dpi = as.numeric(dpi)) } -plot_gettor <- function(start, end, bundle, path, dpi) { +plot_gettor <- function(start, end, language, path, dpi) { drv <- dbDriver("PostgreSQL") con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db) - condition <- ifelse(bundle == "all", "<> 'none'", - paste("LIKE 'tor-%browser-bundle_", tolower(bundle), "'", sep = "")) + condition <- ifelse(language == "all", "<> 'none'", + paste("LIKE '%_", tolower(language), "'", sep = "")) q <- paste("SELECT date, SUM(downloads) AS downloads ", "FROM gettor_stats WHERE bundle ", condition, " AND date >= '", start, "' AND date <= '", end, @@ -725,10 +737,10 @@ plot_gettor <- function(start, end, bundle, path, dpi) { downloads <- rbind(downloads, data.frame(date = as.Date(missing, origin = "1970-01-01"), downloads = NA)) - title <- ifelse(bundle == "all", + title <- ifelse(language == "all", "Total packages requested from GetTor per day\n", - paste("Tor Browser Bundles (", bundle, - ") requested from GetTor per day\n", sep = "")) + paste(languagename(language), " (", language, + ") packages requested from GetTor per day\n", sep = "")) date_breaks <- date_breaks( as.numeric(max(as.Date(downloads$date, "%Y-%m-%d")) - min(as.Date(downloads$date, "%Y-%m-%d")))) diff --git a/src/org/torproject/ernie/web/GraphParameterChecker.java b/src/org/torproject/ernie/web/GraphParameterChecker.java index 7349bf9..8189f19 100644 --- a/src/org/torproject/ernie/web/GraphParameterChecker.java +++ b/src/org/torproject/ernie/web/GraphParameterChecker.java @@ -53,7 +53,7 @@ public class GraphParameterChecker { "start,end,country,events,filename,dpi"); this.availableGraphs.put("bridge-users", "start,end,country,filename,dpi"); - this.availableGraphs.put("gettor", "start,end,bundle,filename,dpi"); + this.availableGraphs.put("gettor", "start,end,language,filename,dpi"); this.availableGraphs.put("torperf", "start,end,source,filesize,filename,dpi"); this.availableGraphs.put("torperf-failures", @@ -71,7 +71,7 @@ public class GraphParameterChecker { } this.knownParameterValues.put("country", sb.toString()); this.knownParameterValues.put("events", "on,off"); - this.knownParameterValues.put("bundle", "all,en,zh_CN,fa"); + this.knownParameterValues.put("language", "all,en,zh_CN,fa"); this.knownParameterValues.put("source", "all,siv,moria,torperf"); this.knownParameterValues.put("filesize", "50kb,1mb,5mb"); this.knownParameterValues.put("dpi", "72,150,300"); @@ -218,24 +218,24 @@ public class GraphParameterChecker { recognizedGraphParameters.put("events", eventsParameter); } - /* Parse GetTor bundle if supported by the graph type. Only a single - * bundle can be passed. If no bundle is passed, use "all" as + /* Parse language if supported by the graph type. Only a single + * language can be passed. If no language is passed, use "all" as * default. */ - if (supportedGraphParameters.contains("bundle")) { - String[] bundleParameter = (String[]) requestParameters.get( - "bundle"); + if (supportedGraphParameters.contains("language")) { + String[] languageParameter = (String[]) requestParameters.get( + "language"); List<String> knownBundles = Arrays.asList( - this.knownParameterValues.get("bundle").split(",")); - if (bundleParameter != null) { - if (bundleParameter.length != 1 || - bundleParameter[0].length() == 0 || - !knownBundles.contains(bundleParameter[0])) { + this.knownParameterValues.get("language").split(",")); + if (languageParameter != null) { + if (languageParameter.length != 1 || + languageParameter[0].length() == 0 || + !knownBundles.contains(languageParameter[0])) { return null; } } else { - bundleParameter = new String[] { "all" }; + languageParameter = new String[] { "all" }; } - recognizedGraphParameters.put("bundle", bundleParameter); + recognizedGraphParameters.put("language", languageParameter); } /* Parse torperf data source if supported by the graph type. Only a diff --git a/web/WEB-INF/packages.jsp b/web/WEB-INF/packages.jsp index 9a19aca..9bfee70 100644 --- a/web/WEB-INF/packages.jsp +++ b/web/WEB-INF/packages.jsp @@ -33,11 +33,12 @@ graph shows the number of packages requested from GetTor per day.</p> <input type="text" name="end" size="10" value="<c:choose><c:when test="${fn:length(gettor_end) == 0}">${default_end_date}</c:when><c:otherwise>${gettor_end[0]}</c:otherwise></c:choose>"> </p><p> - Packages: - <input type="radio" name="bundle" value="all" <c:if test="${fn:length(gettor_bundle) == 0 or gettor_bundle[0] eq 'all'}"> checked</c:if>> Total packages - <input type="radio" name="bundle" value="en" <c:if test="${gettor_bundle[0] eq 'en'}"> checked</c:if>> TBB (en) - <input type="radio" name="bundle" value="zh_CN" <c:if test="${gettor_bundle[0] eq 'zh_CN'}"> checked</c:if>> TBB (zh_CN) - <input type="radio" name="bundle" value="fa" <c:if test="${gettor_bundle[0] eq 'fa'}"> checked</c:if>> TBB (fa) + Language: <select name="language"> + <option value="all"<c:if test="${fn:length(gettor_language) == 0 or gettor_language[0] eq 'all'}"> selected</c:if>>All languages</option> + <option value="en"<c:if test="${gettor_language[0] eq 'en'}"> selected</c:if>>English (en)</option> + <option value="zh_CN"<c:if test="${gettor_language[0] eq 'zh_CN'}"> selected</c:if>>Simplified Chinese (zh_CN)</option> + <option value="fa"<c:if test="${gettor_language[0] eq 'fa'}"> selected</c:if>>Farsi (fa)</option> + </select> </p><p> Resolution: <select name="dpi"> <option value="72"<c:if test="${gettor_dpi[0] eq '72'}"> selected</c:if>>Screen - 576x360</option>

1 0

[arm/master] Util for getting condensed exit policies
by atagar＠torproject.org 13 Sep '11

13 Sep '11

commit 8a40c745ce5f040aef38963b2a047ba2f92ee392 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Sep 12 19:27:14 2011 -0700 Util for getting condensed exit policies Expanding the ExitPolicy class to provide condensed exit policies like the tor consensus. --- src/util/torTools.py | 92 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 92 insertions(+), 0 deletions(-) diff --git a/src/util/torTools.py b/src/util/torTools.py index 7c14d18..b9d9ac8 100644 --- a/src/util/torTools.py +++ b/src/util/torTools.py @@ -2286,6 +2286,21 @@ class Controller(TorCtl.PostEventListener): self.connLock.release() +class ExitPolicyIterator: + """ + Basic iterator for cycling through ExitPolicy entries. + """ + + def __init__(self, head): + self.head = head + + def next(self): + if self.head: + lastHead = self.head + self.head = self.head.nextRule + return lastHead + else: raise StopIteration + class ExitPolicy: """ Single rule from the user's exit policy. These are chained together to form @@ -2301,6 +2316,9 @@ class ExitPolicy: nextRule - next rule to be checked when queries don't match this policy """ + # cached summary string + self.summaryStr = None + # sanitize the input a bit, cleaning up tabs and stripping quotes ruleEntry = ruleEntry.replace("\\t", " ").replace("\"", "") @@ -2405,6 +2423,80 @@ class ExitPolicy: if self.nextRule: return self.nextRule.check(ipAddress, port) else: return True # fell off the chain without a conclusion (shouldn't happen...) + def getSummary(self): + """ + Provides a summary description of the policy chain similar to the + consensus. This excludes entries that don't cover all ips, and is either + a whitelist or blacklist policy based on the final entry. For instance... + accept 80, 443 # just accepts ports 80/443 + reject 1-1024, 5555 # just accepts non-privilaged ports, excluding 5555 + """ + + if not self.summaryStr: + # determines if we're a whitelist or blacklist + isWhitelist = False # default in case we don't have a catch-all policy at the end + + for rule in self: + if rule.isIpWildcard and rule.isPortWildcard: + isWhitelist = not rule.isAccept + break + + # Iterates over the rules and adds the the ports we'll return (ie, allows + # if a whitelist and rejects if a blacklist). Reguardless of a port's + # allow/reject policy, all further entries with that port are ignored since + # policies respect the first matching rule. + + displayPorts, skipPorts = [], [] + + for rule in self: + if not rule.isIpWildcard: continue + + if rule.minPort == rule.maxPort: + portRange = [rule.minPort] + else: + portRange = range(rule.minPort, rule.maxPort + 1) + + for port in portRange: + if port in skipPorts: continue + + # if accept + whitelist or reject + blacklist then add + if rule.isAccept == isWhitelist: + displayPorts.append(port) + + # all further entries with this port are to be ignored + skipPorts.append(port) + + # gets a list of the port ranges + if displayPorts: + displayRanges, tmpRange = [], [] + displayPorts.sort() + displayPorts.append(None) # ending item to include last range in loop + + for port in displayPorts: + if not tmpRange or tmpRange[-1] + 1 == port: + tmpRange.append(port) + else: + if len(tmpRange) > 1: + displayRanges.append("%i-%i" % (tmpRange[0], tmpRange[-1])) + else: + displayRanges.append(str(tmpRange[0])) + + tmpRange = [port] + else: + # everything for the inverse + isWhitelist = not isWhitelist + displayRanges = ["1-65535"] + + # constructs the summary string + labelPrefix = "accept " if isWhitelist else "reject " + + self.summaryStr = (labelPrefix + ", ".join(displayRanges)).strip() + + return self.summaryStr + + def __iter__(self): + return ExitPolicyIterator(self) + def __str__(self): # This provides the actual policy rather than the entry used to construct # it so the 'private' keyword is expanded.

1 0

[arm/master] Replacing displayed consensus policy with util
by atagar＠torproject.org 13 Sep '11

13 Sep '11

commit aa3e3ee7a397edb18595d86dc2652d63053b3584 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Sep 12 19:44:58 2011 -0700 Replacing displayed consensus policy with util Using the ExitPolicy class' summary rather than the string from the consensus. Unless I'm misunderstanding how the consensus policy is derived, these should be equivilant. However, since this doesn't rely on the policy being in the consensus this will be rendered the same for both new and old versions of tor. --- src/cli/connections/connEntry.py | 20 ++++---------------- 1 files changed, 4 insertions(+), 16 deletions(-) diff --git a/src/cli/connections/connEntry.py b/src/cli/connections/connEntry.py index 075dfb2..de3fb01 100644 --- a/src/cli/connections/connEntry.py +++ b/src/cli/connections/connEntry.py @@ -711,28 +711,16 @@ class ConnectionLine(entries.ConnectionPanelLine): if len(nsLines) >= 2 and nsLines[1].startswith("s "): flags = nsLines[1][2:] - # The network status exit policy doesn't exist for older tor versions. - # If unavailable we'll need the full exit policy which is on the - # descriptor (if that's available). + exitPolicy = conn.getRelayExitPolicy(fingerprint) - exitPolicy = "unknown" - if len(nsLines) >= 4 and nsLines[3].startswith("p "): - exitPolicy = nsLines[3][2:].replace(",", ", ") - elif descEntry: - # the descriptor has an individual line for each entry in the exit policy - exitPolicyEntries = [] - - for line in descEntry.split("\n"): - if line.startswith("accept") or line.startswith("reject"): - exitPolicyEntries.append(line.strip()) - - exitPolicy = ", ".join(exitPolicyEntries) + if exitPolicy: policyLabel = exitPolicy.getSummary() + else: policyLabel = "unknown" dirPortLabel = "" if dirPort == "0" else "dirport: %s" % dirPort lines[2] = "nickname: %-25s orport: %-10s %s" % (nickname, orPort, dirPortLabel) lines[3] = "published: %s %s" % (pubDate, pubTime) lines[4] = "flags: %s" % flags.replace(" ", ", ") - lines[5] = "exit policy: %s" % exitPolicy + lines[5] = "exit policy: %s" % policyLabel if descEntry: torVersion, platform, contact = "", "", ""

1 0

[arm/master] Implementing /info interpretor option
by atagar＠torproject.org 13 Sep '11

13 Sep '11

commit 6b4af377238ac7e863f4b001987d6d4786a003ad Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Sep 12 20:23:03 2011 -0700 Implementing /info interpretor option This provides a reader friendly summary of relay information. Relayed can be queried this way by fingerprint, nickname, or IP address. --- src/cli/connections/connEntry.py | 2 +- src/cli/controller.py | 3 +- src/util/torInterpretor.py | 131 +++++++++++++++++++++++++++++++++++++- 3 files changed, 133 insertions(+), 3 deletions(-) diff --git a/src/cli/connections/connEntry.py b/src/cli/connections/connEntry.py index de3fb01..02f7069 100644 --- a/src/cli/connections/connEntry.py +++ b/src/cli/connections/connEntry.py @@ -718,7 +718,7 @@ class ConnectionLine(entries.ConnectionPanelLine): dirPortLabel = "" if dirPort == "0" else "dirport: %s" % dirPort lines[2] = "nickname: %-25s orport: %-10s %s" % (nickname, orPort, dirPortLabel) - lines[3] = "published: %s %s" % (pubDate, pubTime) + lines[3] = "published: %s %s" % (pubTime, pubDate) lines[4] = "flags: %s" % flags.replace(" ", ", ") lines[5] = "exit policy: %s" % policyLabel diff --git a/src/cli/controller.py b/src/cli/controller.py index a0b465c..a841492 100644 --- a/src/cli/controller.py +++ b/src/cli/controller.py @@ -24,7 +24,7 @@ import cli.connections.connPanel from TorCtl import TorCtl -from util import connections, conf, enum, log, panel, sysTools, torConfig, torTools +from util import connections, conf, enum, hostnames, log, panel, sysTools, torConfig, torTools ARM_CONTROLLER = None @@ -568,6 +568,7 @@ def shutdownDaemons(): # joins on utility daemon threads - this might take a moment since the # internal threadpools being joined might be sleeping + hostnames.stop() resourceTrackers = sysTools.RESOURCE_TRACKERS.values() resolver = connections.getResolver("tor") if connections.isResolverAlive("tor") else None for tracker in resourceTrackers: tracker.stop() diff --git a/src/util/torInterpretor.py b/src/util/torInterpretor.py index 7eaec16..d6f9340 100644 --- a/src/util/torInterpretor.py +++ b/src/util/torInterpretor.py @@ -9,7 +9,7 @@ import readline import version -from util import enum, torTools +from util import connections, enum, hostnames, torTools INIT_MSG = """Arm %s Control Interpretor Enter \"/help\" for usage information and \"/quit\" to stop. @@ -269,6 +269,130 @@ class ControlInterpretor: outputEntry.append((lineText[match.end():] + "\n", OUTPUT_FORMAT)) printedLines.append(lineText) + def doInfo(self, arg, outputEntry): + """ + Performs the '/info' operation, looking up a relay by fingerprint, IP + address, or nickname and printing its descriptor and consensus entries in a + pretty fashion. + """ + + fingerprint, conn = None, torTools.getConn() + + # TODO: also recognize <ip>:<port> entries? + + # determines the fingerprint, leaving it unset and adding an error message + # if unsuccessful + if not arg: + outputEntry.append(("No fingerprint or nickname to look up", ERROR_FORMAT)) + elif len(arg) == 40 and re.match("^[0-9a-fA-F]+$", arg): + # we got a fingerprint (fourty character hex string) + fingerprint = arg + elif connections.isValidIpAddress(arg): + # we got an ip address, look up the fingerprint + fpMatches = conn.getRelayFingerprint(arg, getAllMatches = True) + + if len(fpMatches) == 0: + outputEntry.append(("No relays found at %s" % arg, ERROR_FORMAT)) + elif len(fpMatches) == 1: + fingerprint = fpMatches[0][1] + else: + outputEntry.append(("Multiple relays at %s, specify which by giving a port" % arg, ERROR_FORMAT)) + + for i in range(len(fpMatches)): + relayEntry = outputEntry[i] + outputEntry.append((" %i. or port: %-5s fingerprint: %s" % (i + 1, relayEntry[0], relayEntry[1]), ERROR_FORMAT)) + else: + # we got something else, treat it as a nickname + fingerprint = conn.getNicknameFingerprint(arg) + + if not fingerprint: + outputEntry.append(("No relay with the nickname of '%s' found" % arg, ERROR_FORMAT)) + + if fingerprint: + consensusEntry = conn.getConsensusEntry(fingerprint) + + # The nickname, address, and port lookups are all based on the consensus + # entry so if this succeeds we should be pretty confident that those + # queries will work too. + + if not consensusEntry: + outputEntry.append(("Unable to find consensus information for %s" % fingerprint, ERROR_FORMAT)) + return + + address, port = conn.getRelayAddress(fingerprint, (None, None)) + + # ... but not sure enough that we won't check + if not address or not port: return + + locale = conn.getInfo("ip-to-country/%s" % address, "??") + hostname = hostnames.resolve(address, 10) + + # TODO: Most of the following is copied from the _getDetailContent method + # of cli/connections/connEntry.py - useful bits should be refactored. + consensusLines = consensusEntry.split("\n") + + firstLineComp = consensusLines[0].split(" ") + if len(firstLineComp) >= 9: + _, nickname, _, _, pubDate, pubTime, _, orPort, _ = firstLineComp[:9] + else: nickname, pubDate, pubTime, orPort = "", "", "", "" + + flags = "unknown" + if len(consensusLines) >= 2 and consensusLines[1].startswith("s "): + flags = consensusLines[1][2:] + + exitPolicy = conn.getRelayExitPolicy(fingerprint) + + if exitPolicy: policyLabel = exitPolicy.getSummary() + else: policyLabel = "unknown" + + # fetches information from the descriptor if it's available + torVersion, platform, contact = "", "", "" + descriptorEntry = conn.getDescriptorEntry(fingerprint) + + if descriptorEntry: + for descLine in descriptorEntry.split("\n"): + if descLine.startswith("platform"): + # has the tor version and platform, ex: + # platform Tor 0.2.1.29 (r318f470bc5f2ad43) on Linux x86_64 + + torVersion = descLine[13:descLine.find(" ", 13)] + platform = descLine[descLine.rfind(" on ") + 4:] + elif descLine.startswith("contact"): + contact = descLine[8:] + + # clears up some highly common obscuring + for alias in (" at ", " AT "): contact = contact.replace(alias, "@") + for alias in (" dot ", " DOT "): contact = contact.replace(alias, ".") + + break # contact lines come after the platform + + headingAttr, infoAttr = (Attr.BOLD, Color.BLUE), () + + outputEntry.append(("%s (%s)\n" % (nickname, fingerprint), infoAttr)) + + outputEntry.append(("address: ", headingAttr)) + outputEntry.append(("%s:%s (%s, %s)\n" % (address, port, locale, hostname), infoAttr)) + + outputEntry.append(("published: ", headingAttr)) + outputEntry.append(("%s %s" % (pubTime, pubDate) + "\n", infoAttr)) + + if torVersion and platform: + outputEntry.append(("os: ", headingAttr)) + outputEntry.append((platform + "\n", infoAttr)) + + outputEntry.append(("version: ", headingAttr)) + outputEntry.append((torVersion + "\n", infoAttr)) + + outputEntry.append(("flags: ", headingAttr)) + outputEntry.append((flags.replace(" ", ", ") + "\n", infoAttr)) + + outputEntry.append(("exit policy: ", headingAttr)) + outputEntry.append((policyLabel + "\n", infoAttr)) + + if contact: + outputEntry.append(("contact: ", headingAttr)) + outputEntry.append((contact + "\n", infoAttr)) + def handleQuery(self, input): """ Processes the given input. Requests starting with a '/' are special @@ -308,6 +432,7 @@ class ControlInterpretor: if cmd == "/quit": raise InterpretorClosed() elif cmd == "/write": self.doWrite(arg, outputEntry) elif cmd == "/find": self.doFind(arg, outputEntry) + elif cmd == "/info": self.doInfo(arg, outputEntry) else: outputEntry.append(("Not yet implemented...", ERROR_FORMAT)) # TODO: implement @@ -393,6 +518,10 @@ def prompt(): # moves cursor to the next line and terminates (most commonly # KeyboardInterrupt and EOFErro) print + + # stop daemons + hostnames.stop() + break for line in outputEntry:

1 0

r25065: {} my talk at lookout (projects/presentations)
by Runa Sandvik 12 Sep '11

12 Sep '11

Author: runa Date: 2011-09-12 21:14:14 +0000 (Mon, 12 Sep 2011) New Revision: 25065 Added: projects/presentations/2011-tor-online-anonymity-privacy-security-Lookout.pdf projects/presentations/2011-tor-online-anonymity-privacy-security-Lookout.tex Log: my talk at lookout Added: projects/presentations/2011-tor-online-anonymity-privacy-security-Lookout.pdf =================================================================== (Binary files differ) Property changes on: projects/presentations/2011-tor-online-anonymity-privacy-security-Lookout.pdf ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: projects/presentations/2011-tor-online-anonymity-privacy-security-Lookout.tex =================================================================== --- projects/presentations/2011-tor-online-anonymity-privacy-security-Lookout.tex (rev 0) +++ projects/presentations/2011-tor-online-anonymity-privacy-security-Lookout.tex 2011-09-12 21:14:14 UTC (rev 25065) @@ -0,0 +1,328 @@ +\documentclass{beamer} +\mode<presentation> +\usetheme{Boadilla} +\title{Tor: Online anonymity, privacy, and security.} +\author{Runa A. Sandvik \\ runa(a)torproject.org} +\date{12 September 2011} +\begin{document} + +\begin{frame} +\maketitle +\begin{center} +\includegraphics[height=3cm]{../images/2009-tor-logo} +\end{center} +\end{frame} + +% Introduce myself, just to be nice +\begin{frame} +\frametitle{About Runa} +\begin{itemize} +\item Studied at the Norwegian University of Science and Technology +\item Worked for the Tor Project during Google Summer of Code in 2009 +\item Developer, security researcher, translation coordinator +\end{itemize} +\end{frame} + +% And here's what we'll talk about +\begin{frame} +\frametitle{What are we talking about?} +\begin{itemize} +\item Crash course on anonymous communications +\item Quick overview of Tor +\item Tor and circumvention +\item Future work +\end{itemize} +\end{frame} + +\begin{frame} +\frametitle{The Tor Project, Inc.} +501(c)(3) non-profit organization dedicated to the research and development of technologies for online anonymity and privacy +\begin{center} +\includegraphics[height=5cm]{../images/2009-oval_sticker_new} +\end{center} +\end{frame} + +% Crash course on anonymous communications +\begin{frame} +\frametitle{What is anonymity?} +\includegraphics[width=10cm]{../images/2llg3ts} +\end{frame} + +% What is the threat model here? +\begin{frame} +\frametitle{Threat model: what can the attacker do?} +\begin{overlayarea}{9cm}{6cm} +\only<1>{\includegraphics[height=7cm]{../images/single_hop_relay}} +\only<2>{\includegraphics[height=7cm]{../images/evil_single_hop_relay}} +\only<3>{\includegraphics[height=7cm]{../images/data_snooping_single_hop_relay}} +\end{overlayarea} +\end{frame} + +\begin{frame} +\frametitle{Anonymity isn't cryptography} +\begin{itemize} +\item Cryptography protects the contents in transit +\item You still know who is talking to whom, how often, and how much data is sent. +\end{itemize} +\begin{center} +\includegraphics[width=5cm]{../images/encryption-cc-by-sa} +\end{center} +\end{frame} + +\begin{frame} +\frametitle{Anonymity isn't steganography} +Attacker can tell Alice is talking to someone, how often, and how much data is sent. +\bigskip + +\begin{center} +\includegraphics[width=5cm]{../images/steganography-cc-by-sa} +\end{center} +\end{frame} + +\begin{frame} +\frametitle{Anonymity isn't just wishful thinking...} +\begin{itemize} +\item "You can't prove it was me!" +\pause \item "Promise you won't look" +\pause \item "Promise you won't remember" +\pause \item "Promise you won't tell" +\pause \item "I didn't write my name on it!" +\pause \item "Isn't the Internet already anonymous?" +\end{itemize} +\end{frame} + +\begin{frame} +\frametitle{Anonymous communication} +\begin{itemize} +\item People have to hide in a crowd of other people ("anonymity loves company") +\item The goal of the system is to make all users look as similar as possible, to give a bigger crowd +\item Hide who is communicating with whom +\item Layered encryption and random delays hide correlation between input traffic and output traffic +\end{itemize} +\end{frame} + +\begin{frame} +\frametitle{Anonymity serves different interests for different user groups} +\begin{itemize} +\item Private citizens: it's privacy +\item Businesses: it's network security +\item Governments: it's traffic-analysis resistance +\item Human rights activists: it's reachability +\end{itemize} +\end{frame} + +\begin{frame} +\frametitle{What is Tor?} +\begin{itemize} +\item Online anonymity software and network +\pause \item Open source, freely available (3-clause BSD license) +\pause \item Active research environment: \\ +Rice, UMN, NSF, NRL, Drexel, Waterloo, Cambridge UK, Bamberg Germany, Boston Univ, Harvard, MIT, RPI, Georgia Tech +\pause \item Funding from US DoD, EFF, Voice of America, Google, NLNet, +Human Rights Watch \\ +\pause \item Increasingly diverse toolset: \\ +Tor, Torbutton, Tor Browser Bundle, TAILS Anonymous Operating System, +Tor Weather, GetTor, Thandy, Orbot, Tor Check, Arm, Torouter, Tor Cloud +and more +\end{itemize} +\end{frame} + +% And what makes Tor different? +\begin{frame} +\frametitle{How is Tor different from other systems?} +\begin{overlayarea}{9cm}{6cm} +\only<1>{\includegraphics[height=7cm]{../../website/images/htw1}} +\only<2>{\includegraphics[height=7cm]{../../website/images/htw2}} +\only<3>{\includegraphics[height=7cm]{../../website/images/htw3}} +\end{overlayarea} +\end{frame} + +\begin{frame} +\frametitle{Tor uses a simple centralized directory protocol} +\begin{itemize} +\item Relays publish self-signed descriptors to directory authorities +\item Authorities publish a consensus list of all relay descriptors +\item Clients download latest consensus from a directory authority or a directory cache +\end{itemize} +\end{frame} + +\begin{frame} +\frametitle{Bridges versus relays} +\begin{itemize} +\item A step forward in the blocking resistance race +\item Bridge relays (or "bridges" for short) are Tor relays that aren't listed in the main Tor directory +\item To use a bridge, you will need to locate one first (can be done using bridges.torproject.org, email, social media etc) +\item A bridge will act as the first hop in the circuit +\end{itemize} +\end{frame} + +\begin{frame} +\frametitle{Hidden services} +\begin{itemize} +\item Tor makes it possible for users to hide their locations while offering various kinds of services, such a website or an im server +\item Using Tor "rendezvous points," other Tor users can connect to these hidden services, each without knowing the other's network identity +\item A hidden service will have an address that ends in .onion, e.g. http://duskgytldkxiuqc6.onion/ +\end{itemize} +\end{frame} + +\begin{frame} +\frametitle{Encryption} +\begin{itemize} +\item Tor uses the 128-bit AES cipher in counter mode to generate a cipher stream +\item And the signing keys are 1024-bit RSA +\item We used to use a 1024-bit safe prime from RFC 2409, section 6.2 as the DH parameter... +\end{itemize} +\end{frame} + +\begin{frame} +\frametitle{} +\includegraphics[scale=0.5]{../images/direct-users-2010-12-01-off-300-2011-01-15-ir} +\end{frame} + +\begin{frame} +\frametitle{} +\includegraphics[scale=0.5]{../images/bridge-users-2010-11-01-300-2011-01-14-ir} +\end{frame} + +\begin{frame} +\frametitle{Encryption} +\begin{itemize} +\item But then we made the DH parameter we use for TLS match the one from Apache's mod\_ssl... +\end{itemize} +\end{frame} + +\begin{frame} +\frametitle{} +\includegraphics[scale=0.5]{../images/direct-users-2011-01-01-off-300-2011-03-01-ir.png} +\end{frame} + +\begin{frame} +\frametitle{} +\includegraphics[scale=0.5]{../images/bridge-users-2011-01-01-300-2011-03-01-ir.png} +\end{frame} + +\begin{frame} +\frametitle{Keys} +\begin{itemize} +\item Each relay maintains a long-term identity key and a short term onion key: +\begin{itemize} +\item The identity key is used to sign relay descriptors +\item The directory authorities also use the identity key to sign the consensus +\item The onion key is used to decrypt requests from clients to set up a circuit and negotiate ephemeral keys +\item The TLS protocol also establishes a short-term link key when communicating between relays +\end{itemize} +\end{itemize} +\end{frame} + +\begin{frame} +\frametitle{Cells} +\includegraphics[width=10cm]{../images/cell-struct.png} +\begin{itemize} +\item Traffic passes along circuits in the Tor network in fixed-size +cells (512 bytes): +\begin{itemize} +\item The header includes a circuit identifier that specifies which +circuit the cell refers to +\item The command describes what to do with the cells payload +\item The entire contents of the header and payload is +encrypted/decrypted together as the relay cell moves along the circuit +\end{itemize} +\end{itemize} +\end{frame} + +\begin{frame} +\frametitle{Tor on the wire} +\includegraphics[width=12cm]{../images/tor-keys3} +\end{frame} + +\begin{frame} +\frametitle{How many people use Tor daily?} +\includegraphics[scale=0.5]{../images/direct-users-2010-09-11-off-300-2011-09-11-all} +\end{frame} + +% Tor and circumvention +\begin{frame} +\frametitle{Attackers can block access to the network} +\begin{itemize} +\item By blocking access to the directory authorities +\item By blocking access to all the relays in the network +\item By blocking access to all known bridges in the network +\item By preventing users from finding the software +\end{itemize} +\end{frame} + +% China +\begin{frame} +\frametitle{Tor and circumvention in China} +\includegraphics[scale=0.5]{../images/direct-users-2010-09-11-off-300-2011-09-11-cn} +\end{frame} + +\begin{frame} +\frametitle{Tor and circumvention in China} +\includegraphics[scale=0.5]{../images/bridge-users-2010-09-11-300-2011-09-11-cn} +\end{frame} + +% Egypt +\begin{frame} +\frametitle{Tor and circumvention in Egypt} +\includegraphics[scale=0.5]{../images/direct-users-2010-09-11-off-300-2011-09-11-eg} +\end{frame} + +\begin{frame} +\frametitle{Tor and circumvention in Egypt} +\includegraphics[scale=0.5]{../images/bridge-users-2010-09-11-300-2011-09-11-eg} +\end{frame} + +% Libya +\begin{frame} +\frametitle{Tor and circumvention in Libya} +\includegraphics[scale=0.5]{../images/direct-users-2010-09-11-off-300-2011-09-11-ly} +\end{frame} + +\begin{frame} +\frametitle{Tor and circumvention in Libya} +\includegraphics[scale=0.5]{../images/bridge-users-2010-09-11-300-2011-09-11-ly} +\end{frame} + +% Future work +\begin{frame} +\frametitle{Future work, part 1} +\begin{itemize} +\item The Torouter project: hardware project to provide an easy to setup Tor bridge or relay +\item The Tor Cloud project: provides bridge-by-default and relay-by-default images for Amazon EC2 +\end{itemize} +\end{frame} + +\begin{frame} +\frametitle{Future work, part 2} +\begin{itemize} +\item Pluggable transports: a plug-in system that can evade many censorship systems by disguising Tor traffic as, for example, standard HTTP traffic +\item Obfuscated proxy: protocol obfuscation for TCP protocols prevent third party from identifying protocol based on message contents +\end{itemize} +\end{frame} + +\begin{frame} +\frametitle{Future work, part 3} +\begin{itemize} +\item Censorship resistance research: reachability testing of the Tor network from within certain countries +\item IPv6: goal for Tor 0.2.3.x is for bridges to handle IPv6-only clients and exits can handle IPv6 addresses +\end{itemize} +\end{frame} + +% Demonstration of TBB +\begin{frame} +\frametitle{Time for a demo} +Demonstration of Tor Browser Bundle +\end{frame} + +\begin{frame} +\frametitle{} +\begin{center} +\large Questions? \\ +\vspace{10 mm} +runa(a)torproject.org \\ +https://www.torproject.org/ +\end{center} +\end{frame} + +\end{document}

1 0