- tor-commits - lists.torproject.org

[tor/master] Merge branch 'bufev_ratelim_and_wm'
by nickm＠torproject.org 24 Aug '11

24 Aug '11

commit b161674d5dc312f95a495bed3c25d892fc9f0550 Merge: ede9cd4 dfcd3d9 Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Aug 24 17:57:55 2011 -0400 Merge branch 'bufev_ratelim_and_wm' changes/bug3804 | 9 +++++++++ changes/bug3805 | 5 +++++ src/common/compat_libevent.c | 26 ++++++++++++++++++++++++++ src/common/compat_libevent.h | 6 ++++++ src/common/tortls.c | 5 +++++ src/or/connection.c | 7 ++++++- src/or/connection_or.c | 4 ++-- 7 files changed, 59 insertions(+), 3 deletions(-)

1 0

[tor/master] lround() missing in MSVC
by nickm＠torproject.org 24 Aug '11

24 Aug '11

commit 5939c09d35de5be4b135bb0ad6dde24700e75fc2 Author: Gisle Vanem <gvanem(a)broadpark.no> Date: Wed Aug 24 13:52:44 2011 -0400 lround() missing in MSVC lround() is missing in MS Visual-C's <math.h>. Not available anywhere. Here is an easy patch. --- changes/msvc_lround | 4 ++++ src/common/util.c | 4 ++++ 2 files changed, 8 insertions(+), 0 deletions(-) diff --git a/changes/msvc_lround b/changes/msvc_lround new file mode 100644 index 0000000..e4aea95 --- /dev/null +++ b/changes/msvc_lround @@ -0,0 +1,4 @@ + o Build fixes: + - Provide a substitute implementation of lround() for MSVC, which + apparently lacks it. Patch from Gisle Vanem. + diff --git a/src/common/util.c b/src/common/util.c index b47b9c5..ee0acbb 100644 --- a/src/common/util.c +++ b/src/common/util.c @@ -334,7 +334,11 @@ tor_mathlog(double d) long tor_lround(double d) { +#ifdef _MSC_VER + return (long)(d > 0 ? d + 0.5 : ceil(d - 0.5)); +#else return lround(d); +#endif } /** Returns floor(log2(u64)). If u64 is 0, (incorrectly) returns 0. */

1 0

[tor/master] Merge remote-tracking branch 'origin/maint-0.2.2'
by nickm＠torproject.org 24 Aug '11

24 Aug '11

commit ede9cd4f99e4cd1c0c4bcf1a3ac994e87054cc2d Merge: dd6a9a9 5939c09 Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Aug 24 13:53:17 2011 -0400 Merge remote-tracking branch 'origin/maint-0.2.2' changes/msvc_lround | 4 ++++ src/common/util.c | 4 ++++ 2 files changed, 8 insertions(+), 0 deletions(-)

1 0

[tor/maint-0.2.2] lround() missing in MSVC
by nickm＠torproject.org 24 Aug '11

24 Aug '11

commit 5939c09d35de5be4b135bb0ad6dde24700e75fc2 Author: Gisle Vanem <gvanem(a)broadpark.no> Date: Wed Aug 24 13:52:44 2011 -0400 lround() missing in MSVC lround() is missing in MS Visual-C's <math.h>. Not available anywhere. Here is an easy patch. --- changes/msvc_lround | 4 ++++ src/common/util.c | 4 ++++ 2 files changed, 8 insertions(+), 0 deletions(-) diff --git a/changes/msvc_lround b/changes/msvc_lround new file mode 100644 index 0000000..e4aea95 --- /dev/null +++ b/changes/msvc_lround @@ -0,0 +1,4 @@ + o Build fixes: + - Provide a substitute implementation of lround() for MSVC, which + apparently lacks it. Patch from Gisle Vanem. + diff --git a/src/common/util.c b/src/common/util.c index b47b9c5..ee0acbb 100644 --- a/src/common/util.c +++ b/src/common/util.c @@ -334,7 +334,11 @@ tor_mathlog(double d) long tor_lround(double d) { +#ifdef _MSC_VER + return (long)(d > 0 ? d + 0.5 : ceil(d - 0.5)); +#else return lround(d); +#endif } /** Returns floor(log2(u64)). If u64 is 0, (incorrectly) returns 0. */

1 0

[arm/master] Interpretor panel for raw control port access
by atagar＠torproject.org 24 Aug '11

24 Aug '11

commit 0016b2be0fac62ae840a1259ee65269defa3654c Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Aug 24 10:07:24 2011 -0700 Interpretor panel for raw control port access This adds a new page to the interface, providing an interactive interpretor that lets the user send/receive control commands. This includes history search and will be expanded to have other usability features (tab completion, search, etc). --- README | 1 + src/cli/__init__.py | 2 +- src/cli/controller.py | 3 + src/cli/interpretorPanel.py | 232 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 237 insertions(+), 1 deletions(-) diff --git a/README b/README index 16d0965..cbaf2f7 100644 --- a/README +++ b/README @@ -193,6 +193,7 @@ Layout: logPanel.py - (page 1) displays tor, arm, and torctl events configPanel.py - (page 3) editor panel for the tor configuration torrcPanel.py - (page 4) displays torrc and validation + interpretorPanel.py - (page 5) interpretor for control port access util/ __init__.py diff --git a/src/cli/__init__.py b/src/cli/__init__.py index b0d6bd8..fa23640 100644 --- a/src/cli/__init__.py +++ b/src/cli/__init__.py @@ -2,5 +2,5 @@ Panels, popups, and handlers comprising the arm user interface. """ -__all__ = ["configPanel", "controller", "headerPanel", "logPanel", "popups", "torrcPanel", "wizard"] +__all__ = ["configPanel", "controller", "headerPanel", "interpretorPanel", "logPanel", "popups", "torrcPanel", "wizard"] diff --git a/src/cli/controller.py b/src/cli/controller.py index d2194ca..bc69e79 100644 --- a/src/cli/controller.py +++ b/src/cli/controller.py @@ -15,6 +15,7 @@ import cli.headerPanel import cli.logPanel import cli.configPanel import cli.torrcPanel +import cli.interpretorPanel import cli.graphing.graphPanel import cli.graphing.bandwidthStats import cli.graphing.connStats @@ -97,6 +98,8 @@ def initController(stdscr, startTime): if CONFIG["features.panels.show.torrc"]: pagePanels.append([cli.torrcPanel.TorrcPanel(stdscr, cli.torrcPanel.Config.TORRC, config)]) + pagePanels.append([cli.interpretorPanel.InterpretorPanel(stdscr)]) + # initializes the controller ARM_CONTROLLER = Controller(stdscr, stickyPanels, pagePanels) diff --git a/src/cli/interpretorPanel.py b/src/cli/interpretorPanel.py new file mode 100644 index 0000000..b7c3fe0 --- /dev/null +++ b/src/cli/interpretorPanel.py @@ -0,0 +1,232 @@ +""" +Panel providing raw control port access with syntax hilighting, usage +information, tab completion, and other usability features. +""" + +import curses + +from util import enum, panel, textInput, torTools, uiTools + +from TorCtl import TorCtl + +Formats = enum.Enum("PROMPT", "INPUT", "INPUT_INTERPRETOR", "INPUT_CMD", "INPUT_ARG", "OUTPUT", "USAGE", "HELP", "ERROR") + +PROMPT = ">>> " +USAGE_INFO = "to use this panel press enter" + +# limits used for cropping +COMMAND_BACKLOG = 100 +LINES_BACKLOG = 2000 + +class InterpretorPanel(panel.Panel): + def __init__(self, stdscr): + panel.Panel.__init__(self, stdscr, "interpretor", 0) + self.isInputMode = False + self.scroll = 0 + self.formats = {} # lazy loaded curses formatting constants + self.previousCommands = [] # user input, newest to oldest + + # contents of the panel (oldest to newest), each line is a list of (msg, + # format enum) tuples + + self.contents = [[(PROMPT, Formats.PROMPT), (USAGE_INFO, Formats.USAGE)]] + + def prompt(self): + """ + Enables the interpretor, prompting for input until the user enters esc or + a blank line. + """ + + if not self.formats: self._initFormats() + self.isInputMode = True + + panel.CURSES_LOCK.acquire() + + while self.isInputMode: + self.redraw(True) + + # intercepts input so user can cycle through the history + validator = textInput.BasicValidator() + validator = textInput.HistoryValidator(self.previousCommands, validator) + + xOffset = len(PROMPT) + if len(self.contents) > self.maxY - 1: + xOffset += 3 # offset for scrollbar + + input = self.getstr(min(self.maxY - 1, len(self.contents)), xOffset, "", self.formats[Formats.INPUT], validator = validator) + + isDone = self.handleQuery(input) + + if isDone: + self.isInputMode = False + self.redraw(True) + + panel.CURSES_LOCK.release() + + def handleQuery(self, input): + """ + Processes the given input. Requests starting with a '/' are special + commands to the interpretor, and anything else is sent to the control port. + This returns a boolean to indicate if the interpretor should terminate or + not. + + Arguments: + input - user input to be processed + """ + + if not input or not input.strip(): return True + input = input.strip() + inputEntry, outputEntry = [(PROMPT, Formats.PROMPT)], [] + conn = torTools.getConn() + + # input falls into three general categories: + # - interpretor command which starts with a '/' + # - controller commands handled by torTools (this allows for caching, + # proper handling by the rest of arm, etc) + # - unrecognized controller command, this has the possability of confusing + # arm... + + if input.startswith("/"): + # interpretor command + inputEntry.append((input, Formats.INPUT_INTERPRETOR)) + outputEntry.append(("Not yet implemented...", Formats.ERROR)) # TODO: implement + + # TODO: add /help option + # TODO: add /write option + else: + # controller command + if " " in input: cmd, arg = input.split(" ", 1) + else: cmd, arg = input, "" + + inputEntry.append((cmd + " ", Formats.INPUT_CMD)) + if arg: inputEntry.append((arg, Formats.INPUT_ARG)) + + if cmd.upper() == "GETINFO": + try: + response = conn.getInfo(arg, suppressExc = False) + outputEntry.append((response, Formats.OUTPUT)) + except Exception, exc: + outputEntry.append((str(exc), Formats.ERROR)) + elif cmd.upper() == "SETCONF": + if "=" in arg: + param, value = arg.split("=", 1) + + try: + conn.setOption(param.strip(), value.strip()) + except Exception, exc: + outputEntry.append((str(exc), Formats.ERROR)) + else: + # TODO: resets the attribute + outputEntry.append(("Not yet implemented...", Formats.ERROR)) # TODO: implement + else: + try: + response = conn.getTorCtl().sendAndRecv("%s\r\n" % input) + + for entry in response: + # Response entries are tuples with the response code, body, and + # extra info. For instance: + # ('250', 'version=0.2.2.23-alpha (git-b85eb949b528f4d7)', None) + + if len(entry) == 3: + outputEntry.append((entry[1], Formats.OUTPUT)) + except Exception, exc: + outputEntry.append((str(exc), Formats.ERROR)) + + self.previousCommands.insert(0, input) + self.previousCommands = self.previousCommands[:COMMAND_BACKLOG] + + promptEntry = self.contents.pop() # removes old prompt entry + self.contents += _splitOnNewlines(inputEntry) + self.contents += _splitOnNewlines(outputEntry) + self.contents.append(promptEntry) + + # if too long then crop lines + cropLines = len(self.contents) - LINES_BACKLOG + if cropLines > 0: self.contents = self.contents[cropLines:] + + return False + + def handleKey(self, key): + # TODO: allow contents to be searched (with hilighting?) + + isKeystrokeConsumed = True + if uiTools.isSelectionKey(key): + self.prompt() + elif uiTools.isScrollKey(key) and not self.isInputMode: + pageHeight = self.getPreferredSize()[0] - 1 + newScroll = uiTools.getScrollPosition(key, self.scroll, pageHeight, len(self.contents)) + + if self.scroll != newScroll: + self.scroll = newScroll + self.redraw(True) + else: isKeystrokeConsumed = False + + return isKeystrokeConsumed + + def draw(self, width, height): + if not self.formats: self._initFormats() + + # page title + usageMsg = " (enter \"/help\" for usage or a blank line to stop)" if self.isInputMode else "" + self.addstr(0, 0, "Control Interpretor%s:" % usageMsg, curses.A_STANDOUT) + + xOffset = 0 + if len(self.contents) > height - 1: + # if we're in input mode then make sure the last line is visible + if self.isInputMode: + self.scroll = len(self.contents) - height + 1 + + xOffset = 3 + self.addScrollBar(self.scroll, self.scroll + height - 1, len(self.contents), 1) + + # draws prior commands and output + drawLine = 1 + for entry in self.contents[self.scroll:]: + cursor = xOffset + + for msg, formatEntry in entry: + format = self.formats.get(formatEntry, curses.A_NORMAL) + self.addstr(drawLine, cursor, msg, format) + cursor += len(msg) + + drawLine += 1 + if drawLine >= height: break + + def _initFormats(self): + self.formats[Formats.PROMPT] = curses.A_BOLD | uiTools.getColor("green") + self.formats[Formats.INPUT] = uiTools.getColor("cyan") + self.formats[Formats.INPUT_INTERPRETOR] = curses.A_BOLD | uiTools.getColor("magenta") + self.formats[Formats.INPUT_CMD] = curses.A_BOLD | uiTools.getColor("green") + self.formats[Formats.INPUT_ARG] = curses.A_BOLD | uiTools.getColor("cyan") + self.formats[Formats.OUTPUT] = uiTools.getColor("blue") + self.formats[Formats.USAGE] = uiTools.getColor("cyan") + self.formats[Formats.HELP] = uiTools.getColor("magenta") + self.formats[Formats.ERROR] = curses.A_BOLD | uiTools.getColor("red") + +def _splitOnNewlines(entry): + """ + Splits a list of (msg, format) tuples on newlines into a list of lines. + + Arguments: + entry - list of display tuples + """ + + results, tmpLine = [], [] + entry = list(entry) # shallow copy + + while entry: + msg, format = entry.pop(0) + + if "\n" in msg: + msg, remainder = msg.split("\n", 1) + entry.insert(0, (remainder, format)) + + tmpLine.append((msg, format)) + results.append(tmpLine) + tmpLine = [] + else: + tmpLine.append((msg, format)) + + if tmpLine: results.append(tmpLine) + return results +

1 0

[torouter/master] Require login for tor status page
by hellais＠torproject.org 24 Aug '11

24 Aug '11

commit 817713dc33ae88217b61c73d4b9b0446a1a89855 Author: Arturo Filastò <hellais(a)torproject.org> Date: Wed Aug 24 18:22:37 2011 +0200 Require login for tor status page --- packages/torouter-web/src/tui/controllers/tor.py | 32 ++++++++++++++------- 1 files changed, 21 insertions(+), 11 deletions(-) diff --git a/packages/torouter-web/src/tui/controllers/tor.py b/packages/torouter-web/src/tui/controllers/tor.py index 491c6b7..e671d80 100644 --- a/packages/torouter-web/src/tui/controllers/tor.py +++ b/packages/torouter-web/src/tui/controllers/tor.py @@ -9,16 +9,23 @@ The main Tor status page """ class status: def GET(self): - trc = parsing.torrc(config.torrc_file) - trc.parse() - output = trc.html_output() - return render.base(render.torstatus(output,config.torrc_file)) + if session.is_logged() > 0: + trc = parsing.torrc(config.torrc_file) + trc.parse() + output = trc.html_output() + return render.base(render.torstatus(output,config.torrc_file)) + else: + return render.base(render.login()) + def POST(self): - trc = parsing.torrc(config.torrc_file) - trc.parse() - output = trc.html_output() - return render.base(render.torstatus(output,config.torrc_file)) + if session.is_logged() > 0: + trc = parsing.torrc(config.torrc_file) + trc.parse() + output = trc.html_output() + return render.base(render.torstatus(output,config.torrc_file)) + else: + return render.base(render.login()) """ Tor configuration page @@ -32,9 +39,12 @@ class torrc: return True def GET(self): - trc = parsing.torrc(config.torrc_file) - output = trc.output() - return render.base(render.torconfig(output)) + if session.is_logged() > 0: + trc = parsing.torrc(config.torrc_file) + output = trc.output() + return render.base(render.torconfig(output)) + else: + return render.base(render.login()) def POST(self): if session.is_logged() > 0:

1 0

[torouter/master] use the system torrc by default here
by ioerror＠torproject.org 24 Aug '11

24 Aug '11

commit b0984f4e97cfd7a3274b1510c40e5d2bdf10b531 Author: Jacob Appelbaum <jacob(a)appelbaum.net> Date: Wed Aug 24 16:42:14 2011 +0200 use the system torrc by default here --- packages/torouter-web/src/tui/utils/parsing.py | 6 +++++- 1 files changed, 5 insertions(+), 1 deletions(-) diff --git a/packages/torouter-web/src/tui/utils/parsing.py b/packages/torouter-web/src/tui/utils/parsing.py index 4d52c87..690d15d 100644 --- a/packages/torouter-web/src/tui/utils/parsing.py +++ b/packages/torouter-web/src/tui/utils/parsing.py @@ -6,7 +6,11 @@ import os, re class interfaces: def __init__(self,filename): self.filename = filename - self.fp = open(filename, "r") + try: + self.fp = open(filename, "r") + except: + # this will happen on our first run or at reboot + self.fp = open("/etc/tor/torrc", "r") self.wifi = {} self.eth1 = {} self.eth0 = {}

1 0

[metrics-web/master] Add George's censorship detector PDF to the website.
by karsten＠torproject.org 24 Aug '11

24 Aug '11

commit 30084d8db5750806ad448132e2c042bb158455d6 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Aug 22 16:58:32 2011 +0200 Add George's censorship detector PDF to the website. --- web/WEB-INF/papers.jsp | 3 +++ web/WEB-INF/users.jsp | 6 ++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/web/WEB-INF/papers.jsp b/web/WEB-INF/papers.jsp index 6011a20..a4ce6c2 100644 --- a/web/WEB-INF/papers.jsp +++ b/web/WEB-INF/papers.jsp @@ -45,6 +45,9 @@ been included in the <a href="#papers">papers</a> above or in the daily updated <a href="graphs.html">graphs</a>. <ul> + <li>An anomaly-based censorship-detection system for Tor (<a + href="papers/detector-2011-08-11.pdf">PDF</a>, 106K from August + 11, 2011).</li> <li>An Analysis of Tor Relay Stability (<a href="papers/relay-stability-2011-06-30.pdf">PDF</a>, 1.3M from June 30, 2011, diff --git a/web/WEB-INF/users.jsp b/web/WEB-INF/users.jsp index bb4e11c..624d956 100644 --- a/web/WEB-INF/users.jsp +++ b/web/WEB-INF/users.jsp @@ -45,7 +45,8 @@ based on the requests seen by a few dozen directory mirrors.</p> </c:forEach> </select> </p><p> - Show possible censorship events if available (BETA) + Show possible censorship events if available (<a + href="papers/detector-2011-08-11.pdf">BETA</a>) <input type="checkbox" name="events" value="on" <c:if test="${direct_users_events[0] eq 'on'}"> checked</c:if> ></input> @@ -91,7 +92,8 @@ based on the requests seen by a few dozen directory mirrors.</p> </div> </form> <a name="censorship-events"></a> -<p><b>Top-10 countries by possible censorship events (BETA):</b></p> +<p><b>Top-10 countries by possible censorship events (<a + href="papers/detector-2011-08-11.pdf">BETA</a>):</b></p> <table> <tr> <th>Country</th>

1 0

[metrics-tasks/master] Add George's censorship detector LaTeX sources (#2718).
by karsten＠torproject.org 24 Aug '11

24 Aug '11

commit 83ba791573db7d407aa098d43a43e86692bd7d5c Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Aug 22 16:52:40 2011 +0200 Add George's censorship detector LaTeX sources (#2718). --- task-2718/detector.tex | 169 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 169 insertions(+), 0 deletions(-) diff --git a/task-2718/detector.tex b/task-2718/detector.tex new file mode 100644 index 0000000..d1510c6 --- /dev/null +++ b/task-2718/detector.tex @@ -0,0 +1,169 @@ +\documentclass{article} +\begin{document} +\author{George Danezis\\{\tt gdane(a)microsoft.com}} +\title{An anomaly-based censorship-detection\\system for Tor} +\date{August 11, 2011} +\maketitle + +\section{Introduction} + +The Tor project is currently the most widely used anonymity and censorship +resistance system worldwide. +As a result, national governments occasionally or regularly block access +to its facilities for replaying traffic. +Major blocking might be easy to detect, but blocking from smaller +jurisdictions, with fewer users, could take some time to detect. +Yet, early detection may be key to deploy countermeasures. +We have designed an ``early warning'' systems that looks for anomalies in +the volumes of connections from users in different jurisdictions and flags +potential censorship events. +Special care has been taken to ensure the detector is robust to +manipulations and noise that could be used to block without raising an +alert. + +The detector works on aggregate number of users connecting to a fraction +of directory servers per day. +That set of statistics are gathered and provided by the Tor project in a +sanitised form to minimise the potential for harm to active users. +The data collection has been historically patchy, introducing wild +variations over time that is not due to censorship. +The detector is based on a simple model of the number of users per day per +jurisdictions. +That model is used to assess whether the number of users we observe is +typical, too high or too low. +In a nutshell the prediction on any day is based on activity of previous +days locally as well as worldwide. + +\section{The model intuition} + +The detector is based on a model of the number of connection from every +jurisdiction based on the number of connections in the past as well as a +model of ``natural'' variation or evolution of the number of connections. +More concretely, consider that at time $t_i$ we have observed $C_{ij}$ +connections from country $j$. +Since we are concerned with abnormal increases or falls in the volume of +connections we compare this with the number of connections we observed at +a past time $t_{i-1}$ denoted as $C_{(t-1)j}$ from the same country $j$. +The ratio $R_{ij} = C_{ij} / C_{(i-1)j}$ summarises the change in the +number of users. +Inferring whether the ratio $R_{ij}$ is within an expected or unexpected +range allows us to detect potential censorship events. + +We consider that a ratio $R_{ij}$ within a jurisdiction $j$ is ``normal'' +if it follows the trends we are observing in other jurisdictions. +Therefore for every time $t_i$ we use the ratios $R_{ij}$ of many +countries to understand the global trends of usage of Tor, and we compare +specific countries' ratios to this model. +If they are broadly within the global trends we assume no censorship is +taking place, otherwise we raise an alarm. + +\section{The model details} + +We model each data point $C_{ij}$ of the number of users connected at time +$t_i$ from country $j$ as a single sample of a Poisson process with a rate +$\lambda_{ij}$ modelling the underlying number of users. +The Poisson process allows us to take into account that in jurisdictions +with very few users we will naturally have some days of relatively low or +high usage---just because a handful of users may or may not use Tor in a +day. +Even observing zero users from such jurisdictions on some days may not be +a significant event. + +We are trying to detect normal or abnormal changes in the rate of change +of the rate $\lambda_{ij}$ between time $t_i$ and a previous time +$t_{i-1}$ for jurisdiction $j$ compared with other jurisdictions. +This is $\lambda_{ij} / \lambda_{(i-1)j}$ which for jurisdictions with a +high number of users is very close to $C_{ij} / C_{(i-1)j} = R_{ij}$. +We model $R_{ij}$'s from all jurisdictions as following a Normal +distribution $N(m,v)$ with a certain mean ($m$) and variance ($v$) to be +inferred. +This is of course a modelling assumption. +We use a normal distribution because given its parameters it represents +the distribution with most uncertainty: as a result the model has higher +variance than the real world, ensuring that it gives fewer false alarms of +censorship. + +The parameters of $N(m,v)$ are inferred directly as point estimates from +the readings in a set of jurisdictions. +Then the probability of a given country ratio $R_{ij}$ is compared with +that distribution: an alarm is raised if the probability of the ratio is +above or below a certain threshold. + +\section{The model robustness} + +At every stage of detections we follow special steps to ensure the +detection is robust to manipulation by jurisdictions interested in +censoring fast without being detected. +First the parameter estimation for $N(m,v)$ is hardened: we only use the +largest jurisdictions to model ratios and within those we remove any +outliers that fall outside four inter-quartile ranges off the median. +This ensures that a jurisdiction with a very high or very low ratio does +not influence the model of ratios (and can be subsequently detected as +abnormal). + +Since we chose jurisdictions with many users to build the model of ratios, +we can approximate the rates $\lambda_{ij}$ by the actual observed number +of users $C_{ij}$. +On the other hand when we try to detect whether a jurisdiction has a +typical rate we cannot make this assumption. +The rate of a Poisson variable $\lambda_{ij}$ can be inferred by a single +sample $C_{ij}$ using a Gamma prior, in which case it follows a Gamma +distribution. +In practice (because we are using a single sample) this in turn can be +approximated using a Poisson distribution with parameter $C_{ij}$. +Using this observation we extract a range of possible rates for each +jurisdiction based on $C_{ij}$, namely $\lambda_{ij_{min}}$ and +$\lambda_{ij_{max}}$. +Then we test whether that full range is within the typical range +distribution---if not we raise an alarm. + +\section{The parameters} + +The deployed model considers a time interval of seven (7) days to model +connection rates (i.e. $t_i$ - $t_{i-1} = 7$ days). +The key reason for a weekly model is our observation that some +jurisdictions exhibit weekly patterns. +A previous day model would then raise alarms every time weekly patterns +emerged. +We use the 50 largest jurisdictions to build our models of typical ratios +of traffic over time---as expected most of them are in countries where no +mass censorship has been reported. +This strengthens the model as describing ``normal'' Tor connection +patterns. + +We consider that a ratio of connections is typical if it falls within the +99.99~\% percentile of the Normal distribution $N(m,v)$ modelling ratios. +This ensures that the expected rate of false alarms is about $1 / 10000$, +and therefore only a handful a week (given the large number of +jurisdictions). +Similarly, we infer the range of the rate of usage from each jurisdiction +(given $C_{ij}$) to be the 99.99~\% percentile range of a Poisson +distribution with parameter $C_{ij}$. +This full range must be within the typical range of ratios to avoid +raising an alarm. + +\section{Further work} + +The detector uses time series of user connections to directory servers to +detect censorship. +Any censorship method that does not influence these numbers would as a +result not be detected. +This includes active attacks: a censor could substitute genuine requests +with requests from adversary controlled machines to keep numbers within +the typical ranges. + +A better model, making use of multiple previous readings, may improve the +accuracy of detection. +In particular, when a censorship event occurs there is a structural +change, and a model based on modelling the future on user loads before the +event will fail. +This is not a critical problem, as these ``false positives'' are +concentrated after real censorship events, but the effect may be confusing +to a reader. +On the other hand, a jurisdiction can still censor by limiting the rate of +censorship to be within the typical range for the time period concerned. +Therefore adapting the detector to run on longer periods would be +necessary to detect such attacks. + +\end{document} +

1 0

[arm/master] Making curses text input extensible
by atagar＠torproject.org 23 Aug '11

23 Aug '11

commit dc18065b395ef409cfff58525f24bb0727ac3ed6 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Aug 23 09:34:14 2011 -0700 Making curses text input extensible Adding utility for text input interceptors which can be chained to add capabilities to text input fields (like tab completion). They'll be implemented as needed. --- README | 1 + src/util/__init__.py | 2 +- src/util/panel.py | 59 +++------------------------- src/util/textInput.py | 101 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 110 insertions(+), 53 deletions(-) diff --git a/README b/README index cbaf31a..16d0965 100644 --- a/README +++ b/README @@ -205,6 +205,7 @@ Layout: procTools.py - queries process & system information from /proc contents procName.py - renames our process to a friendlier name sysTools.py - helper for system calls, providing client side caching + textInput.py - expands the capabilities of text input fields torConfig.py - functions for working with the torrc and config options torTools.py - TorCtl wrapper, providing caching and derived information uiTools.py - helper functions for presenting the user interface diff --git a/src/util/__init__.py b/src/util/__init__.py index cd5d721..973527e 100644 --- a/src/util/__init__.py +++ b/src/util/__init__.py @@ -4,5 +4,5 @@ application's status, making cross platform system calls, parsing tor data, and safely working with curses (hiding some of the gory details). """ -__all__ = ["conf", "connections", "enum", "gtkTools", "hostnames", "log", "panel", "procTools", "procName", "sysTools", "torConfig", "torTools", "uiTools"] +__all__ = ["conf", "connections", "enum", "gtkTools", "hostnames", "log", "panel", "procTools", "procName", "sysTools", "textInput", "torConfig", "torTools", "uiTools"] diff --git a/src/util/panel.py b/src/util/panel.py index 0e84005..50ae2c0 100644 --- a/src/util/panel.py +++ b/src/util/panel.py @@ -9,7 +9,7 @@ import curses.ascii import curses.textpad from threading import RLock -from util import log, uiTools +from util import log, textInput, uiTools # global ui lock governing all panel instances (curses isn't thread save and # concurrency bugs produce especially sinister glitches) @@ -575,7 +575,7 @@ class Panel(): baseMsg = "Unclosed formatting tag%s:" % ("s" if len(expectedCloseTags) > 1 else "") raise ValueError("%s: '%s'\n \"%s\"" % (baseMsg, "', '".join(expectedCloseTags), msg)) - def getstr(self, y, x, initialText = "", format = None, maxWidth = None): + def getstr(self, y, x, initialText = "", format = None, maxWidth = None, validator = None): """ Provides a text field where the user can input a string, blocking until they've done so and returning the result. If the user presses escape then @@ -592,6 +592,7 @@ class Panel(): initialText - starting text in this field format - format used for the text maxWidth - maximum width for the text field + validator - custom TextInputValidator for handling keybindings """ if not format: format = curses.A_NORMAL @@ -619,8 +620,11 @@ class Panel(): textbox = curses.textpad.Textbox(inputSubwindow) + if not validator: + validator = textInput.BasicValidator() + textbox.win.attron(format) - userInput = textbox.edit(lambda key: _textboxValidate(textbox, key)).strip() + userInput = textbox.edit(lambda key: validator.validate(key, textbox)).strip() textbox.win.attroff(format) if textbox.lastcmd == curses.ascii.BEL: userInput = None @@ -727,52 +731,3 @@ class Panel(): log.log(CONFIG["log.panelRecreated"], msg) return recreate -def _textboxValidate(textbox, key): - """ - Interceptor for keystrokes given to a textbox, doing the following: - - quits by setting the input to curses.ascii.BEL when escape is pressed - - stops the cursor at the end of the box's content when pressing the right - arrow - - home and end keys move to the start/end of the line - """ - - y, x = textbox.win.getyx() - - if curses.ascii.isprint(key) and x < textbox.maxx: - # Shifts the existing text forward so input is an insert method rather - # than replacement. The curses.textpad accepts an insert mode flag but - # this has a couple issues... - # - The flag is only available for Python 2.6+, before that the - # constructor only accepted a subwindow argument as per: - # https://trac.torproject.org/projects/tor/ticket/2354 - # - The textpad doesn't shift text that has text attributes. This is - # because keycodes read by textbox.win.inch() includes formatting, - # causing the curses.ascii.isprint() check it does to fail. - - currentInput = textbox.gather() - textbox.win.addstr(y, x + 1, currentInput[x:textbox.maxx - 1]) - textbox.win.move(y, x) # reverts cursor movement during gather call - elif key == 27: - # curses.ascii.BEL is a character codes that causes textpad to terminate - return curses.ascii.BEL - elif key == curses.KEY_HOME: - textbox.win.move(y, 0) - return None - elif key in (curses.KEY_END, curses.KEY_RIGHT): - msgLen = len(textbox.gather()) - textbox.win.move(y, x) # reverts cursor movement during gather call - - if key == curses.KEY_END and msgLen > 0 and x < msgLen - 1: - # if we're in the content then move to the end - textbox.win.move(y, msgLen - 1) - return None - elif key == curses.KEY_RIGHT and x >= msgLen - 1: - # don't move the cursor if there's no content after it - return None - elif key == 410: - # if we're resizing the display during text entry then cancel it - # (otherwise the input field is filled with nonprintable characters) - return curses.ascii.BEL - - return key - diff --git a/src/util/textInput.py b/src/util/textInput.py new file mode 100644 index 0000000..9422f81 --- /dev/null +++ b/src/util/textInput.py @@ -0,0 +1,101 @@ +""" +Provides input validators that provide text input with various capabilities. +These can be chained together with the first matching validator taking +precidence. +""" + +import curses + +PASS = -1 + +class TextInputValidator: + """ + Basic interface for validators. Implementations should override the handleKey + method. + """ + + def __init__(self, nextValidator = None): + self.nextValidator = nextValidator + + def validate(self, key, textbox): + """ + Processes the given key input for the textbox. This may modify the + textbox's content, cursor position, etc depending on the functionality + of the validator. This returns the key that the textbox should interpret, + PASS if this validator doesn't want to take any action. + + Arguments: + key - key code input from the user + textbox - curses Textbox instance the input came from + """ + + result = self.handleKey(key, textbox) + + if result != PASS: + return result + elif self.nextValidator: + return self.nextValidator.validate(key, textbox) + else: return key + + def handleKey(self, key, textbox): + """ + Process the given keycode with this validator, returning the keycode for + the textbox to process, and PASS if this doesn't want to modify it. + + Arguments: + key - key code input from the user + textbox - curses Textbox instance the input came from + """ + + return PASS + +class BasicValidator(TextInputValidator): + """ + Interceptor for keystrokes given to a textbox, doing the following: + - quits by setting the input to curses.ascii.BEL when escape is pressed + - stops the cursor at the end of the box's content when pressing the right + arrow + - home and end keys move to the start/end of the line + """ + + def handleKey(self, key, textbox): + y, x = textbox.win.getyx() + + if curses.ascii.isprint(key) and x < textbox.maxx: + # Shifts the existing text forward so input is an insert method rather + # than replacement. The curses.textpad accepts an insert mode flag but + # this has a couple issues... + # - The flag is only available for Python 2.6+, before that the + # constructor only accepted a subwindow argument as per: + # https://trac.torproject.org/projects/tor/ticket/2354 + # - The textpad doesn't shift text that has text attributes. This is + # because keycodes read by textbox.win.inch() includes formatting, + # causing the curses.ascii.isprint() check it does to fail. + + currentInput = textbox.gather() + textbox.win.addstr(y, x + 1, currentInput[x:textbox.maxx - 1]) + textbox.win.move(y, x) # reverts cursor movement during gather call + elif key == 27: + # curses.ascii.BEL is a character codes that causes textpad to terminate + return curses.ascii.BEL + elif key == curses.KEY_HOME: + textbox.win.move(y, 0) + return None + elif key in (curses.KEY_END, curses.KEY_RIGHT): + msgLen = len(textbox.gather()) + textbox.win.move(y, x) # reverts cursor movement during gather call + + if key == curses.KEY_END and msgLen > 0 and x < msgLen - 1: + # if we're in the content then move to the end + textbox.win.move(y, msgLen - 1) + return None + elif key == curses.KEY_RIGHT and x >= msgLen - 1: + # don't move the cursor if there's no content after it + return None + elif key == 410: + # if we're resizing the display during text entry then cancel it + # (otherwise the input field is filled with nonprintable characters) + return curses.ascii.BEL + + return PASS +

1 0