- tor-commits - lists.torproject.org

[metrics-tasks/master] Add bridge pool assignment file parser for #2680.
by karsten＠torproject.org 14 Mar '11

14 Mar '11

commit 1e8dbd3857c58e26f57973da4d64170eae0e1be6 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Mar 14 14:15:33 2011 +0100 Add bridge pool assignment file parser for #2680. --- task-2680/.gitignore | 2 + task-2680/ProcessSanitizedAssignments.java | 102 ++++++++++++++++++++++++++++ task-2680/README | 40 ++++++++++- task-2680/verify.R | 6 ++ 4 files changed, 147 insertions(+), 3 deletions(-) diff --git a/task-2680/.gitignore b/task-2680/.gitignore index 6378a79..b394fe6 100644 --- a/task-2680/.gitignore +++ b/task-2680/.gitignore @@ -4,4 +4,6 @@ bridge-descriptors/ commons-codec-1.4.jar consensuses/ *.tar.bz2 +*.swp +bridge-pool-assignments/ diff --git a/task-2680/ProcessSanitizedAssignments.java b/task-2680/ProcessSanitizedAssignments.java new file mode 100644 index 0000000..9289aa1 --- /dev/null +++ b/task-2680/ProcessSanitizedAssignments.java @@ -0,0 +1,102 @@ +import java.io.*; +import java.util.*; + +public class ProcessSanitizedAssignments { + public static void main(String[] args) throws IOException { + + /* Validate command-line arguments. */ + if (args.length != 1 || !new File(args[0]).exists()) { + System.out.println("Usage: java ProcessSanitizedAssignments <dir>"); + System.exit(1); + } + + /* Find all files that we should parse. Somewhat fragile, but should + * work. */ + System.out.println("Creating list of files we should parse."); + SortedMap<String, File> assignments = new TreeMap<String, File>(); + Stack<File> files = new Stack<File>(); + files.add(new File(args[0])); + while (!files.isEmpty()) { + File file = files.pop(); + if (file.isDirectory()) { + files.addAll(Arrays.asList(file.listFiles())); + } else { + assignments.put(file.getName(), file); + } + } + System.out.println("We found " + assignments.size() + " bridge pool " + + "assignment files."); + + /* Parse assignments. */ + if (!assignments.isEmpty()) { + System.out.println("Parsing bridge pool assignment files."); + BufferedWriter bw = new BufferedWriter(new FileWriter( + "assignments.csv")); + bw.write("assignment,fingerprint,type,ring,port,flag,bucket\n"); + int parsedAssignments = 0, totalAssignments = assignments.size(), + writtenOutputLines = 1; + long started = System.currentTimeMillis(); + for (File file : assignments.values()) { + BufferedReader br = new BufferedReader(new FileReader(file)); + String line, assignmentTime = null; + while ((line = br.readLine()) != null) { + if (line.startsWith("bridge-pool-assignment ")) { + assignmentTime = line.substring("bridge-pool-assignment ". + length()); + } else { + String[] parts = line.split(" "); + String fingerprint = parts[0]; + String type = parts[1]; + String ring = null, port = null, flag = null, bucket = null; + for (int i = 2; i < parts.length; i++) { + String[] parts2 = parts[i].split("="); + String key = parts2[0]; + String value = parts2[1]; + if (key.equals("ring")) { + } else if (key.equals("ring")) { + ring = value; + } else if (key.equals("port")) { + port = value; + } else if (key.equals("flag")) { + flag = value; + } else if (key.equals("bucket")) { + bucket = value; + } else { + System.out.println("Unknown keyword in line '" + line + + "'. Please check. Exiting."); + System.exit(1); + } + } + bw.write(assignmentTime + "," + fingerprint + "," + type + "," + + (ring != null ? ring : "NA") + "," + + (port != null ? port : "NA") + "," + + (flag != null ? flag : "NA") + "," + + (bucket != null ? bucket : "NA") + "\n"); + writtenOutputLines++; + } + } + br.close(); + parsedAssignments++; + if (parsedAssignments % (totalAssignments / 10) == 0) { + double fractionDone = (double) (parsedAssignments) / + (double) totalAssignments; + double fractionLeft = 1.0D - fractionDone; + long now = System.currentTimeMillis(); + double millisLeft = ((double) (now - started)) * fractionLeft / + fractionDone; + long secondsLeft = (long) millisLeft / 1000L; + System.out.println(" " + (parsedAssignments / (totalAssignments + / 10)) + "0% done, " + secondsLeft + " seconds left."); + } + } + bw.close(); + System.out.println("Parsed " + parsedAssignments + " bridge pool " + + "assignment files and wrote " + writtenOutputLines + " lines " + + "to assignments.csv."); + } + + /* This is it. */ + System.out.println("Terminating."); + } +} + diff --git a/task-2680/README b/task-2680/README index a00856f..65d8b85 100644 --- a/task-2680/README +++ b/task-2680/README @@ -6,9 +6,9 @@ This ticket contains Java and R code to This README has a separate section for each Java or R code snippet. -The Java applications produce three output formats containing bridge -descriptors, bridge status lines, and hashed relay identities. The data -formats are described below. +The Java applications produce four output formats containing bridge +descriptors, bridge status lines, bridge pool assignments, and hashed +relay identities. The data formats are described below. -------------------------------------------------------------------------- @@ -33,6 +33,23 @@ ProcessSanitizedBridges.java -------------------------------------------------------------------------- +ProcessSanitizedAssignments.java + + - Download sanitized bridge pool assignments from the metrics website, + e.g., https://metrics.torproject.org/data/bridge-pool-assignments-2011-01.tar.bz2 + and extract them in a local directory, e.g., bridge-pool-assignments/. + + - Compile the Java class, e.g., + $ javac ProcessSanitizedAssignments.java + + - Run the Java class, e.g., + $ java ProcessSanitizedAssignments bridge-pool-assignments/ + + - Once the Java application is done, you'll find a file assignments.csv + in this directory. + +-------------------------------------------------------------------------- + ProcessRelayConsensuses.java - Download v3 relay consensuses from the metrics website, e.g., @@ -130,6 +147,23 @@ The columns in statuses.csv are: -------------------------------------------------------------------------- +assignments.csv + +The assignments.csv file contains one line for every running bridge and +the rings, subrings, and buckets that BridgeDB assigned it to. + +The columns in assignments.csv are: + + - assignment: ISO-formatted bridge pool assignment time + - fingerprint: Hex-formatted SHA-1 hash of identity fingerprint + - type: Name of the distributor: "https", "email", or "unallocated" + - ring: Ring number, only for distributor "https" + - port: Port subring + - flag: Flag subring + - bucket: File bucket, only for distributor "unallocated" + +-------------------------------------------------------------------------- + relays.csv The relays.csv file contains SHA-1 hashes of identity fingerprints of diff --git a/task-2680/verify.R b/task-2680/verify.R index 63ef233..241a196 100644 --- a/task-2680/verify.R +++ b/task-2680/verify.R @@ -25,3 +25,9 @@ if (file.exists("relays.csv")) { summary(as.POSIXct(r$consensus)) } +if (file.exists("assignments.csv")) { + cat("Verifying assignments.csv. This may take a while.\n") + r <- read.csv("assignments.csv", stringsAsFactors = FALSE) + summary(as.POSIXct(r$assignment)) +} +

1 0

[metrics-tasks/master] Check in data tech report w/o bridge assignments.
by karsten＠torproject.org 14 Mar '11

14 Mar '11

commit 071651c75885a21a0a40dde1dfc7808b0c27d645 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Mar 14 12:52:31 2011 +0100 Check in data tech report w/o bridge assignments. --- task-2754/.gitignore | 6 + task-2754/data.bib | 45 ++++ task-2754/data.tex | 717 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 768 insertions(+), 0 deletions(-) diff --git a/task-2754/.gitignore b/task-2754/.gitignore new file mode 100644 index 0000000..e4b915f --- /dev/null +++ b/task-2754/.gitignore @@ -0,0 +1,6 @@ +*.aux +*.bbl +*.blg +*.log +*.pdf + diff --git a/task-2754/data.bib b/task-2754/data.bib new file mode 100644 index 0000000..caec3b8 --- /dev/null +++ b/task-2754/data.bib @@ -0,0 +1,45 @@ +@misc{dirspec, + author = {Roger Dingledine and Nick Mathewson}, + title = {Tor directory protocol, version 3}, + note = {\url{https://gitweb.torproject.org/tor.git/blob_plain/HEAD:/doc/spec/dir-spec.txt}}, +} + +@inproceedings{loesing2009measuring, + title = {Measuring the {Tor} Network from Public Directory Information}, + author = {Karsten Loesing}, + booktitle = {Proc.\ HotPETS}, + year = {2009}, + month = Aug, + address = {Seattle, WA}, + note = {\url{https://metrics.torproject.org/papers/hotpets09.pdf}}, +} + +@inproceedings{loesing2010case, + title = {A Case Study on Measuring Statistical Data in the {Tor} Anonymity Network}, + author = {Karsten Loesing and Steven J. Murdoch and Roger Dingledine}, + booktitle = {Proc.\ Workshop on Ethics in Computer Security Research}, + year = {2010}, + month = Jan, + address = {Tenerife, Canary Islands, Spain}, + note = {\url{https://metrics.torproject.org/papers/wecsr10.pdf}}, +} + +@techreport{hahn2010privacy, + title = {Privacy-preserving Ways to Estimate the Number of Tor Users}, + author = {Sebastian Hahn and Karsten Loesing}, + institution = {The Tor Project}, + year = {2010}, + month = Nov, + type = {Technical Report}, + note = {\url{https://metrics.torproject.org/papers/countingusers-2010-11-30.pdf}}, +} + +@techreport{loesing2009analysis, + title = {Analysis of Circuit Queues in {Tor}}, + author = {Karsten Loesing}, + institution = {The Tor Project}, + year = {2009}, + month = Aug, + note = {\url{https://metrics.torproject.org/papers/bufferstats-2009-08-25.pdf}}, +} + diff --git a/task-2754/data.tex b/task-2754/data.tex new file mode 100644 index 0000000..fe8e322 --- /dev/null +++ b/task-2754/data.tex @@ -0,0 +1,717 @@ +\documentclass{article} +\usepackage{url} +\usepackage[pdftex]{graphicx} +\usepackage{graphics} +\usepackage{color} +\begin{document} +\title{Overview of Statistical Data in the Tor Network} +\author{Karsten Loesing} +\maketitle + +\section{Introduction} + +Statistical analysis in the Tor network can be performed using various +kinds of data. +In this report we give an overview of three major data sources for +statistics in the Tor network: +First, we recap measuring the Tor network from public directory +information \cite{loesing2009measuring} in Section~\ref{sec:serverdesc} +and explain the sanitzation process of (non-public) bridge directory +information in Section~\ref{sec:bridgesan}. +Second, we describe the numerous aggregate statistics that relays publish +about their usage \cite{loesing2010case} +in Sections~\ref{sec:bytehist} to \ref{fig:connbidirect}. +Third, we delineate the output of various Tor services like GetTor or +Tor Check as well as specific measurement tools like Torperf in +Sections~\ref{sec:torperf} to \ref{sec:exitlist}. +All data described in this report are available for download on the +metrics +website.\footnote{\texttt{https://metrics.torproject.org/data.html}} + +\section{Server descriptors and network statuses} +\label{sec:serverdesc} + +Relays in the Tor network report their capabilities by publishing server +descriptors to the directory authorities. +The directory authorities confirm reachability of relays and assign flags +to help clients make good path selections. +Every hour, the directory authorities publish a network status consensus +with all known running relays at the time. +Both server descriptors and network statuses constitute a solid data basis +for statistical analysis in the Tor network. +We described the approach to measure the Tor network from public directory +information in \cite{loesing2009measuring} and provide interactive +graphs on the metrics +website.\footnote{\texttt{https://metrics.torproject.org/graphs.html}} +In this section, we briefly describe the most interesting pieces of the +two descriptor formats that can be used for statistics. + +\paragraph{Server descriptors} + +The server descriptors published by relays at least once every 18 hours +contain the necessary information for clients to build circuits using a +given relay. +These server descriptors can also be useful for statistical analysis of +the Tor network infrastructure. + +We assume that the majority of server descriptors are correct. +But when performing statistical analysis on server descriptors, one has to +keep in mind that only a small subset of the information written to server +descriptors is confirmed by the trusted directory authorities. +In theory, relays can provide false information in their server +descriptors, even though the incentive to do so is probably low. + +Figure~\ref{fig:serverdesc} shows an example server descriptor. +The following data fields in server descriptors may be relevant to +statistical analysis: + +\begin{figure} +\begin{verbatim} +router blutmagie 192.251.226.206 443 0 80 +platform Tor 0.2.2.20-alpha on Linux x86_64 +opt protocols Link 1 2 Circuit 1 +published 2010-12-27 14:35:27 +opt fingerprint 6297 B13A 687B 521A 59C6 BD79 188A 2501 EC03 A065 +uptime 445412 +bandwidth 14336000 18432000 15905178 +opt extra-info-digest 5C1D5D6F8B243304079BC15CD96C7FCCB88322D4 +opt caches-extra-info +onion-key +[...] +signing-key +[...] +family $66CA87E164F1CFCE8C3BB5C095217A28578B8BAF + $67EC84376D9C4C467DCE8621AACA109160B5264E + $7B698D327F1695590408FED95CDEE1565774D136 +opt hidden-service-dir +contact abuse(a)blutmagie.de +reject 0.0.0.0/8:* +reject 169.254.0.0/16:* +reject 127.0.0.0/8:* +reject 192.168.0.0/16:* +reject 10.0.0.0/8:* +reject 172.16.0.0/12:* +reject 192.251.226.206:* +reject *:25 +reject *:119 +reject *:135-139 +reject *:445 +reject *:465 +reject *:563 +reject *:587 +reject *:1214 +reject *:4661-4666 +reject *:6346-6429 +reject *:6660-6999 +accept *:* +router-signature +[...] +\end{verbatim} +\vspace{-1em} +\caption{Server descriptor published by relay \texttt{blugmagie} (without +cryptographic keys and hashes)} +\label{fig:serverdesc} +%---------------------------------------------------------------- +\end{figure} + +\begin{itemize} +\item \textit{IP address and ports:} Relays provide their IP address +and ports where they accept requests to build circuits and directory +requests. +These data fields are contained in the first line of a server descriptor +starting with \verb+router+. +Note that in rare cases, the IP address provided here can be different +from the IP address used for exiting to the Internet. +The latter can be found in the exit lists produced by Tor Check as +described in Section~\ref{sec:exitlist}. +\item \textit{Operating system and Tor software version:} Relays include +their operating system and Tor software version in their server +descriptors in the \verb+platform+ line. +While this information is very likely correct in most cases, a few relay +operators may try to impede hacking attempts by providing false platform +strings. +\item \textit{Uptime:} Relays include the number of seconds since the +last restart in their server descriptor in the \verb+uptime+ line. +\item \textit{Own measured bandwidth:} Relays report the bandwidth that +they are willing to provide on average and for short periods of time. +Relays also perform periodic bandwidth self-tests and report their actual +available bandwidth. +The latter was used by clients to weight relays in the path selection +algorithm and was sometimes subject to manipulation by malicious relays. +All three bandwidth values can be found in a server descriptor's +\verb+bandwidth+ line. +With the introduction of bandwidth scanners, the self-reported relay +bandwidth in server descriptors has become less +relevant.\footnote{\url{http://gitweb.torproject.org/torflow.git/}} +\item \textit{Relay family:} Some relay operators who run more than one +relay organize their relays in relay families, so that clients don't pick +more than one of these relays for a single circuit. +Each relay belonging to a relay family lists the members of that family +either by nickname or fingerprint in its server descriptor in the +\verb+family+ line. +\item \textit{Exit policy:} Relays define their exit policy by including +firewall-like rules which outgoing connections they reject or accept in +the \verb+reject+ and \verb+accept+ lines. +\end{itemize} + +These are just a subset of the fields in a server descriptor that seem +relevant for statistical analysis. +For a complete list of fields in server descriptors, see the directory +procol specification \cite{dirspec}. + +\paragraph{Network statuses} + +Every hour, the directory authorities publish a new network status that +contains a list of all running relays. +The directory authorities confirm reachability of the contained relays and +assign flags based on the relays' characteristics. +The entries in a network status reference the last published server +descriptor of a relay. + +The network statuses are relevant for statistical analysis, because they +constitute trusted snapshots of the Tor network. +Anyone can publish as many server descriptors as they want, but only the +directory authorities can confirm that a relay was running at a given +time. +Most statistics on the Tor network infrastructure rely on network statuses +and possibly combine them with the referenced server descriptors. +Figure~\ref{fig:statusentry} shows the network status entry referencing +the server descriptor from Figure~\ref{fig:serverdesc}. +In addition to the reachability information, network statuses contain the +following fields that may be relevant for statistical analysis: + +\begin{figure} +\begin{verbatim} +r blutmagie YpexOmh7UhpZxr15GIolAewDoGU + lFY7WmD/yvVFp9drmZzNeTxZ6dw 2010-12-27 14:35:27 192.251.226.206 + 443 80 +s Exit Fast Guard HSDir Named Running Stable V2Dir Valid +v Tor 0.2.2.20-alpha +w Bandwidth=30800 +p reject 25,119,135-139,445,465,563,587,1214,4661-4666,6346-6429, + 6660-6999 +\end{verbatim} +%---------------------------------------------------------------- +\vspace{-1em} +\caption{Network status entry of relay \texttt{blutmagie}} +\label{fig:statusentry} +\end{figure} + +\begin{itemize} +\item \textit{Relay flags:} The directory authorities assign flags to +relays based on their characteristics to the line starting with \verb+s+. +Examples are the \verb+Exit+ flag if a relay permits exiting to the +Internet and the \verb+Guard+ flag if a relay is stable enough to be +picked as guard node +\item \textit{Relay version:} The directory authorities include the +version part of the platform string written to server descriptors in the +network status in the line starting with \verb+v+. +\item \textit{Bandwidth weights:} The network status contains a bandwidth +weight for every relay in the lines with \verb+w+ that clients shall use +for weighting relays in their path selection algorithm. +This bandwidth weight is either the self-reported bandwidth of the relay +or the bandwidth measured by the bandwidth scanners. +\item \textit{Exit policy summary:} Every entry in a network status +contains a summary version of a relay's exit policy in the line starting +with \verb+p+. +This summary is a list of accepted or rejected ports for exit to most IP +addresses. +\end{itemize} + +\section{Sanitized bridge descriptors} +\label{sec:bridgesan} + +Bridges in the Tor network publish server descriptors to the bridge +authority which in turn generates a bridge network status. +We cannot, however, make the bridge server descriptors and bridge network +statuses available for statistical analysis as we do with the relay server +descriptors and relay network statuses. +The problem is that bridge server descriptors and network statuses contain +bridge IP addresses and other sensitive information that shall not be made +publicly available. +We therefore sanitize bridge descriptors by removing all potentially +identifying information and publish sanitized versions of the descriptors. +The processing steps for sanitizing bridge descriptors are as follows: + +\begin{enumerate} +\item \textit{Replace the bridge identity with its SHA1 value:} Clients +can request a bridge's current descriptor by sending its identity string +to the bridge authority. +This is a feature to make bridges on dynamic IP addresses useful. +Therefore, the original identities (and anything that could be used to +derive them) need to be removed from the descriptors. +The bridge identity is replaced with its SHA1 hash value. +The idea is to have a consistent replacement that remains stable over +months or even years (without keeping a secret for a keyed hash function). +\item \textit{Remove all cryptographic keys and signatures:} It would be +straightforward to learn about the bridge identity from the bridge's +public key. +Replacing keys by newly generated ones seemed to be unnecessary (and would +involve keeping a state over months/years), so that all cryptographic +objects have simply been removed. +\item \textit{Replace IP address with IP address hash:} Of course, the IP +address needs to be removed, too. +It is replaced with \verb+10.x.x.x+ with \verb+x.x.x+ being the 3 byte +output of \verb+H(IP address | bridge identity | secret)[:3]+. +The input \verb+IP address+ is the 4-byte long binary representation of +the bridge's current IP address. +The \verb+bridge identity+ is the 20-byte long binary representation of +the bridge's long-term identity fingerprint. +The \verb+secret+ is a 31-byte long secure random string that changes once +per month for all descriptors and statuses published in that month. +\verb+H()+ is SHA-256. +The \verb+[:3]+ operator means that we pick the 3 most significant bytes +of the result. +\item \textit{Replace contact information:} If there is contact +information in a descriptor, the contact line is changed to +\verb+somebody+. +\item \textit{Replace nickname with Unnamed:} The bridge nicknames might +give hints on the location of the bridge if chosen without care; e.g.\ a +bridge nickname might be very similar to the operators' relay nicknames +which might be located on adjacent IP addresses. +All bridge nicknames are therefore replaced with the string +\verb+Unnamed+. +\end{enumerate} + +Figure~\ref{fig:bridgeserverdesc} shows an example bridge server +descriptor that is referenced from the bridge network status entry in +Figure~\ref{fig:bridgestatusentry}. +For more details about this process, see the bridge descriptor sanitizer +and the metrics database +software.\footnote{\texttt{https://metrics.torproject.org/tools.html}} + +\begin{figure} +\begin{verbatim} +router Unnamed 10.74.150.129 443 0 0 +platform Tor 0.2.2.19-alpha (git-1988927edecce4c7) on Linux i686 +opt protocols Link 1 2 Circuit 1 +published 2010-12-27 18:55:01 +opt fingerprint A5FA 7F38 B02A 415E 72FE 614C 64A1 E5A9 2BA9 9BBD +uptime 2347112 +bandwidth 5242880 10485760 1016594 +opt extra-info-digest 86E6E9E68707AF586FFD09A36FAC236ADA0D11CC +opt hidden-service-dir +contact somebody +reject *:* +\end{verbatim} +\vspace{-1em} +\caption{Sanitized bridge server descriptor} +\label{fig:bridgeserverdesc} +%---------------------------------------------------------------- +\end{figure} + +\begin{figure} +\begin{verbatim} + +r Unnamed pfp/OLAqQV5y/mFMZKHlqSupm70 dByzfWWLas9cen7PtZ3XGYIJHt4 + 2010-12-27 18:55:01 10.74.150.129 443 0 +s Fast Guard HSDir Running Stable Valid +\end{verbatim} +\vspace{-1em} +\caption{Sanitized bridge network status entry} +\label{fig:bridgestatusentry} +%---------------------------------------------------------------- +\end{figure} + +\section{Byte histories} +\label{sec:bytehist} + +Relays include aggregate statistics in their descriptors that they upload +to the directory authorities. +These aggregate statistics are contained in extra-info descriptors that +are published in companion with server descriptors. +Extra-info descriptors are not required for clients to build circuits. +An extra-info descriptor belonging to a server descriptor is referenced by +its SHA1 hash value. + +Byte histories were the first statistical data that relays published about +their usage. +Relays report the number of written and read bytes in 15-minute intervals +throughout the last 24 hours. +The extra-info descriptor in Figure~\ref{fig:extrainfo} contains the byte +histories in the two lines starting with \verb+write-history+ and +\verb+read-history+. +More details about these statistics can be found in the directory protocol +specification~\cite{dirspec}. + +\begin{figure} +\begin{verbatim} +extra-info blutmagie 6297B13A687B521A59C6BD79188A2501EC03A065 +published 2010-12-27 14:35:27 +write-history 2010-12-27 14:34:05 (900 s) 12902389760, + 12902402048,12859373568,12894131200,[...] +read-history 2010-12-27 14:34:05 (900 s) 12770249728,12833485824, + 12661140480,12872439808,[...] +dirreq-write-history 2010-12-27 14:26:13 (900 s) 51731456, + 60808192,56740864,54948864,[...] +dirreq-read-history 2010-12-27 14:26:13 (900 s) 4747264,4767744, + 4511744,4752384,[...] +dirreq-stats-end 2010-12-27 10:51:09 (86400 s) +dirreq-v3-ips us=2000,de=1344,fr=744,kr=712,[...] +dirreq-v2-ips ??=8,au=8,cn=8,cz=8,[...] +dirreq-v3-reqs us=2368,de=1680,kr=1048,fr=800,[...] +dirreq-v2-reqs id=48,??=8,au=8,cn=8,[...] +dirreq-v3-resp ok=12504,not-enough-sigs=0,unavailable=0, + not-found=0,not-modified=0,busy=128 +dirreq-v2-resp ok=64,unavailable=0,not-found=8,not-modified=0, + busy=8 +dirreq-v2-share 1.03% +dirreq-v3-share 1.03% +dirreq-v3-direct-dl complete=316,timeout=4,running=0,min=4649, + d1=36436,d2=68056,q1=76600,d3=87891,d4=131294,md=173579, + d6=229695,d7=294528,q3=332053,d8=376301,d9=530252,max=2129698 +dirreq-v2-direct-dl complete=16,timeout=52,running=0,min=9769, + d1=9769,d2=9844,q1=9981,d3=9981,d4=27297,md=33640,d6=60814, + d7=205884,q3=205884,d8=361137,d9=628256,max=956009 +dirreq-v3-tunneled-dl complete=12088,timeout=92,running=4, + min=534,d1=31351,d2=49166,q1=58490,d3=70774,d4=88192,md=109778, + d6=152389,d7=203435,q3=246377,d8=323837,d9=559237,max=26601000 +dirreq-v2-tunneled-dl complete=0,timeout=0,running=0 +entry-stats-end 2010-12-27 10:51:09 (86400 s) +entry-ips de=11024,us=10672,ir=5936,fr=5040,[...] +exit-stats-end 2010-12-27 10:51:09 (86400 s) +exit-kibibytes-written 80=6758009,443=498987,4000=227483, + 5004=1182656,11000=22767,19371=1428809,31551=8212,41500=965584, + 51413=3772428,56424=1912605,other=175227777 +exit-kibibytes-read 80=197075167,443=5954607,4000=1660990, + 5004=1808563,11000=1893893,19371=130360,31551=7588414, + 41500=756287,51413=2994144,56424=1646509,other=288412366 +exit-streams-opened 80=5095484,443=359256,4000=4508,5004=22288, + 11000=124,19371=24,31551=40,41500=96,51413=16840,56424=28, + other=1970964 +\end{verbatim} +%---------------------------------------------------------------- +\vspace{-1em} +\caption{Extra-info descriptor published by relay \texttt{blutmagie} +(without cryptographic signature and with long lines being truncated)} +\label{fig:extrainfo} +\end{figure} + +\section{Directory requests} + +The directory authorities and directory mirrors report statistical data +about processed directory requests. +Starting with Tor version 0.2.2.15-alpha, all directories report the +number of written and read bytes for answering directory requests. +The format is similar to the format of byte histories as described in the +previous section. +The relevant lines are \verb+dirreq-write-history+ and +\verb+dirreq-read-history+ in Figure~\ref{fig:extrainfo}. +These two lines contain the subset of total read and written bytes that +the directory mirror spent on responding to any kind of directory request, +including network statuses, server descriptors, extra-info descriptors, +authority certificates, etc. + +The directories further report statistics on answering directory requests +for network statuses only. +For Tor versions before 0.2.3.x, relay operators had to manually enable +these statistics, which is why only a few directories report them. +The lines starting with \verb+dirreq-v3-+ all belong to the directory +request statistics (the lines starting with \verb+dirreq-v2-+ report +similar statistics for version 2 of the directory protocol which is +deprecated at the time of writing this report). +The following fields may be relevant for statistical analysis: + +\begin{itemize} +\item \textit{Unique IP addresses:} The numbers in \verb+dirreq-v3-ips+ +denote the unique IP addresses of clients requesting network statuses by +country. +\item \textit{Network status requests:} The numbers in +\verb+dirreq-v3-reqs+ constitute the total network status requests by +country. +\item \textit{Request share:} The percentage in \verb+dirreq-v3-share+ is +an estimate of the share of directory requests that the reporting relay +expects to see in the Tor network. +In \cite{hahn2010privacy} we found that this estimate isn't very useful +for statistical analysis because of the different approaches that clients +take to select directory mirrors. +The fraction of written directory bytes (\verb+dirreq-write-history+) can +be used to derive a better metric for the share of directory requests. +\item \textit{Network status responses:} The directories also report +whether they could provide the requested network status to clients in +\verb+dirreq-v3-resp+. +This information was mostly used to diagnose error rates in version 2 of +the directory protocol where a lot of directories replied to network +status requests with \verb+503 Busy+. +In version 3 of the directory protocol, most responses contain the status +code \verb+200 OK+. +\item \textit{Network status download times:} The line +\verb+dirreq-v3-direct-dl+ contains statistics on the download of network +statuses via the relay's directory port. +The line \verb+dirreq-v3-tunneled-dl+ contains similar statistics on +downloads via a 1-hop circuit between client and directory (which is the +common approach in version 3 of the directory protocol). +Relays report how many requests have been completed, have timed out, and +are still running at the end of a 24-hour time interval as well as the +minimum, maximum, median, quartiles, and deciles of download times. +\end{itemize} +More details about these statistics can be found in the directory protocol +specification~\cite{dirspec}. + +\section{Connecting clients} + +Relays can be configured to report per-country statistics on directly +connecting clients. +This metric includes clients connecting to a relay in order to build +circuits and clients creating a 1-hop circuit to request directory +information. +In practice, the latter number outweighs the former number. +The \verb+entry-ips+ line in Figure~\ref{fig:extrainfo} shows the number +of unique IP addresses connecting to the relay by country. +More details about these statistics can be found in the directory protocol +specification~\cite{dirspec}. + +\section{Bridge users} + +Bridges report statistics on connecting bridge clients in their extra-info +descriptors. +Figure~\ref{fig:bridgeextrainfo} shows a bridge extra-info descriptor +with the bridge user statistics in the \verb+bridge-ips+ line. + +\begin{figure} +\begin{verbatim} +extra-info Unnamed A5FA7F38B02A415E72FE614C64A1E5A92BA99BBD +published 2010-12-27 18:55:01 +write-history 2010-12-27 18:43:50 (900 s) 151712768,176698368, + 180030464,163150848,[...] +read-history 2010-12-27 18:43:50 (900 s) 148109312,172274688, + 172168192,161094656,[...] +bridge-stats-end 2010-12-27 14:56:29 (86400 s) +bridge-ips sa=48,us=40,de=32,ir=32,[...] +\end{verbatim} +\vspace{-1em} +\caption{Sanitized bridge extra-info descriptor} +%---------------------------------------------------------------- +\label{fig:bridgeextrainfo} +\end{figure} + +Bridges running Tor version 0.2.2.3-alpha or earlier report bridge users +in a similar line starting with \verb+geoip-client-origins+. +The reason for switching to \verb+bridge-ips+ was that the measurement +interval in \verb+geoip-client-origins+ had a variable length, whereas the +measurement interval in 0.2.2.4-alpha and later is set to exactly +24~hours. +In order to clearly distinguish the new measurement intervals from the old +ones, the new keywords have been introduced. +More details about these statistics can be found in the directory protocol +specification~\cite{dirspec}. + +\section{Cell-queue statistics} + +Relays can be configured to report aggregate statistics on their cell +queues. +These statistics include average processed cells, average number of queued +cells, and average time that cells spend in circuits. +Circuits are split into deciles based on the number of processed cells. +The statistics are provided for circuit deciles from loudest to quietest +circuits. +Figure~\ref{fig:cellstats} shows the cell statistics contained in an +extra-info descriptor by relay \texttt{gabelmoo}. +An early analysis of cell-queue statistics can be found in +\cite{loesing2009analysis}. +More details about these statistics can be found in the directory protocol +specification~\cite{dirspec}. + +\begin{figure} +\begin{verbatim} +cell-stats-end 2010-12-27 09:59:50 (86400 s) +cell-processed-cells 4563,153,42,15,7,7,6,5,4,2 +cell-queued-cells 9.39,0.98,0.09,0.01,0.00,0.00,0.00,0.01,0.00, + 0.01 +cell-time-in-queue 2248,807,277,92,49,22,52,55,81,148 +cell-circuits-per-decile 7233 +\end{verbatim} +%---------------------------------------------------------------- +\vspace{-1em} +\caption{Cell statistics in extra-info descriptor by relay +\texttt{gabelmoo}} +\label{fig:cellstats} +\end{figure} + +\section{Exit-port statistics} + +Exit relays running Tor version 0.2.1.1-alpha or higher can be configured +to report aggregate statistics on exiting connections. +These relays report the number of opened streams, written and read bytes +by exiting port. +Until version 0.2.2.19-alpha, relays reported all ports exceeding a +threshold of 0.01~\% of all written and read exit bytes. +Starting with version 0.2.2.20-alpha, relays only report the top 10 ports +in exit-port statistics in order not to exceed the maximum extra-info +descriptor length of 50 KB. +Figure~\ref{fig:extrainfo} on page \pageref{fig:extrainfo} contains +exit-port statistics in the lines starting with \verb+exit-+. +More details about these statistics can be found in the directory protocol +specification~\cite{dirspec}. + +\section{Bidirectional connection use} +\label{sec:bidistats} + +Relays running Tor version 0.2.3.1-alpha or higher can be configured to +report what fraction of connections is used uni- or bi-directionally. +Every 10 seconds, relays determine for every connection whether they read +and wrote less than a threshold of 20 KiB. +Connections below this threshold are labeled as ``Below Threshold''. +For the remaining connections, relays report whether they read/wrote at +least 10 times as many bytes as they wrote/read. +If so, they classify a connection as ``Mostly reading'' or ``Mostly +writing,'' respectively. +All other connections are classified as ``Both reading and writing.'' +After classifying connections, read and write counters are reset for the +next 10-second interval. +Statistics are aggregated over 24 hours. +Figure~\ref{fig:connbidirect} shows the bidirectional connection use +statistics in an extra-info descriptor by relay \texttt{zweifaltigkeit}. +The four numbers denote the number of connections ``Below threshold,'' +``Mostly reading,'' ``Mostly writing,'' and ``Both reading and writing.'' +More details about these statistics can be found in the directory protocol +specification~\cite{dirspec}. + +\begin{figure} +\begin{verbatim} +conn-bi-direct 2010-12-28 15:55:11 (86400 s) 387465,45285,55361, + 81786 +\end{verbatim} +%---------------------------------------------------------------- +\vspace{-1em} +\caption{Bidirectional connection use statistic in extra-info descriptor +by relay \texttt{zweifaltigkeit}} +\label{fig:connbidirect} +\end{figure} + +\section{Torperf output files} +\label{sec:torperf} + +Torperf is a little tool that measures Tor's performance as users +experience it. +Torperf uses a trivial SOCKS client to download files of various sizes +over the Tor network and notes how long substeps take. +Torperf can be downloaded from the metrics +website.\footnote{\texttt{https://metrics.torproject.org/tools.html}} + +Torperf can produce two output files: \verb+.data+ and \verb+.extradata+. +The \verb+.data+ file contains timestamps for nine substeps and the byte +summaries for downloading a test file via Tor. +Figure~\ref{fig:torperf} shows an example output of a Torperf run. +The timestamps in the upper part of this output are seconds and +nanoseconds since 1970-01-01 00:00:00.000000. + +Torperf can be configured to write \verb+.extradata+ files by attaching +a Tor controller and writing certain controller events to disk. +The content of a \verb+.extradata+ line is shown in the lower part of +Figure~\ref{fig:torperf}. +The first column indicates if this circuit was actually used to fetch +the data (\verb+ok+) or if Tor chose a different circuit because this +circuit was problematic (\verb+error+). +For every \verb+error+ entry there should be a following \verb+ok+ entry, +unless the network of the Torperf instance is dead or the resource is +unavailable. +The circuit build completion time in the \verb+.extradata+ line is the +time between Torperf sent a SOCKS request and received a SOCKS response in +the \verb+.data+ file. +The three or more hops of the circuit are listed by relay fingerprint and +nickname. +An \verb+=+ sign between the two means that a relay has the \texttt{Named} +flag, whereas the \verb+~+ sign means it doesn't. + +\begin{figure} +\begin{verbatim} +# Timestamps and byte summaries contained in .data files: +1293543301 762678 # Connection process started +1293543301 762704 # After socket is created +1293543301 763074 # After socket is connected +1293543301 763190 # After authentication methods are negotiated + # (SOCKS 5 only) +1293543301 763816 # After SOCKS request is sent +1293543302 901783 # After SOCKS response is received +1293543302 901818 # After HTTP request is written +1293543304 445732 # After first response is received +1293543305 456664 # After payload is complete +75 # Written bytes +51442 # Read bytes + +# Path information contained in .extradata files: +ok # Status code +1293543302 # Circuit build completion time +$2F265B37920BDFE474BF795739978EEFA4427510=fejk4 # 1st hop +$66CA87E164F1CFCE8C3BB5C095217A28578B8BAF=blutmagie3 # 2nd hop +$76997E6557828E8E57F70FDFBD93FB3AA470C620~Amunet8 # 3rd hop +\end{verbatim} +%---------------------------------------------------------------- +\vspace{-1em} +\caption{Torperf output lines for a single request to download a 50~KiB +file (reformatted and annotated with comments)} +\label{fig:torperf} +\end{figure} + +\section{GetTor statistics file} + +GetTor allows users to fetch the Tor software via email. +GetTor keeps internal statistics on the number of packages requested +every day and writes these statistics to a file. +Figure~\ref{fig:gettor} shows the statistics file for December 27, 2010. +The \verb+None+ entry stands for requests that don't ask for a specific +bundle, e.g.\ requests for the bundle list. + +\begin{figure} +\begin{verbatim} +2010-12-27 - None:167 macosx-i386-bundle:0 macosx-ppc-bundle:0 + source-bundle:2 tor-browser-bundle:0 tor-browser-bundle_ar:0 + tor-browser-bundle_de:0 tor-browser-bundle_en:39 + tor-browser-bundle_es:0 tor-browser-bundle_fa:5 + tor-browser-bundle_fr:0 tor-browser-bundle_it:0 + tor-browser-bundle_nl:0 tor-browser-bundle_pl:0 + tor-browser-bundle_pt:0 tor-browser-bundle_ru:0 + tor-browser-bundle_zh_CN:77 tor-im-browser-bundle:0 + tor-im-browser-bundle_ar:0 tor-im-browser-bundle_de:0 + tor-im-browser-bundle_en:1 tor-im-browser-bundle_es:0 + tor-im-browser-bundle_fa:0 tor-im-browser-bundle_fr:0 + tor-im-browser-bundle_it:0 tor-im-browser-bundle_nl:0 + tor-im-browser-bundle_pl:0 tor-im-browser-bundle_pt:0 + tor-im-browser-bundle_ru:0 tor-im-browser-bundle_zh_CN:0 +\end{verbatim} +%---------------------------------------------------------------- +\vspace{-1em} +\caption{GetTor statistics file for December 27, 2010} +\label{fig:gettor} +\end{figure} + +\section{Tor Check exit lists} +\label{sec:exitlist} + +TorDNSEL is an implementation of the active testing, DNS-based exit list +for Tor exit +nodes.\footnote{\texttt{https://www.torproject.org/tordnsel/dist/}} +Tor Check makes the list of known exits and corresponding exit IP +addresses available in a specific format. +Figure~\ref{fig:exitlist} shows an entry of the exit list written on +December 28, 2010 at 15:21:44 UTC. +This entry means that the relay with fingerprint \verb+63Ba..+ which +published a descriptor at 07:35:55 and was contained in a version 2 +network status from 08:10:11 uses two different IP addresses for exiting. +The first address \verb+91.102.152.236+ was found in a test performed at +07:10:30. +When looking at the corresponding server descriptor, one finds that this +is also the IP address on which the relay accepts connections from inside +the Tor network. +A second test performed at 10:35:30 reveals that the relay also uses IP +address \verb+91.102.152.227+ for exiting. + +\begin{figure} +\begin{verbatim} +ExitNode 63BA28370F543D175173E414D5450590D73E22DC +Published 2010-12-28 07:35:55 +LastStatus 2010-12-28 08:10:11 +ExitAddress 91.102.152.236 2010-12-28 07:10:30 +ExitAddress 91.102.152.227 2010-12-28 10:35:30 +\end{verbatim} +\vspace{-1em} +\caption{Exit list entry written on December 28, 2010 at 15:21:44 UTC} +\label{fig:exitlist} +\end{figure} + +\bibliography{data} +\bibliographystyle{plain} + +\end{document} +

1 0

[metrics-web/master] Add bridge pool assignments to metrics website.
by karsten＠torproject.org 14 Mar '11

14 Mar '11

commit 117a75e2ccc2f4a7215619367a8ab154e47788d1 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Mar 14 12:25:48 2011 +0100 Add bridge pool assignments to metrics website. --- .../torproject/ernie/web/ResearchDataServlet.java | 19 +++++++++++++++++ web/WEB-INF/data.jsp | 22 ++++++++++++++++++++ web/WEB-INF/index.jsp | 15 +++++-------- 3 files changed, 47 insertions(+), 9 deletions(-) diff --git a/src/org/torproject/ernie/web/ResearchDataServlet.java b/src/org/torproject/ernie/web/ResearchDataServlet.java index 2d45c0f..ca6bf7b 100644 --- a/src/org/torproject/ernie/web/ResearchDataServlet.java +++ b/src/org/torproject/ernie/web/ResearchDataServlet.java @@ -75,6 +75,8 @@ public class ResearchDataServlet extends HttpServlet { SortedMap<String, Map<String, String[]>> torperfData = new TreeMap<String, Map<String, String[]>>(); SortedMap<Date, String[]> exitLists = new TreeMap<Date, String[]>(); + SortedMap<Date, String[]> bridgePoolAssignments = + new TreeMap<Date, String[]>(); /* Prepare rewriting Torperf sources. */ Map<String, String> torperfSources = new HashMap<String, String>(); @@ -207,6 +209,22 @@ public class ResearchDataServlet extends HttpServlet { exitLists.put(month, new String[2]); } exitLists.get(month)[0] = url; + + /* URL contains bridge pool assignments. */ + } else if (filename.startsWith("bridge-pool-assignments-20")) { + String yearMonth = filename.substring(filename.indexOf("20")); + yearMonth = yearMonth.substring(0, 7); + Date month = null; + try { + month = monthFormat.parse(yearMonth); + } catch (ParseException e) { + /* Ignore this URL. */ + continue; + } + if (!bridgePoolAssignments.containsKey(month)) { + bridgePoolAssignments.put(month, new String[2]); + } + bridgePoolAssignments.get(month)[0] = url; } } @@ -217,6 +235,7 @@ public class ResearchDataServlet extends HttpServlet { request.setAttribute("relayStatistics", relayStatistics); request.setAttribute("torperfData", torperfData); request.setAttribute("exitLists", exitLists); + request.setAttribute("bridgePoolAssignments", bridgePoolAssignments); request.getRequestDispatcher("WEB-INF/data.jsp").forward(request, response); } diff --git a/web/WEB-INF/data.jsp b/web/WEB-INF/data.jsp index 86028c2..30d42f6 100644 --- a/web/WEB-INF/data.jsp +++ b/web/WEB-INF/data.jsp @@ -1,5 +1,6 @@ <%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core" %> <%@ taglib prefix="fmt" uri="http://java.sun.com/jsp/jstl/fmt" %> +<fmt:setLocale value="en_US"/> <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> <html> <head> @@ -26,6 +27,7 @@ <ul> <li><a href="#relaydesc">Relay descriptor archives</a></li> <li><a href="#bridgedesc">Bridge descriptor archives</a></li> + <li><a href="#bridgeassignments">Bridge pool assignments</a></li> <li><a href="#stats">Statistics produced by relays</a></li> <li><a href="#performance">Performance data</a></li> <li><a href="#exitlist">Exit lists</a></li> @@ -124,6 +126,26 @@ </table> <p></p> <br> + <a name="bridgeassignments"></a> + <h3>Bridge pool assignments</h3> + <br> + <p>BridgeDB periodically dumps the list of running bridges with + information about the rings, subrings, and file buckets to which + they are assigned to a local file. We are archiving sanitized + versions of these files here to analyze how the pool assignment + affects a bridge's usage.</p> + <table width="100%" border="0" cellpadding="5" cellspacing="0" summary=""> + <c:forEach var="item" items="${bridgePoolAssignments}" > + <fmt:formatDate var="longDate" pattern="MMMM yyyy" + value="${item.key}"/> + <tr> + <td> + <a href="${item.value[0]}">${longDate}</a> + </td> + </tr> + </c:forEach> + </table> + <br> <a name="stats"></a> <h3>Statistics produced by relays</h3> <br> diff --git a/web/WEB-INF/index.jsp b/web/WEB-INF/index.jsp index f82a7b3..0da160b 100644 --- a/web/WEB-INF/index.jsp +++ b/web/WEB-INF/index.jsp @@ -32,6 +32,11 @@ <br> <h3>News</h3> <ul> + <li>March 14, 2011: + <a href="data.html#bridgeassignments">Sanitized bridge pool + assignments</a> containing lists of running bridges with + information about the rings, subrings, and file buckets to which + they are assigned are now available.</li> <li>January 27, 2011: New <a href="performance.html">Torperf</a> graphs combining the download times of all sources and showing the fraction of timeouts and failures are now available.</li> @@ -45,15 +50,7 @@ <li>November 30, 2010: Tech report on <a href="papers/countingusers-2010-11-30.pdf">Privacy-preserving Ways to Estimate the Number of Tor Users</a> is available for - download on the <a href="papers.html">Papers</a> page. - <li>October 7, 2010: Custom graphs are now available for all - <a href="graphs.html">graphs</a>. Based on work by Kevin - Berry.</li> - <li>September 9, 2010: Custom - graphs on network size, relay platforms, versions, and - observed bandwidth available. Implemented by Kevin Berry.</li> - <li>September 2, 2010: New <a href="relay-search.html">relay - search</a> feature available.</li> + download on the <a href="papers.html">Papers</a> page.</li> </ul> </div> </div>

1 0

[metrics-db/master] Sanitize bridge pool assignments.
by karsten＠torproject.org 14 Mar '11

14 Mar '11

commit 01d8d919512d8dfe3df3440e5dec0bccb2942baa Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Mar 14 10:54:59 2011 +0100 Sanitize bridge pool assignments. --- config.template | 11 ++ .../ernie/db/BridgePoolAssignmentsProcessor.java | 128 ++++++++++++++++++++ src/org/torproject/ernie/db/Configuration.java | 23 +++- src/org/torproject/ernie/db/Main.java | 7 + 4 files changed, 167 insertions(+), 2 deletions(-) diff --git a/config.template b/config.template index 0d6743e..47a8c8d 100644 --- a/config.template +++ b/config.template @@ -44,6 +44,13 @@ ## Download exit list and store it to disk #DownloadExitList 0 # +## Process bridge pool assignment files by sanitizing bridge fingerprints +## and sorting sanitized files into subdirectories +#ProcessBridgePoolAssignments 0 +# +## Relative path to directory to read bridge pool assignment files from +#AssignmentsDirectory assignments/ +# #### Data sinks #### # ## Write directory archives to disk @@ -66,4 +73,8 @@ # ## Relative path to directory to write sanitized bridges to #SanitizedBridgesWriteDirectory sanitized-bridges/ +# +## Relative path to directory to write sanitized bridge pool assignment +## files to +#SanitizedAssignmentsDirectory sanitized-assignments/ diff --git a/src/org/torproject/ernie/db/BridgePoolAssignmentsProcessor.java b/src/org/torproject/ernie/db/BridgePoolAssignmentsProcessor.java new file mode 100644 index 0000000..583d36e --- /dev/null +++ b/src/org/torproject/ernie/db/BridgePoolAssignmentsProcessor.java @@ -0,0 +1,128 @@ +/* Copyright 2011 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.ernie.db; + +import java.io.*; +import java.text.*; +import java.util.*; +import java.util.logging.*; +import org.apache.commons.codec.*; +import org.apache.commons.codec.binary.*; +import org.apache.commons.codec.digest.*; + +public class BridgePoolAssignmentsProcessor { + + public BridgePoolAssignmentsProcessor(File assignmentsDirectory, + File sanitizedAssignmentsDirectory) { + + Logger logger = + Logger.getLogger(BridgePoolAssignmentsProcessor.class.getName()); + if (assignmentsDirectory == null || + sanitizedAssignmentsDirectory == null) { + IllegalArgumentException e = new IllegalArgumentException("Neither " + + "assignmentsDirectory nor sanitizedAssignmentsDirectory may " + + "be null!"); + throw e; + } + + List<File> assignmentFiles = new ArrayList<File>(); + Stack<File> files = new Stack<File>(); + files.add(assignmentsDirectory); + while (!files.isEmpty()) { + File file = files.pop(); + if (file.isDirectory()) { + files.addAll(Arrays.asList(file.listFiles())); + } else { + assignmentFiles.add(file); + } + } + + SimpleDateFormat assignmentFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + assignmentFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + SimpleDateFormat filenameFormat = + new SimpleDateFormat("yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss"); + filenameFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + for (File assignmentFile : assignmentFiles) { + try { + BufferedReader br = new BufferedReader(new FileReader( + assignmentFile)); + String line, bridgePoolAssignmentLine = null; + SortedSet<String> sanitizedAssignments = new TreeSet<String>(); + boolean wroteLastLine = false; + while ((line = br.readLine()) != null || !wroteLastLine) { + if (line == null || + line.startsWith("bridge-pool-assignment ")) { + if (bridgePoolAssignmentLine != null) { + try { + long bridgePoolAssignmentTime = assignmentFormat.parse( + bridgePoolAssignmentLine.substring( + "bridge-pool-assignment ".length())).getTime(); + File sanitizedAssignmentsFile = new File( + sanitizedAssignmentsDirectory, filenameFormat.format( + bridgePoolAssignmentTime)); + if (!sanitizedAssignmentsFile.exists()) { + sanitizedAssignmentsFile.getParentFile().mkdirs(); + BufferedWriter bw = new BufferedWriter(new FileWriter( + sanitizedAssignmentsFile)); + bw.write(bridgePoolAssignmentLine + "\n"); + for (String assignmentLine : sanitizedAssignments) { + bw.write(assignmentLine + "\n"); + } + bw.close(); + } + } catch (IOException e) { + logger.log(Level.WARNING, "Could not write sanitized " + + "bridge pool assignment file for line '" + + bridgePoolAssignmentLine + "' to disk. Skipping " + + "bridge pool assignment file '" + + assignmentFile.getAbsolutePath() + "'.", e); + break; + } catch (ParseException e) { + logger.log(Level.WARNING, "Could not write sanitized " + + "bridge pool assignment file for line '" + + bridgePoolAssignmentLine + "' to disk. Skipping " + + "bridge pool assignment file '" + + assignmentFile.getAbsolutePath() + "'.", e); + break; + } + sanitizedAssignments.clear(); + } + if (line == null) { + wroteLastLine = true; + } else { + bridgePoolAssignmentLine = line; + } + } else { + String[] parts = line.split(" "); + if (parts.length < 2 || parts[0].length() < 40) { + logger.warning("Unrecognized line '" + line + + "'. Skipping."); + continue; + } + String hashedFingerprint = null; + try { + hashedFingerprint = DigestUtils.shaHex(Hex.decodeHex( + line.split(" ")[0].toCharArray())).toLowerCase(); + } catch (DecoderException e) { + logger.warning("Unable to decode hex fingerprint in line '" + + line + "'. Skipping."); + continue; + } + String assignmentDetails = line.substring(40); + sanitizedAssignments.add(hashedFingerprint + + assignmentDetails); + } + } + br.close(); + } catch (IOException e) { + logger.log(Level.WARNING, "Could not read bridge pool assignment " + + "file '" + assignmentFile.getAbsolutePath() + + "'. Skipping.", e); + } + } + + logger.info("Finished processing bridge pool assignment file(s)."); + } +} + diff --git a/src/org/torproject/ernie/db/Configuration.java b/src/org/torproject/ernie/db/Configuration.java index c1cdbea..ae24175 100644 --- a/src/org/torproject/ernie/db/Configuration.java +++ b/src/org/torproject/ernie/db/Configuration.java @@ -37,6 +37,9 @@ public class Configuration { + "~gettor/gettor_stats.txt"; private String getTorDirectory = "gettor/"; private boolean downloadExitList = false; + private boolean processBridgePoolAssignments = false; + private String assignmentsDirectory = "assignments/"; + private String sanitizedAssignmentsDirectory = "sanitized-assignments/"; public Configuration() { /* Initialize logger. */ @@ -127,6 +130,13 @@ public class Configuration { } else if (line.startsWith("DownloadExitList")) { this.downloadExitList = Integer.parseInt( line.split(" ")[1]) != 0; + } else if (line.startsWith("ProcessBridgePoolAssignments")) { + this.processBridgePoolAssignments = Integer.parseInt( + line.split(" ")[1]) != 0; + } else if (line.startsWith("AssignmentsDirectory")) { + this.assignmentsDirectory = line.split(" ")[1]; + } else if (line.startsWith("SanitizedAssignmentsDirectory")) { + this.sanitizedAssignmentsDirectory = line.split(" ")[1]; } else { logger.severe("Configuration file contains unrecognized " + "configuration key in line '" + line + "'! Exiting!"); @@ -156,8 +166,8 @@ public class Configuration { if (!this.importCachedRelayDescriptors && !this.importDirectoryArchives && !this.downloadRelayDescriptors && !this.importBridgeSnapshots && !this.downloadGetTorStats && - !this.downloadExitList && !this.writeDirectoryArchives && - !this.writeSanitizedBridges) { + !this.downloadExitList && !this.processBridgePoolAssignments && + !this.writeDirectoryArchives && !this.writeSanitizedBridges) { logger.warning("We have not been configured to read data from any " + "data source or write data to any data sink. You need to " + "edit your config file (" + configFile.getAbsolutePath() @@ -246,5 +256,14 @@ public class Configuration { public boolean getDownloadExitList() { return this.downloadExitList; } + public boolean getProcessBridgePoolAssignments() { + return processBridgePoolAssignments; + } + public String getAssignmentsDirectory() { + return assignmentsDirectory; + } + public String getSanitizedAssignmentsDirectory() { + return sanitizedAssignmentsDirectory; + } } diff --git a/src/org/torproject/ernie/db/Main.java b/src/org/torproject/ernie/db/Main.java index 657cdfc..50c06bb 100644 --- a/src/org/torproject/ernie/db/Main.java +++ b/src/org/torproject/ernie/db/Main.java @@ -128,6 +128,13 @@ public class Main { new ExitListDownloader(); } + // Process bridge pool assignments + if (config.getProcessBridgePoolAssignments()) { + new BridgePoolAssignmentsProcessor( + new File(config.getAssignmentsDirectory()), + new File(config.getSanitizedAssignmentsDirectory())); + } + // Remove lock file lf.releaseLock();

1 0

r24351: {arm} Inbound connections are only private if it doesn't match any (arm/trunk/src/interface/connections)
by Damian Johnson 13 Mar '11

13 Mar '11

Author: atagar Date: 2011-03-13 21:24:23 +0000 (Sun, 13 Mar 2011) New Revision: 24351 Modified: arm/trunk/src/interface/connections/connEntry.py Log: Inbound connections are only private if it doesn't match any relays (not if it can't be resolved due to duplicates) Modified: arm/trunk/src/interface/connections/connEntry.py =================================================================== --- arm/trunk/src/interface/connections/connEntry.py 2011-03-13 19:11:09 UTC (rev 24350) +++ arm/trunk/src/interface/connections/connEntry.py 2011-03-13 21:24:23 UTC (rev 24351) @@ -299,7 +299,9 @@ # if the connection doesn't belong to a known relay then it might be # client traffic - return self.foreign.getFingerprint() == "UNKNOWN" + conn = torTools.getConn() + allMatches = conn.getRelayFingerprint(self.foreign.getIpAddr(), getAllMatches = True) + return allMatches == [] elif myType == Category.EXIT: # DNS connections exiting us aren't private (since they're hitting our # resolvers). Everything else, however, is.

1 0

r24350: {website} make (broken) arabic pages, so when phobos repushes the webs (website/trunk)
by Roger Dingledine 13 Mar '11

13 Mar '11

Author: arma Date: 2011-03-13 19:11:09 +0000 (Sun, 13 Mar 2011) New Revision: 24350 Modified: website/trunk/Makefile.common Log: make (broken) arabic pages, so when phobos repushes the website he won't erase about/overview.html.ar Modified: website/trunk/Makefile.common =================================================================== --- website/trunk/Makefile.common 2011-03-13 04:58:18 UTC (rev 24349) +++ website/trunk/Makefile.common 2011-03-13 19:11:09 UTC (rev 24350) @@ -18,7 +18,7 @@ -D STABLETAG=$(STABLETAG) #LANGS=ar bms de en es et fa it fi fr ja ko nl no pl pt ru se tr zh-cn -LANGS=en es fr ru +LANGS=en ar es fr ru WMLFILES=$(wildcard $(patsubst %, %/*.wml, $(LANGS))) WMIFILES=$(wildcard $(patsubst %, %/*.wmi, $(LANGS)) $(WMLBASE)/include/*.wmi )

1 0

[metrics-tasks/master] Add code to turn BridgeDB logs into assignment.log files.
by karsten＠torproject.org 13 Mar '11

13 Mar '11

commit e1ece10fe3629d9a7268780f77ff4b0653ff72a6 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sun Mar 13 18:43:24 2011 +0100 Add code to turn BridgeDB logs into assignment.log files. --- task-2537/.gitignore | 4 + task-2537/BridgeDBLogConverter.java | 136 +++++++++++++++++++++++++++++++++++ task-2537/README | 15 ++++ 3 files changed, 155 insertions(+), 0 deletions(-) diff --git a/task-2537/.gitignore b/task-2537/.gitignore new file mode 100644 index 0000000..d169b51 --- /dev/null +++ b/task-2537/.gitignore @@ -0,0 +1,4 @@ +*.log +*.class +*.swp + diff --git a/task-2537/BridgeDBLogConverter.java b/task-2537/BridgeDBLogConverter.java new file mode 100644 index 0000000..5a8d69c --- /dev/null +++ b/task-2537/BridgeDBLogConverter.java @@ -0,0 +1,136 @@ +import java.io.*; +import java.text.*; +import java.util.*; + +public class BridgeDBLogConverter { + public static void main(String[] args) throws Exception { + if (args.length != 3) { + System.out.println("Usage: java BridgeDBLogConverter logfile year " + + "outfile"); + System.exit(1); + } + File logfile = new File(args[0]), outfile = new File(args[2]); + String year = args[1]; + SimpleDateFormat logFormat = new SimpleDateFormat( + "yyyy MMM dd HH:mm:ss"); + SimpleDateFormat isoFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + SimpleDateFormat fileFormat = new SimpleDateFormat( + "yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss"); + logFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + isoFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + fileFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + BufferedReader br = new BufferedReader(new FileReader(logfile)); + BufferedWriter bw = new BufferedWriter(new FileWriter(outfile)); + String line; + SortedMap<String, String> entries = new TreeMap<String, String>(); + long lastTimestamp = -1L; + String fingerprint = null, lastFingerprint = null, type = null, + port = "", flag = ""; + while ((line = br.readLine()) != null) { + long timestamp = logFormat.parse(year + " " + + line.substring(0, 15)).getTime(); + if (timestamp > lastTimestamp + 10L * 60L * 1000L) { + if (lastTimestamp > 0L) { + bw.write("bridge-pool-assignment " + + isoFormat.format(lastTimestamp) + "\n"); + for (String entry : entries.values()) { + bw.write(entry + "\n"); + } + entries.clear(); + } + } + lastTimestamp = timestamp; + String[] parts = line.split(" "); + fingerprint = parts[7]; + String assignment = line.substring(line.indexOf(parts[7]) + 41); + if (!fingerprint.equals(lastFingerprint)) { + if (lastFingerprint != null) { + entries.put(lastFingerprint, lastFingerprint + " " + type + + port + flag); + } + type = null; + port = ""; + flag = ""; + } + if (assignment.startsWith("to IP ")) { + int ring = -1; + if (assignment.startsWith("to IP category ring")) { + ring = 4; // TODO This is fragile! + } else { + ring = Integer.parseInt(assignment.split(" ")[3]) - 1; + } + String newType = "https ring=" + ring; + if (type != null && !type.equals(newType)) { + System.out.println("type inconsistency in line '" + line + "'"); + System.exit(1); + } + type = newType; + if (assignment.endsWith(" (port-443 subring)")) { + String newPort = " port=443"; + if (port.length() > 0 && !port.equals(newPort)) { + System.out.println("port inconsistency in line '" + line + + "'"); + System.exit(1); + } + port = newPort; + } else if (assignment.endsWith(" (stable subring)")) { + String newFlag = " flag=stable"; + if (flag.length() > 0 && !flag.equals(newFlag)) { + System.out.println("flag inconsistency in line '" + line + + "'"); + System.exit(1); + } + flag = newFlag; + } + } else if (assignment.equals("to email ring")) { + String newType = "email"; + if (type != null && !type.equals(newType)) { + System.out.println("type inconsistency in line '" + line + "'"); + System.exit(1); + } + type = newType; + } else if (assignment.startsWith("to Ring ")) { + String newType = "email"; + if (type != null && !type.equals(newType)) { + System.out.println("type inconsistency in line '" + line + "'"); + System.exit(1); + } + type = newType; + if (assignment.equals("to Ring (port-443 subring)")) { + String newPort = " port=443"; + if (port.length() > 0 && !port.equals(newPort)) { + System.out.println("port inconsistency in line '" + line + + "'"); + System.exit(1); + } + port = newPort; + } else if (assignment.equals("to Ring (stable subring)")) { + String newFlag = " flag=stable"; + if (flag.length() > 0 && !flag.equals(newFlag)) { + System.out.println("flag inconsistency in line '" + line + + "'"); + System.exit(1); + } + flag = newFlag; + } else { + System.out.println("type inconsistency in line '" + line + + "'"); + System.exit(1); + } + } else { + type = "unallocated"; + } + lastFingerprint = fingerprint; + } + if (lastTimestamp > 0L) { + bw.write("bridge-pool-assignment " + + isoFormat.format(lastTimestamp) + "\n"); + for (String entry : entries.values()) { + bw.write(entry + "\n"); + } + } + bw.close(); + } +} + diff --git a/task-2537/README b/task-2537/README new file mode 100644 index 0000000..1d5375f --- /dev/null +++ b/task-2537/README @@ -0,0 +1,15 @@ +BridgeDBLogConverter.java + +The BridgeDBLogConverter takes BridgeDB's logs as input and writes +(non-sanitized) bridge pool assignments to disk as output. + + - Compile the Java class, e.g., + $ javac BridgeDBLogConverter.java + + - If your bridgedb.log file contains log lines from more than one + calendar year, split it into as many log files as necessary to ensure + that each file only contains log lines from a single calendar year! + + - Run the Java class, e.g., + $ java BridgeDBLogConverter bridgedb.log 2011 assignments.log +

1 0

[bridgedb/master] Merge branch 'dump-rebased'
by karsten＠torproject.org 13 Mar '11

13 Mar '11

commit 128063053f21ddde0f8c3dbc61a4890bef06618e Merge: daa394e 4d00328 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sun Mar 13 11:35:55 2011 +0100 Merge branch 'dump-rebased' bridgedb.conf | 3 +++ lib/bridgedb/Bridges.py | 43 ++++++++++++++++++++++++++++++++++++++++++- lib/bridgedb/Dist.py | 6 ++++++ lib/bridgedb/Main.py | 18 ++++++++++++++++-- 4 files changed, 67 insertions(+), 3 deletions(-)

1 0

[bridgedb/master] Dump bridge pool assignments to a file for statistics.
by karsten＠torproject.org 13 Mar '11

13 Mar '11

commit 4d00328af81cfe6606c57cf01241c770bd5559d5 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Mar 9 15:17:45 2011 +0100 Dump bridge pool assignments to a file for statistics. With this patch we dump the list of currently running bridges with information about their assigned rings, subrings, and file buckets to a local file. The idea is to publish sanitized versions of these assignment files and analyze how the assignment affects a bridge's usage. The assignment file is written on startup and after receiving a HUP signal and parsing new bridge descriptors. Note that the assignments file is not updated when bridges are dumped to file buckets; in that case the changed assignments to file buckets will be reflected in the assignments file after the next HUP. Also note that the assignment file only contains bridges that are believed to be running from parsing the last network status. As a result, bridges that are contained in file buckets, but that are not believed to be running, won't be contained in the assignment file. --- bridgedb.conf | 3 +++ lib/bridgedb/Bridges.py | 43 ++++++++++++++++++++++++++++++++++++++++++- lib/bridgedb/Dist.py | 6 ++++++ lib/bridgedb/Main.py | 18 ++++++++++++++++-- 4 files changed, 67 insertions(+), 3 deletions(-) diff --git a/bridgedb.conf b/bridgedb.conf index 44422a7..e486e0a 100644 --- a/bridgedb.conf +++ b/bridgedb.conf @@ -30,6 +30,9 @@ DB_LOG_FILE = "./bridgedist.log" # File in which we store our secret HMAC root key. MASTER_KEY_FILE = "./secret_key" +# File to which we dump bridge pool assignments for statistics. +ASSIGNMENTS_FILE = "assignments.log" + # How many clusters do we group IPs in when distributing bridges based on IP? # Note that if PROXY_LIST_FILES is set (below), what we actually do here # is use one higher than the number here, and the extra cluster is used diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index 67db9af..eca6db5 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -211,6 +211,9 @@ class BridgeHolder: def assignmentsArePersistent(self): return True + def dumpAssignments(self, f, description=""): + pass + class BridgeRingParameters: """DOCDOC""" def __init__(self, needPorts=(), needFlags=()): @@ -349,6 +352,15 @@ class BridgeRing(BridgeHolder): return self.bridgesByID.get(fp) + def dumpAssignments(self, f, description=""): + for b in self.bridges.itervalues(): + desc = [ description ] + ident = b.getID() + for tp,val,_,subring in self.subrings: + if subring.getBridgeByID(ident): + desc.append("%s=%s"%(tp,val)) + f.write("%s %s\n"%( toHex(ident), " ".join(desc).strip())) + class FixedBridgeSplitter(BridgeHolder): """A bridgeholder that splits bridges up based on an hmac and assigns them to several sub-bridgeholders with equal probability. @@ -376,19 +388,45 @@ class FixedBridgeSplitter(BridgeHolder): n += len(r) return n + def dumpAssignments(self, f, description=""): + for i,r in zip(xrange(len(self.rings)), self.rings): + r.dumpAssignments(f, "%s ring=%s" % (description, i)) class UnallocatedHolder(BridgeHolder): """A pseudo-bridgeholder that ignores its bridges and leaves them unassigned. """ + def __init__(self): + self.fingerprints = [] + def insert(self, bridge): logging.debug("Leaving %s unallocated", bridge.getConfigLine(True)) + if not bridge.fingerprint in self.fingerprints: + self.fingerprints.append(bridge.fingerprint) def assignmentsArePersistent(self): return False def __len__(self): - return 0 + return len(self.fingerprints) + + def clear(self): + self.fingerprints = [] + + def dumpAssignments(self, f, description=""): + db = bridgedb.Storage.getDB() + allBridges = db.getAllBridges() + for bridge in allBridges: + if bridge.hex_key not in self.fingerprints: + continue + dist = bridge.distributor + desc = [ description ] + if dist.startswith(bridgedb.Bucket.PSEUDO_DISTRI_PREFIX): + dist = dist.replace(bridgedb.Bucket.PSEUDO_DISTRI_PREFIX, "") + desc.append("bucket=%s" % dist) + elif dist != "unallocated": + continue + f.write("%s %s\n" % (bridge.hex_key, " ".join(desc).strip())) class BridgeSplitter(BridgeHolder): """A BridgeHolder that splits incoming bridges up based on an hmac, @@ -470,3 +508,6 @@ class BridgeSplitter(BridgeHolder): ring = self.ringsByName.get(ringname) ring.insert(bridge) + def dumpAssignments(self, f, description=""): + for name,ring in self.ringsByName.iteritems(): + ring.dumpAssignments(f, "%s %s" % (description, name)) diff --git a/lib/bridgedb/Dist.py b/lib/bridgedb/Dist.py index 53fdda0..e11d21e 100644 --- a/lib/bridgedb/Dist.py +++ b/lib/bridgedb/Dist.py @@ -112,6 +112,9 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder): def __len__(self): return sum(len(r) for r in self.rings) + def dumpAssignments(self, f, description=""): + self.splitter.dumpAssignments(f, description) + # These characters are the ones that RFC2822 allows. #ASPECIAL = '!#$%&*+-/=?^_`{|}~' #ASPECIAL += "\\\'" @@ -282,3 +285,6 @@ class EmailBasedDistributor(bridgedb.Bridges.BridgeHolder): else: db.commit() + def dumpAssignments(self, f, description=""): + self.ring.dumpAssignments(f, description) + diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py index 82181b8..99d43b6 100644 --- a/lib/bridgedb/Main.py +++ b/lib/bridgedb/Main.py @@ -9,6 +9,7 @@ This module sets up a bridgedb and starts the servers running. import os import signal import sys +import time import logging import gettext @@ -57,6 +58,8 @@ CONFIG = Conf( N_IP_CLUSTERS = 4, MASTER_KEY_FILE = "./secret_key", + ASSIGNMENTS_FILE = "assignments.log", + FORCE_PORTS = [(443, 1)], FORCE_FLAGS = [("Stable", 1)], PROXY_LIST_FILES = [ ], @@ -197,8 +200,9 @@ def startup(cfg): # Expand any ~ characters in paths in the configuration. cfg.BRIDGE_FILES = [ os.path.expanduser(fn) for fn in cfg.BRIDGE_FILES ] for key in ("RUN_IN_DIR", "DB_FILE", "DB_LOG_FILE", "MASTER_KEY_FILE", - "HTTPS_CERT_FILE", "HTTPS_KEY_FILE", "PIDFILE", "LOGFILE", - "STATUS_FILE"): + "ASSIGNMENTS_FILE", "HTTPS_CERT_FILE", "HTTPS_KEY_FILE", + "PIDFILE", "LOGFILE", "STATUS_FILE"): + v = getattr(cfg, key, None) if v: setattr(cfg, key, os.path.expanduser(v)) @@ -299,6 +303,16 @@ def startup(cfg): for name, b in r.bridges.items(): logging.info("%s" % b.getConfigLine(True)) + # Dump bridge pool assignments to disk. + try: + f = open(cfg.ASSIGNMENTS_FILE, 'a') + f.write("bridge-pool-assignment %s\n" % + time.strftime("%Y-%m-%d %H:%M:%S")) + splitter.dumpAssignments(f) + f.close() + except IOError: + logging.info("I/O error while writing assignments") + global _reloadFn _reloadFn = reload signal.signal(signal.SIGHUP, _handleSIGHUP)

1 0

r24349: {arm} Rearranging connection panel resources, abstracting the cont (in arm/trunk/src: interface interface/connections util)
by Damian Johnson 13 Mar '11

13 Mar '11

Author: atagar Date: 2011-03-13 04:58:18 +0000 (Sun, 13 Mar 2011) New Revision: 24349 Added: arm/trunk/src/interface/connections/connEntry.py arm/trunk/src/interface/connections/entries.py Removed: arm/trunk/src/interface/connections/listings.py Modified: arm/trunk/src/interface/connections/__init__.py arm/trunk/src/interface/connections/connPanel.py arm/trunk/src/interface/controller.py arm/trunk/src/util/connections.py arm/trunk/src/util/enum.py arm/trunk/src/util/uiTools.py Log: Rearranging connection panel resources, abstracting the content away from the panel itself. This is to make it more extendable and supporting of multi-line entries (pre-reqs for my plans to display client circuits). Modified: arm/trunk/src/interface/connections/__init__.py =================================================================== --- arm/trunk/src/interface/connections/__init__.py 2011-03-13 01:38:32 UTC (rev 24348) +++ arm/trunk/src/interface/connections/__init__.py 2011-03-13 04:58:18 UTC (rev 24349) @@ -2,5 +2,5 @@ Panels, popups, and handlers comprising the arm user interface. """ -__all__ = ["connPanel", "entry"] +__all__ = ["connEntry", "connPanel", "entries"] Added: arm/trunk/src/interface/connections/connEntry.py =================================================================== --- arm/trunk/src/interface/connections/connEntry.py (rev 0) +++ arm/trunk/src/interface/connections/connEntry.py 2011-03-13 04:58:18 UTC (rev 24349) @@ -0,0 +1,694 @@ +""" +Connection panel entries related to actual connections to or from the system +(ie, results seen by netstat, lsof, etc). +""" + +import time +import curses + +from util import connections, enum, torTools, uiTools +from interface.connections import entries + +# Connection Categories: +# Inbound Relay connection, coming to us. +# Outbound Relay connection, leaving us. +# Exit Outbound relay connection leaving the Tor network. +# Client Circuits for our client traffic. +# Application Socks connections using Tor. +# Directory Fetching tor consensus information. +# Control Tor controller (arm, vidalia, etc). + +Category = enum.Enum("INBOUND", "OUTBOUND", "EXIT", "CLIENT", "APPLICATION", "DIRECTORY", "CONTROL") +CATEGORY_COLOR = {Category.INBOUND: "green", Category.OUTBOUND: "blue", + Category.EXIT: "red", Category.CLIENT: "cyan", + Category.APPLICATION: "yellow", Category.DIRECTORY: "magenta", + Category.CONTROL: "red"} + +# static data for listing format +# <src> --> <dst> <etc><padding> +LABEL_FORMAT = "%s --> %s %s%s" +LABEL_MIN_PADDING = 2 # min space between listing label and following data + +CONFIG = {"features.connection.showColumn.fingerprint": True, + "features.connection.showColumn.nickname": True, + "features.connection.showColumn.destination": True, + "features.connection.showColumn.expanedIp": True} + +def loadConfig(config): + config.update(CONFIG) + +class Endpoint: + """ + Collection of attributes associated with a connection endpoint. This is a + thin wrapper for torUtil functions, making use of its caching for + performance. + """ + + def __init__(self, ipAddr, port): + self.ipAddr = ipAddr + self.port = port + + # if true, we treat the port as an ORPort when searching for matching + # fingerprints (otherwise the ORPort is assumed to be unknown) + self.isORPort = False + + def getIpAddr(self): + """ + Provides the IP address of the endpoint. + """ + + return self.ipAddr + + def getPort(self): + """ + Provides the port of the endpoint. + """ + + return self.port + + def getHostname(self, default = None): + """ + Provides the hostname associated with the relay's address. This is a + non-blocking call and returns None if the address either can't be resolved + or hasn't been resolved yet. + + Arguments: + default - return value if no hostname is available + """ + + # TODO: skipping all hostname resolution to be safe for now + #try: + # myHostname = hostnames.resolve(self.ipAddr) + #except: + # # either a ValueError or IOError depending on the source of the lookup failure + # myHostname = None + # + #if not myHostname: return default + #else: return myHostname + + return default + + def getLocale(self): + """ + Provides the two letter country code for the IP address' locale. This + proivdes None if it can't be determined. + """ + + conn = torTools.getConn() + return conn.getInfo("ip-to-country/%s" % self.ipAddr) + + def getFingerprint(self): + """ + Provides the fingerprint of the relay, returning "UNKNOWN" if it can't be + determined. + """ + + conn = torTools.getConn() + orPort = self.port if self.isORPort else None + myFingerprint = conn.getRelayFingerprint(self.ipAddr, orPort) + + if myFingerprint: return myFingerprint + else: return "UNKNOWN" + + def getNickname(self): + """ + Provides the nickname of the relay, retuning "UNKNOWN" if it can't be + determined. + """ + + conn = torTools.getConn() + orPort = self.port if self.isORPort else None + myFingerprint = conn.getRelayFingerprint(self.ipAddr, orPort) + + if myFingerprint: return conn.getRelayNickname(myFingerprint) + else: return "UNKNOWN" + +class ConnectionEntry(entries.ConnectionPanelEntry): + """ + Represents a connection being made to or from this system. These only + concern real connections so it includes the inbound, outbound, directory, + application, and controller categories. + """ + + def __init__(self, lIpAddr, lPort, fIpAddr, fPort): + entries.ConnectionPanelEntry.__init__(self) + self.lines = [ConnectionLine(lIpAddr, lPort, fIpAddr, fPort)] + + def getSortValue(self, attr, listingType): + """ + Provides the value of a single attribute used for sorting purposes. + """ + + if attr == entries.SortAttr.IP_ADDRESS: + return self.lines[0].sortIpAddr + elif attr == entries.SortAttr.PORT: + return self.lines[0].sortPort + elif attr == entries.SortAttr.HOSTNAME: + return self.lines[0].foreign.getHostname("") + elif attr == entries.SortAttr.FINGERPRINT: + return self.lines[0].foreign.getFingerprint() + elif attr == entries.SortAttr.NICKNAME: + myNickname = self.lines[0].foreign.getNickname() + if myNickname == "UNKNOWN": return "z" * 20 # orders at the end + else: return myNickname.lower() + elif attr == entries.SortAttr.CATEGORY: + return Category.indexOf(self.lines[0].getType()) + elif attr == entries.SortAttr.UPTIME: + return self.lines[0].startTime + elif attr == entries.SortAttr.COUNTRY: + if connections.isIpAddressPrivate(self.lines[0].foreign.getIpAddr()): return "" + else: return self.lines[0].foreign.getLocale() + else: + return entries.ConnectionPanelEntry.getSortValue(self, attr, listingType) + +class ConnectionLine(entries.ConnectionPanelLine): + """ + Display component of the ConnectionEntry. + """ + + def __init__(self, lIpAddr, lPort, fIpAddr, fPort): + entries.ConnectionPanelLine.__init__(self) + + self.local = Endpoint(lIpAddr, lPort) + self.foreign = Endpoint(fIpAddr, fPort) + self.startTime = time.time() + + # True if the connection has matched the properties of a client/directory + # connection every time we've checked. The criteria we check is... + # client - first hop in an established circuit + # directory - matches an established single-hop circuit (probably a + # directory mirror) + + self._possibleClient = True + self._possibleDirectory = True + + conn = torTools.getConn() + myOrPort = conn.getOption("ORPort") + myDirPort = conn.getOption("DirPort") + mySocksPort = conn.getOption("SocksPort", "9050") + myCtlPort = conn.getOption("ControlPort") + + # the ORListenAddress can overwrite the ORPort + listenAddr = conn.getOption("ORListenAddress") + if listenAddr and ":" in listenAddr: + myOrPort = listenAddr[listenAddr.find(":") + 1:] + + if lPort in (myOrPort, myDirPort): + self.baseType = Category.INBOUND + self.local.isORPort = True + elif lPort == mySocksPort: + self.baseType = Category.APPLICATION + elif lPort == myCtlPort: + self.baseType = Category.CONTROL + else: + self.baseType = Category.OUTBOUND + self.foreign.isORPort = True + + self.cachedType = None + + # cached immutable values used for sorting + self.sortIpAddr = connections.ipToInt(self.foreign.getIpAddr()) + self.sortPort = int(self.foreign.getPort()) + + def getListingEntry(self, width, currentTime, listingType): + """ + Provides the DrawEntry for this connection's listing. The line is made up + of six components: + <src> --> <dst> <etc> <uptime> (<type>) + + ListingType.IP_ADDRESS: + src - <internal addr:port> --> <external addr:port> + dst - <destination addr:port> + etc - <fingerprint> <nickname> + + ListingType.HOSTNAME: + src - localhost:<port> + dst - <destination hostname:port> + etc - <destination addr:port> <fingerprint> <nickname> + + ListingType.FINGERPRINT: + src - localhost + dst - <destination fingerprint> + etc - <nickname> <destination addr:port> + + ListingType.NICKNAME: + src - <source nickname> + dst - <destination nickname> + etc - <fingerprint> <destination addr:port> + + Arguments: + width - maximum length of the line + currentTime - unix timestamp for what the results should consider to be + the current time + listingType - primary attribute we're listing connections by + """ + + # fetch our (most likely cached) display entry for the listing + myListing = entries.ConnectionPanelLine.getListingEntry(self, width, currentTime, listingType) + + # fill in the current uptime and return the results + timeEntry = myListing.getNext() + timeEntry.text = "%5s" % uiTools.getTimeLabel(currentTime - self.startTime, 1) + + return myListing + + def _getListingEntry(self, width, currentTime, listingType): + entryType = self.getType() + + # Lines are split into the following components in reverse: + # content - "<src> --> <dst> <etc> " + # time - "<uptime>" + # preType - " (" + # category - "<type>" + # postType - ") " + + lineFormat = uiTools.getColor(CATEGORY_COLOR[entryType]) + + drawEntry = uiTools.DrawEntry(")" + " " * (9 - len(entryType)), lineFormat) + drawEntry = uiTools.DrawEntry(entryType.upper(), lineFormat | curses.A_BOLD, drawEntry) + drawEntry = uiTools.DrawEntry(" (", lineFormat, drawEntry) + drawEntry = uiTools.DrawEntry(" " * 5, lineFormat, drawEntry) + drawEntry = uiTools.DrawEntry(self._getListingContent(width - 17, listingType), lineFormat, drawEntry) + return drawEntry + + def _getDetails(self, width): + """ + Provides details on the connection, correlated against available consensus + data. + + Arguments: + width - available space to display in + """ + + detailFormat = curses.A_BOLD | uiTools.getColor(CATEGORY_COLOR[self.getType()]) + return [uiTools.DrawEntry(line, detailFormat) for line in self._getDetailContent(width)] + + def resetDisplay(self): + entries.ConnectionPanelLine.resetDisplay(self) + self.cachedType = None + + def isPrivate(self): + """ + Returns true if the endpoint is private, possibly belonging to a client + connection or exit traffic. + """ + + myType = self.getType() + + if myType == Category.INBOUND: + # if the connection doesn't belong to a known relay then it might be + # client traffic + + return self.foreign.getFingerprint() == "UNKNOWN" + elif myType == Category.EXIT: + # DNS connections exiting us aren't private (since they're hitting our + # resolvers). Everything else, however, is. + + # TODO: Ideally this would also double check that it's a UDP connection + # (since DNS is the only UDP connections Tor will relay), however this + # will take a bit more work to propagate the information up from the + # connection resolver. + return self.foreign.getPort() != "53" + + # for everything else this isn't a concern + return False + + def getType(self): + """ + Provides our best guess at the current type of the connection. This + depends on consensus results, our current client circuts, etc. Results + are cached until this entry's display is reset. + """ + + # caches both to simplify the calls and to keep the type consistent until + # we want to reflect changes + if not self.cachedType: + if self.baseType == Category.OUTBOUND: + # Currently the only non-static categories are OUTBOUND vs... + # - EXIT since this depends on the current consensus + # - CLIENT if this is likely to belong to our guard usage + # - DIRECTORY if this is a single-hop circuit (directory mirror?) + # + # The exitability, circuits, and fingerprints are all cached by the + # torTools util keeping this a quick lookup. + + conn = torTools.getConn() + destFingerprint = self.foreign.getFingerprint() + + if destFingerprint == "UNKNOWN": + # Not a known relay. This might be an exit connection. + + if conn.isExitingAllowed(self.foreign.getIpAddr(), self.foreign.getPort()): + self.cachedType = Category.EXIT + elif self._possibleClient or self._possibleDirectory: + # This belongs to a known relay. If we haven't eliminated ourselves as + # a possible client or directory connection then check if it still + # holds true. + + myCircuits = conn.getCircuits() + + if self._possibleClient: + # Checks that this belongs to the first hop in a circuit that's + # either unestablished or longer than a single hop (ie, anything but + # a built 1-hop connection since those are most likely a directory + # mirror). + + for status, _, path in myCircuits: + if path[0] == destFingerprint and (status != "BUILT" or len(path) > 1): + self.cachedType = Category.CLIENT # matched a probable guard connection + + # if we fell through, we can eliminate ourselves as a guard in the future + if not self.cachedType: + self._possibleClient = False + + if self._possibleDirectory: + # Checks if we match a built, single hop circuit. + + for status, _, path in myCircuits: + if path[0] == destFingerprint and status == "BUILT" and len(path) == 1: + self.cachedType = Category.DIRECTORY + + # if we fell through, eliminate ourselves as a directory connection + if not self.cachedType: + self._possibleDirectory = False + + if not self.cachedType: + self.cachedType = self.baseType + + return self.cachedType + + def _getListingContent(self, width, listingType): + """ + Provides the source, destination, and extra info for our listing. + + Arguments: + width - maximum length of the line + listingType - primary attribute we're listing connections by + """ + + conn = torTools.getConn() + myType = self.getType() + dstAddress = self._getDestinationLabel(26, includeLocale = True) + + # The required widths are the sum of the following: + # - room for LABEL_FORMAT and LABEL_MIN_PADDING (11 characters) + # - base data for the listing + # - that extra field plus any previous + + usedSpace = len(LABEL_FORMAT % tuple([""] * 4)) + LABEL_MIN_PADDING + + src, dst, etc = "", "", "" + if listingType == entries.ListingType.IP_ADDRESS: + myExternalIpAddr = conn.getInfo("address", self.local.getIpAddr()) + addrDiffer = myExternalIpAddr != self.local.getIpAddr() + + srcAddress = "%s:%s" % (myExternalIpAddr, self.local.getPort()) + src = "%-21s" % srcAddress # ip:port = max of 21 characters + dst = "%-26s" % dstAddress # ip:port (xx) = max of 26 characters + + usedSpace += len(src) + len(dst) # base data requires 47 characters + + if width > usedSpace + 42 and CONFIG["features.connection.showColumn.fingerprint"]: + # show fingerprint (column width: 42 characters) + etc += "%-40s " % self.foreign.getFingerprint() + usedSpace += 42 + + if addrDiffer and width > usedSpace + 28 and CONFIG["features.connection.showColumn.expanedIp"]: + # include the internal address in the src (extra 28 characters) + internalAddress = "%s:%s" % (self.local.getIpAddr(), self.local.getPort()) + src = "%-21s --> %s" % (internalAddress, src) + usedSpace += 28 + + if width > usedSpace + 10 and CONFIG["features.connection.showColumn.nickname"]: + # show nickname (column width: remainder) + nicknameSpace = width - usedSpace + nicknameLabel = uiTools.cropStr(self.foreign.getNickname(), nicknameSpace, 0) + etc += ("%%-%is " % nicknameSpace) % nicknameLabel + usedSpace += nicknameSpace + 2 + elif listingType == entries.ListingType.HOSTNAME: + # 15 characters for source, and a min of 40 reserved for the destination + src = "localhost:%-5s" % self.local.getPort() + usedSpace += len(src) + minHostnameSpace = 40 + + if width > usedSpace + minHostnameSpace + 28 and CONFIG["features.connection.showColumn.destination"]: + # show destination ip/port/locale (column width: 28 characters) + etc += "%-26s " % dstAddress + usedSpace += 28 + + if width > usedSpace + minHostnameSpace + 42 and CONFIG["features.connection.showColumn.fingerprint"]: + # show fingerprint (column width: 42 characters) + etc += "%-40s " % self.foreign.getFingerprint() + usedSpace += 42 + + if width > usedSpace + minHostnameSpace + 17 and CONFIG["features.connection.showColumn.nickname"]: + # show nickname (column width: min 17 characters, uses half of the remainder) + nicknameSpace = 15 + (width - (usedSpace + minHostnameSpace + 17)) / 2 + nicknameLabel = uiTools.cropStr(self.foreign.getNickname(), nicknameSpace, 0) + etc += ("%%-%is " % nicknameSpace) % nicknameLabel + usedSpace += (nicknameSpace + 2) + + hostnameSpace = width - usedSpace + usedSpace = width # prevents padding at the end + if self.isPrivate(): + dst = ("%%-%is" % hostnameSpace) % "<scrubbed>" + else: + hostname = self.foreign.getHostname(self.foreign.getIpAddr()) + port = self.foreign.getPort() + + # truncates long hostnames and sets dst to <hostname>:<port> + hostname = uiTools.cropStr(hostname, hostnameSpace, 0) + dst = "%s:%-5s" % (hostname, port) + dst = ("%%-%is" % hostnameSpace) % dst + elif listingType == entries.ListingType.FINGERPRINT: + src = "localhost" + if myType == Category.CONTROL: dst = "localhost" + else: dst = self.foreign.getFingerprint() + dst = "%-40s" % dst + + usedSpace += len(src) + len(dst) # base data requires 49 characters + + if width > usedSpace + 17: + # show nickname (column width: min 17 characters, consumes any remaining space) + nicknameSpace = width - usedSpace + + # if there's room then also show a column with the destination + # ip/port/locale (column width: 28 characters) + isIpLocaleIncluded = width > usedSpace + 45 + isIpLocaleIncluded &= CONFIG["features.connection.showColumn.destination"] + if isIpLocaleIncluded: nicknameSpace -= 28 + + if CONFIG["features.connection.showColumn.nickname"]: + nicknameSpace = width - usedSpace - 28 if isIpLocaleIncluded else width - usedSpace + nicknameLabel = uiTools.cropStr(self.foreign.getNickname(), nicknameSpace, 0) + etc += ("%%-%is " % nicknameSpace) % nicknameLabel + usedSpace += nicknameSpace + 2 + + if isIpLocaleIncluded: + etc += "%-26s " % dstAddress + usedSpace += 28 + else: + # base data requires 50 min characters + src = self.local.getNickname() + if myType == Category.CONTROL: dst = self.local.getNickname() + else: dst = self.foreign.getNickname() + minBaseSpace = 50 + + if width > usedSpace + minBaseSpace + 42 and CONFIG["features.connection.showColumn.fingerprint"]: + # show fingerprint (column width: 42 characters) + etc += "%-40s " % self.foreign.getFingerprint() + usedSpace += 42 + + if width > usedSpace + minBaseSpace + 28 and CONFIG["features.connection.showColumn.destination"]: + # show destination ip/port/locale (column width: 28 characters) + etc += "%-26s " % dstAddress + usedSpace += 28 + + baseSpace = width - usedSpace + usedSpace = width # prevents padding at the end + + if len(src) + len(dst) > baseSpace: + src = uiTools.cropStr(src, baseSpace / 3) + dst = uiTools.cropStr(dst, baseSpace - len(src)) + + # pads dst entry to its max space + dst = ("%%-%is" % (baseSpace - len(src))) % dst + + if myType == Category.INBOUND: src, dst = dst, src + padding = " " * (width - usedSpace + LABEL_MIN_PADDING) + return LABEL_FORMAT % (src, dst, etc, padding) + + def _getDetailContent(self, width): + """ + Provides a list with detailed information for this connectoin. + + Arguments: + width - max length of lines + """ + + lines = [""] * 7 + lines[0] = "address: %s" % self._getDestinationLabel(width - 11) + lines[1] = "locale: %s" % ("??" if self.isPrivate() else self.foreign.getLocale()) + + # Remaining data concerns the consensus results, with three possible cases: + # - if there's a single match then display its details + # - if there's multiple potenial relays then list all of the combinations + # of ORPorts / Fingerprints + # - if no consensus data is available then say so (probably a client or + # exit connection) + + fingerprint = self.foreign.getFingerprint() + conn = torTools.getConn() + + if fingerprint != "UNKNOWN": + # single match - display information available about it + nsEntry = conn.getConsensusEntry(fingerprint) + descEntry = conn.getDescriptorEntry(fingerprint) + + # append the fingerprint to the second line + lines[1] = "%-13sfingerprint: %s" % (lines[1], fingerprint) + + if nsEntry: + # example consensus entry: + # r murble R8sCM1ar1sS2GulQYFVmvN95xsk RJr6q+wkTFG+ng5v2bdCbVVFfA4 2011-02-21 00:25:32 195.43.157.85 443 0 + # s Exit Fast Guard Named Running Stable Valid + # w Bandwidth=2540 + # p accept 20-23,43,53,79-81,88,110,143,194,443 + + nsLines = nsEntry.split("\n") + + firstLineComp = nsLines[0].split(" ") + if len(firstLineComp) >= 9: + _, nickname, _, _, pubDate, pubTime, _, orPort, dirPort = firstLineComp[:9] + else: nickname, pubDate, pubTime, orPort, dirPort = "", "", "", "", "" + + flags = nsLines[1][2:] + microExit = nsLines[3][2:] + + dirPortLabel = "" if dirPort == "0" else "dirport: %s" % dirPort + lines[2] = "nickname: %-25s orport: %-10s %s" % (nickname, orPort, dirPortLabel) + lines[3] = "published: %s %s" % (pubDate, pubTime) + lines[4] = "flags: %s" % flags.replace(" ", ", ") + lines[5] = "exit policy: %s" % microExit.replace(",", ", ") + + if descEntry: + torVersion, platform, contact = "", "", "" + + for descLine in descEntry.split("\n"): + if descLine.startswith("platform"): + # has the tor version and platform, ex: + # platform Tor 0.2.1.29 (r318f470bc5f2ad43) on Linux x86_64 + + torVersion = descLine[13:descLine.find(" ", 13)] + platform = descLine[descLine.rfind(" on ") + 4:] + elif descLine.startswith("contact"): + contact = descLine[8:] + + # clears up some highly common obscuring + for alias in (" at ", " AT "): contact = contact.replace(alias, "@") + for alias in (" dot ", " DOT "): contact = contact.replace(alias, ".") + + break # contact lines come after the platform + + lines[3] = "%-35s os: %-14s version: %s" % (lines[3], platform, torVersion) + + # contact information is an optional field + if contact: lines[6] = "contact: %s" % contact + else: + allMatches = conn.getRelayFingerprint(self.foreign.getIpAddr(), getAllMatches = True) + + if allMatches: + # multiple matches + lines[2] = "Muliple matches, possible fingerprints are:" + + for i in range(len(allMatches)): + isLastLine = i == 3 + + relayPort, relayFingerprint = allMatches[i] + lineText = "%i. or port: %-5s fingerprint: %s" % (i, relayPort, relayFingerprint) + + # if there's multiple lines remaining at the end then give a count + remainingRelays = len(allMatches) - i + if isLastLine and remainingRelays > 1: + lineText = "... %i more" % remainingRelays + + lines[3 + i] = lineText + + if isLastLine: break + else: + # no consensus entry for this ip address + lines[2] = "No consensus data found" + + # crops any lines that are too long + for i in range(len(lines)): + lines[i] = uiTools.cropStr(lines[i], width - 2) + + return lines + + def _getDestinationLabel(self, maxLength, includeLocale=False, includeHostname=False): + """ + Provides a short description of the destination. This is made up of two + components, the base <ip addr>:<port> and an extra piece of information in + parentheses. The IP address is scrubbed from private connections. + + Extra information is... + - the port's purpose for exit connections + - the locale and/or hostname if set to do so, the address isn't private, + and isn't on the local network + - nothing otherwise + + Arguments: + maxLength - maximum length of the string returned + includeLocale - possibly includes the locale + includeHostname - possibly includes the hostname + """ + + # destination of the connection + if self.isPrivate(): + dstAddress = "<scrubbed>:%s" % self.foreign.getPort() + else: + dstAddress = "%s:%s" % (self.foreign.getIpAddr(), self.foreign.getPort()) + + # Only append the extra info if there's at least a couple characters of + # space (this is what's needed for the country codes). + if len(dstAddress) + 5 <= maxLength: + spaceAvailable = maxLength - len(dstAddress) - 3 + + if self.getType() == Category.EXIT: + purpose = connections.getPortUsage(self.foreign.getPort()) + + if purpose: + # BitTorrent is a common protocol to truncate, so just use "Torrent" + # if there's not enough room. + if len(purpose) > spaceAvailable and purpose == "BitTorrent": + purpose = "Torrent" + + # crops with a hyphen if too long + purpose = uiTools.cropStr(purpose, spaceAvailable, endType = uiTools.Ending.HYPHEN) + + dstAddress += " (%s)" % purpose + elif not connections.isIpAddressPrivate(self.foreign.getIpAddr()): + extraInfo = [] + + if includeLocale: + foreignLocale = self.foreign.getLocale() + extraInfo.append(foreignLocale) + spaceAvailable -= len(foreignLocale) + 2 + + if includeHostname: + dstHostname = self.foreign.getHostname() + + if dstHostname: + # determines the full space availabe, taking into account the ", " + # dividers if there's multipe pieces of extra data + + maxHostnameSpace = spaceAvailable - 2 * len(extraInfo) + dstHostname = uiTools.cropStr(dstHostname, maxHostnameSpace) + extraInfo.append(dstHostname) + spaceAvailable -= len(dstHostname) + + if extraInfo: + dstAddress += " (%s)" % ", ".join(extraInfo) + + return dstAddress[:maxLength] + Modified: arm/trunk/src/interface/connections/connPanel.py =================================================================== --- arm/trunk/src/interface/connections/connPanel.py 2011-03-13 01:38:32 UTC (rev 24348) +++ arm/trunk/src/interface/connections/connPanel.py 2011-03-13 04:58:18 UTC (rev 24349) @@ -6,8 +6,8 @@ import curses import threading -from interface.connections import listings -from util import connections, enum, log, panel, torTools, uiTools +from interface.connections import entries, connEntry +from util import connections, enum, panel, uiTools DEFAULT_CONFIG = {"features.connection.listingType": 0, "features.connection.refreshRate": 10} @@ -18,7 +18,7 @@ # listing types Listing = enum.Enum(("IP_ADDRESS", "IP Address"), "HOSTNAME", "FINGERPRINT", "NICKNAME") -DEFAULT_SORT_ORDER = (listings.SortAttr.CATEGORY, listings.SortAttr.LISTING, listings.SortAttr.UPTIME) +DEFAULT_SORT_ORDER = (entries.SortAttr.CATEGORY, entries.SortAttr.LISTING, entries.SortAttr.UPTIME) class ConnectionPanel(panel.Panel, threading.Thread): """ @@ -33,12 +33,13 @@ self._sortOrdering = DEFAULT_SORT_ORDER self._config = dict(DEFAULT_CONFIG) + if config: config.update(self._config, { "features.connection.listingType": (0, len(Listing.values()) - 1), "features.connection.refreshRate": 1}) - sortFields = listings.SortAttr.values() + sortFields = entries.SortAttr.values() customOrdering = config.getIntCSV("features.connection.order", None, 3, 0, len(sortFields)) if customOrdering: @@ -48,6 +49,7 @@ self._scroller = uiTools.Scroller(True) self._title = "Connections:" # title line of the panel self._connections = [] # last fetched connections + self._connectionLines = [] # individual lines in the connection listing self._showDetails = False # presents the details panel if true self._lastUpdate = -1 # time the content was last revised @@ -55,13 +57,12 @@ self._pauseTime = None # time when the panel was paused self._halt = False # terminates thread if true self._cond = threading.Condition() # used for pausing the thread + self.valsLock = threading.RLock() # Last sampling received from the ConnectionResolver, used to detect when # it changes. self._lastResourceFetch = -1 - self.valsLock = threading.RLock() - self._update() # populates initial entries # TODO: should listen for tor shutdown @@ -95,6 +96,10 @@ self.valsLock.acquire() if ordering: self._sortOrdering = ordering self._connections.sort(key=lambda i: (i.getSortValues(self._sortOrdering, self._listingType))) + + self._connectionLines = [] + for entry in self._connections: + self._connectionLines += entry.getLines() self.valsLock.release() def setListingType(self, listingType): @@ -109,7 +114,7 @@ self._listingType = listingType # if we're sorting by the listing then we need to resort - if listings.SortAttr.LISTING in self._sortOrdering: + if entries.SortAttr.LISTING in self._sortOrdering: self.setSortOrder() self.valsLock.release() @@ -120,7 +125,7 @@ if uiTools.isScrollKey(key): pageHeight = self.getPreferredSize()[0] - 1 if self._showDetails: pageHeight -= (DETAILS_HEIGHT + 1) - isChanged = self._scroller.handleKey(key, self._connections, pageHeight) + isChanged = self._scroller.handleKey(key, self._connectionLines, pageHeight) if isChanged: self.redraw(True) elif uiTools.isSelectionKey(key): self._showDetails = not self._showDetails @@ -152,14 +157,21 @@ # extra line when showing the detail panel is for the bottom border detailPanelOffset = DETAILS_HEIGHT + 1 if self._showDetails else 0 - isScrollbarVisible = len(self._connections) > height - detailPanelOffset - 1 + isScrollbarVisible = len(self._connectionLines) > height - detailPanelOffset - 1 - scrollLoc = self._scroller.getScrollLoc(self._connections, height - detailPanelOffset - 1) - cursorSelection = self._scroller.getCursorSelection(self._connections) + scrollLoc = self._scroller.getScrollLoc(self._connectionLines, height - detailPanelOffset - 1) + cursorSelection = self._scroller.getCursorSelection(self._connectionLines) # draws the detail panel if currently displaying it if self._showDetails: - self._drawSelectionPanel(cursorSelection, width, isScrollbarVisible) + # This is a solid border unless the scrollbar is visible, in which case a + # 'T' pipe connects the border to the bar. + uiTools.drawBox(self, 0, 0, width, DETAILS_HEIGHT + 2) + if isScrollbarVisible: self.addch(DETAILS_HEIGHT + 1, 1, curses.ACS_TTEE) + + drawEntries = cursorSelection.getDetails(width) + for i in range(min(len(drawEntries), DETAILS_HEIGHT)): + drawEntries[i].render(self, 1 + i, 2) # title label with connection counts title = "Connection Details:" if self._showDetails else self._title @@ -168,38 +180,18 @@ scrollOffset = 0 if isScrollbarVisible: scrollOffset = 3 - self.addScrollBar(scrollLoc, scrollLoc + height - detailPanelOffset - 1, len(self._connections), 1 + detailPanelOffset) + self.addScrollBar(scrollLoc, scrollLoc + height - detailPanelOffset - 1, len(self._connectionLines), 1 + detailPanelOffset) currentTime = self._pauseTime if self._pauseTime else time.time() - for lineNum in range(scrollLoc, len(self._connections)): - entry = self._connections[lineNum] - drawLine = lineNum + detailPanelOffset + 1 - scrollLoc + for lineNum in range(scrollLoc, len(self._connectionLines)): + entryLine = self._connectionLines[lineNum] - entryType = entry.getType() - lineFormat = uiTools.getColor(listings.CATEGORY_COLOR[entryType]) - if entry == cursorSelection: lineFormat |= curses.A_STANDOUT + # hilighting if this is the selected line + extraFormat = curses.A_STANDOUT if entryLine == cursorSelection else curses.A_NORMAL - # Lines are split into three components (prefix, category, and suffix) - # since the category includes the bold attribute (otherwise, all use - # lineFormat). - xLoc = scrollOffset - - # prefix (entry data which is largely static, plus the time label) - # the right content (time and type) takes seventeen columns - entryLabel = entry.getLabel(self._listingType, width - scrollOffset - 17) - timeLabel = uiTools.getTimeLabel(currentTime - entry.startTime, 1) - prefixLabel = "%s%5s (" % (entryLabel, timeLabel) - - self.addstr(drawLine, xLoc, prefixLabel, lineFormat) - xLoc += len(prefixLabel) - - # category - self.addstr(drawLine, xLoc, entryType.upper(), lineFormat | curses.A_BOLD) - xLoc += len(entryType) - - # suffix (ending parentheses plus padding so lines are the same length) - self.addstr(drawLine, xLoc, ")" + " " * (9 - len(entryType)), lineFormat) - + drawEntry = entryLine.getListingEntry(width - scrollOffset, currentTime, self._listingType) + drawLine = lineNum + detailPanelOffset + 1 - scrollLoc + drawEntry.render(self, drawLine, scrollOffset, extraFormat) if drawLine >= height: break self.valsLock.release() @@ -232,25 +224,33 @@ newConnections = [] # preserves any ConnectionEntries they already exist - for conn in self._connections: - connAttr = (conn.local.getIpAddr(), conn.local.getPort(), - conn.foreign.getIpAddr(), conn.foreign.getPort()) - - if connAttr in currentConnections: - newConnections.append(conn) - currentConnections.remove(connAttr) + for entry in self._connections: + if isinstance(entry, connEntry.ConnectionEntry): + connLine = entry.getLines()[0] + connAttr = (connLine.local.getIpAddr(), connLine.local.getPort(), + connLine.foreign.getIpAddr(), connLine.foreign.getPort()) + + if connAttr in currentConnections: + newConnections.append(entry) + currentConnections.remove(connAttr) + # reset any display attributes for the entries we're keeping + for entry in newConnections: + entry.resetDisplay() + # add new entries for any additions for lIp, lPort, fIp, fPort in currentConnections: - newConnections.append(listings.ConnectionEntry(lIp, lPort, fIp, fPort)) + newConnections.append(connEntry.ConnectionEntry(lIp, lPort, fIp, fPort)) # Counts the relays in each of the categories. This also flushes the # type cache for all of the connections (in case its changed since last # fetched). - categoryTypes = listings.Category.values() + categoryTypes = connEntry.Category.values() typeCounts = dict((type, 0) for type in categoryTypes) - for conn in newConnections: typeCounts[conn.getType(True)] += 1 + for entry in newConnections: + if isinstance(entry, connEntry.ConnectionEntry): + typeCounts[entry.getLines()[0].getType()] += 1 # makes labels for all the categories with connections (ie, # "21 outbound", "1 control", etc) @@ -264,116 +264,12 @@ else: self._title = "Connections:" self._connections = newConnections + + self._connectionLines = [] + for entry in self._connections: + self._connectionLines += entry.getLines() + self.setSortOrder() self._lastResourceFetch = currentResolutionCount self.valsLock.release() - - def _drawSelectionPanel(self, selection, width, isScrollbarVisible): - """ - Renders a panel for details on the selected connnection. - """ - - # This is a solid border unless the scrollbar is visible, in which case a - # 'T' pipe connects the border to the bar. - uiTools.drawBox(self, 0, 0, width, DETAILS_HEIGHT + 2) - if isScrollbarVisible: self.addch(DETAILS_HEIGHT + 1, 1, curses.ACS_TTEE) - - selectionFormat = curses.A_BOLD | uiTools.getColor(listings.CATEGORY_COLOR[selection.getType()]) - lines = [""] * 7 - - lines[0] = "address: %s" % selection.getDestinationLabel(width - 11, listings.DestAttr.NONE) - lines[1] = "locale: %s" % ("??" if selection.isPrivate() else selection.foreign.getLocale()) - - # Remaining data concerns the consensus results, with three possible cases: - # - if there's a single match then display its details - # - if there's multiple potenial relays then list all of the combinations - # of ORPorts / Fingerprints - # - if no consensus data is available then say so (probably a client or - # exit connection) - - fingerprint = selection.foreign.getFingerprint() - conn = torTools.getConn() - - if fingerprint != "UNKNOWN": - # single match - display information available about it - nsEntry = conn.getConsensusEntry(fingerprint) - descEntry = conn.getDescriptorEntry(fingerprint) - - # append the fingerprint to the second line - lines[1] = "%-13sfingerprint: %s" % (lines[1], fingerprint) - - if nsEntry: - # example consensus entry: - # r murble R8sCM1ar1sS2GulQYFVmvN95xsk RJr6q+wkTFG+ng5v2bdCbVVFfA4 2011-02-21 00:25:32 195.43.157.85 443 0 - # s Exit Fast Guard Named Running Stable Valid - # w Bandwidth=2540 - # p accept 20-23,43,53,79-81,88,110,143,194,443 - - nsLines = nsEntry.split("\n") - - firstLineComp = nsLines[0].split(" ") - if len(firstLineComp) >= 9: - _, nickname, _, _, pubDate, pubTime, _, orPort, dirPort = firstLineComp[:9] - else: nickname, pubDate, pubTime, orPort, dirPort = "", "", "", "", "" - - flags = nsLines[1][2:] - microExit = nsLines[3][2:] - - dirPortLabel = "" if dirPort == "0" else "dirport: %s" % dirPort - lines[2] = "nickname: %-25s orport: %-10s %s" % (nickname, orPort, dirPortLabel) - lines[3] = "published: %s %s" % (pubDate, pubTime) - lines[4] = "flags: %s" % flags.replace(" ", ", ") - lines[5] = "exit policy: %s" % microExit.replace(",", ", ") - - if descEntry: - torVersion, patform, contact = "", "", "" - - for descLine in descEntry.split("\n"): - if descLine.startswith("platform"): - # has the tor version and platform, ex: - # platform Tor 0.2.1.29 (r318f470bc5f2ad43) on Linux x86_64 - - torVersion = descLine[13:descLine.find(" ", 13)] - platform = descLine[descLine.rfind(" on ") + 4:] - elif descLine.startswith("contact"): - contact = descLine[8:] - - # clears up some highly common obscuring - for alias in (" at ", " AT "): contact = contact.replace(alias, "@") - for alias in (" dot ", " DOT "): contact = contact.replace(alias, ".") - - break # contact lines come after the platform - - lines[3] = "%-36s os: %-14s version: %s" % (lines[3], platform, torVersion) - - # contact information is an optional field - if contact: lines[6] = "contact: %s" % contact - else: - allMatches = conn.getRelayFingerprint(selection.foreign.getIpAddr(), getAllMatches = True) - - if allMatches: - # multiple matches - lines[2] = "Muliple matches, possible fingerprints are:" - - for i in range(len(allMatches)): - isLastLine = i == 3 - - relayPort, relayFingerprint = allMatches[i] - lineText = "%i. or port: %-5s fingerprint: %s" % (i, relayPort, relayFingerprint) - - # if there's multiple lines remaining at the end then give a count - remainingRelays = len(allMatches) - i - if isLastLine and remainingRelays > 1: - lineText = "... %i more" % remainingRelays - - lines[3 + i] = lineText - - if isLastLine: break - else: - # no consensus entry for this ip address - lines[2] = "No consensus data found" - - for i in range(len(lines)): - lineText = uiTools.cropStr(lines[i], width - 2) - self.addstr(1 + i, 2, lineText, selectionFormat) Added: arm/trunk/src/interface/connections/entries.py =================================================================== --- arm/trunk/src/interface/connections/entries.py (rev 0) +++ arm/trunk/src/interface/connections/entries.py 2011-03-13 04:58:18 UTC (rev 24349) @@ -0,0 +1,155 @@ +""" +Interface for entries in the connection panel. These consist of two parts: the +entry itself (ie, Tor connection, client circuit, etc) and the lines it +consists of in the listing. +""" + +from util import enum + +# attributes we can list entries by +ListingType = enum.Enum(("IP_ADDRESS", "IP Address"), "HOSTNAME", "FINGERPRINT", "NICKNAME") + +SortAttr = enum.Enum("CATEGORY", "UPTIME", "LISTING", "IP_ADDRESS", "PORT", + "HOSTNAME", "FINGERPRINT", "NICKNAME", "COUNTRY") + +SORT_COLORS = {SortAttr.CATEGORY: "red", SortAttr.UPTIME: "yellow", + SortAttr.LISTING: "green", SortAttr.IP_ADDRESS: "blue", + SortAttr.PORT: "blue", SortAttr.HOSTNAME: "magenta", + SortAttr.FINGERPRINT: "cyan", SortAttr.NICKNAME: "cyan", + SortAttr.COUNTRY: "blue"} + +class ConnectionPanelEntry: + """ + Common parent for connection panel entries. This consists of a list of lines + in the panel listing. This caches results until the display indicates that + they should be flushed. + """ + + def __init__(self): + self.lines = [] + self.flushCache = True + + def getLines(self): + """ + Provides the individual lines in the connection listing. + """ + + if self.flushCache: + self.lines = self._getLines(self.lines) + self.flushCache = False + + return self.lines + + def _getLines(self, oldResults): + # implementation of getLines + + for line in oldResults: + line.resetDisplay() + + return oldResults + + def getSortValues(self, sortAttrs, listingType): + """ + Provides the value used in comparisons to sort based on the given + attribute. + + Arguments: + sortAttrs - list of SortAttr values for the field being sorted on + listingType - ListingType enumeration for the attribute we're listing + entries by + """ + + return [self.getSortValue(attr, listingType) for attr in sortAttrs] + + def getSortValue(self, attr, listingType): + """ + Provides the value of a single attribute used for sorting purposes. + + Arguments: + attr - list of SortAttr values for the field being sorted on + listingType - ListingType enumeration for the attribute we're listing + entries by + """ + + if attr == SortAttr.LISTING: + if listingType == ListingType.IP_ADDRESS: + return self.getSortValue(SortAttr.IP_ADDRESS, listingType) + elif listingType == ListingType.HOSTNAME: + return self.getSortValue(SortAttr.HOSTNAME, listingType) + elif listingType == ListingType.FINGERPRINT: + return self.getSortValue(SortAttr.FINGERPRINT, listingType) + elif listingType == ListingType.NICKNAME: + return self.getSortValue(SortAttr.NICKNAME, listingType) + + return "" + + def resetDisplay(self): + """ + Flushes cached display results. + """ + + self.flushCache = True + +class ConnectionPanelLine: + """ + Individual line in the connection panel listing. + """ + + def __init__(self): + # cache for displayed information + self._listingCache = None + self._listingCacheArgs = (None, None) + + self._detailsCache = None + self._detailsCacheArgs = None + + def getListingEntry(self, width, currentTime, listingType): + """ + Provides a DrawEntry instance for contents to be displayed in the + connection panel listing. + + Arguments: + width - available space to display in + currentTime - unix timestamp for what the results should consider to be + the current time (this may be ignored due to caching) + listingType - ListingType enumeration for the highest priority content + to be displayed + """ + + if self._listingCacheArgs != (width, listingType): + self._listingCache = self._getListingEntry(width, currentTime, listingType) + self._listingCacheArgs = (width, listingType) + + return self._listingCache + + def _getListingEntry(self, width, currentTime, listingType): + # implementation of getListingEntry + return None + + def getDetails(self, width): + """ + Provides a list of DrawEntry instances with detailed information for this + connection. + + Arguments: + width - available space to display in + """ + + if self._detailsCacheArgs != width: + self._detailsCache = self._getDetails(width) + self._detailsCacheArgs = width + + return self._detailsCache + + def _getDetails(self, width): + # implementation of getListing + return [] + + def resetDisplay(self): + """ + Flushes cached display results. + """ + + self._listingCacheArgs = (None, None) + self._detailsCacheArgs = None + Deleted: arm/trunk/src/interface/connections/listings.py =================================================================== --- arm/trunk/src/interface/connections/listings.py 2011-03-13 01:38:32 UTC (rev 24348) +++ arm/trunk/src/interface/connections/listings.py 2011-03-13 04:58:18 UTC (rev 24349) @@ -1,570 +0,0 @@ -""" -Entries for connections related to the Tor process. -""" - -import time - -from util import connections, enum, hostnames, torTools, uiTools - -# Connection Categories: -# Inbound Relay connection, coming to us. -# Outbound Relay connection, leaving us. -# Exit Outbound relay connection leaving the Tor network. -# Client Circuits for our client traffic. -# Application Socks connections using Tor. -# Directory Fetching tor consensus information. -# Control Tor controller (arm, vidalia, etc). - -DestAttr = enum.Enum("NONE", "LOCALE", "HOSTNAME") -Category = enum.Enum("INBOUND", "OUTBOUND", "EXIT", "CLIENT", "APPLICATION", "DIRECTORY", "CONTROL") -CATEGORY_COLOR = {Category.INBOUND: "green", Category.OUTBOUND: "blue", - Category.EXIT: "red", Category.CLIENT: "cyan", - Category.APPLICATION: "yellow", Category.DIRECTORY: "magenta", - Category.CONTROL: "red"} - -SortAttr = enum.Enum("CATEGORY", "UPTIME", "LISTING", "IP_ADDRESS", "PORT", - "HOSTNAME", "FINGERPRINT", "NICKNAME", "COUNTRY") -SORT_COLORS = {SortAttr.CATEGORY: "red", SortAttr.UPTIME: "yellow", - SortAttr.LISTING: "green", SortAttr.IP_ADDRESS: "blue", - SortAttr.PORT: "blue", SortAttr.HOSTNAME: "magenta", - SortAttr.FINGERPRINT: "cyan", SortAttr.NICKNAME: "cyan", - SortAttr.COUNTRY: "blue"} - -# static data for listing format -# <src> --> <dst> <etc><padding> -LABEL_FORMAT = "%s --> %s %s%s" -LABEL_MIN_PADDING = 2 # min space between listing label and following data - -CONFIG = {"features.connection.showColumn.fingerprint": True, - "features.connection.showColumn.nickname": True, - "features.connection.showColumn.destination": True, - "features.connection.showColumn.expanedIp": True} - -def loadConfig(config): - config.update(CONFIG) - -class Endpoint: - """ - Collection of attributes associated with a connection endpoint. This is a - thin wrapper for torUtil functions, making use of its caching for - performance. - """ - - def __init__(self, ipAddr, port): - self.ipAddr = ipAddr - self.port = port - - # if true, we treat the port as an ORPort when searching for matching - # fingerprints (otherwise the ORPort is assumed to be unknown) - self.isORPort = False - - def getIpAddr(self): - """ - Provides the IP address of the endpoint. - """ - - return self.ipAddr - - def getPort(self): - """ - Provides the port of the endpoint. - """ - - return self.port - - def getHostname(self, default = None): - """ - Provides the hostname associated with the relay's address. This is a - non-blocking call and returns None if the address either can't be resolved - or hasn't been resolved yet. - - Arguments: - default - return value if no hostname is available - """ - - # TODO: skipping all hostname resolution to be safe for now - #try: - # myHostname = hostnames.resolve(self.ipAddr) - #except: - # # either a ValueError or IOError depending on the source of the lookup failure - # myHostname = None - # - #if not myHostname: return default - #else: return myHostname - - return default - - def getLocale(self): - """ - Provides the two letter country code for the IP address' locale. This - proivdes None if it can't be determined. - """ - - conn = torTools.getConn() - return conn.getInfo("ip-to-country/%s" % self.ipAddr) - - def getFingerprint(self): - """ - Provides the fingerprint of the relay, returning "UNKNOWN" if it can't be - determined. - """ - - conn = torTools.getConn() - orPort = self.port if self.isORPort else None - myFingerprint = conn.getRelayFingerprint(self.ipAddr, orPort) - - if myFingerprint: return myFingerprint - else: return "UNKNOWN" - - def getNickname(self): - """ - Provides the nickname of the relay, retuning "UNKNOWN" if it can't be - determined. - """ - - conn = torTools.getConn() - orPort = self.port if self.isORPort else None - myFingerprint = conn.getRelayFingerprint(self.ipAddr, orPort) - - if myFingerprint: return conn.getRelayNickname(myFingerprint) - else: return "UNKNOWN" - -class ConnectionEntry: - """ - Represents a connection being made to or from this system. These only - concern real connections so it only includes the inbound, outbound, - directory, application, and controller categories. - """ - - def __init__(self, lIpAddr, lPort, fIpAddr, fPort): - self.local = Endpoint(lIpAddr, lPort) - self.foreign = Endpoint(fIpAddr, fPort) - self.startTime = time.time() - - self._labelCache = "" - self._labelCacheArgs = (None, None) - - # True if the connection has matched the properties of a client/directory - # connection every time we've checked. The criteria we check is... - # client - first hop in an established circuit - # directory - matches an established single-hop circuit (probably a - # directory mirror) - - self._possibleClient = True - self._possibleDirectory = True - - conn = torTools.getConn() - myOrPort = conn.getOption("ORPort") - myDirPort = conn.getOption("DirPort") - mySocksPort = conn.getOption("SocksPort", "9050") - myCtlPort = conn.getOption("ControlPort") - myAuthorities = conn.getMyDirAuthorities() - - # the ORListenAddress can overwrite the ORPort - listenAddr = conn.getOption("ORListenAddress") - if listenAddr and ":" in listenAddr: - myOrPort = listenAddr[listenAddr.find(":") + 1:] - - if lPort in (myOrPort, myDirPort): - self.baseType = Category.INBOUND - self.local.isORPort = True - elif lPort == mySocksPort: - self.baseType = Category.APPLICATION - elif lPort == myCtlPort: - self.baseType = Category.CONTROL - elif (fIpAddr, fPort) in myAuthorities: - self.baseType = Category.DIRECTORY - else: - self.baseType = Category.OUTBOUND - self.foreign.isORPort = True - - self.cachedType = None - - # cached immutable values used for sorting - self.sortIpAddr = _ipToInt(self.foreign.getIpAddr()) - self.sortPort = int(self.foreign.getPort()) - - def getType(self, reset=False): - """ - Provides the category this connection belongs to. This isn't always static - since it can rely on dynamic information (like the current consensus). - - Arguments: - reset - determines if the type has changed if true, otherwise this - provides the same result as the last call - """ - - # caches both to simplify the calls and to keep the type consistent until - # we want to reflect changes - if reset or not self.cachedType: - self.cachedType = self._getType() - - return self.cachedType - - def getDestinationLabel(self, maxLength, extraAttr=DestAttr.NONE): - """ - Provides a short description of the destination. This is made up of two - components, the base <ip addr>:<port> and an extra piece of information in - parentheses. The IP address is scrubbed from private connections. - - Extra information is... - - the port's purpose for exit connections - - the extraAttr if the address isn't private and isn't on the local network - - nothing otherwise - - Arguments: - maxLength - maximum length of the string returned - """ - - # destination of the connection - if self.isPrivate(): - dstAddress = "<scrubbed>:%s" % self.foreign.getPort() - else: - dstAddress = "%s:%s" % (self.foreign.getIpAddr(), self.foreign.getPort()) - - # Only append the extra info if there's at least a couple characters of - # space (this is what's needed for the country codes). - if len(dstAddress) + 5 <= maxLength: - spaceAvailable = maxLength - len(dstAddress) - 3 - - if self.getType() == Category.EXIT: - purpose = connections.getPortUsage(self.foreign.getPort()) - - if purpose: - # BitTorrent is a common protocol to truncate, so just use "Torrent" - # if there's not enough room. - if len(purpose) > spaceAvailable and purpose == "BitTorrent": - purpose = "Torrent" - - # crops with a hyphen if too long - purpose = uiTools.cropStr(purpose, spaceAvailable, endType = uiTools.Ending.HYPHEN) - - dstAddress += " (%s)" % purpose - elif not connections.isIpAddressPrivate(self.foreign.getIpAddr()): - if extraAttr == DestAttr.LOCALE: - dstAddress += " (%s)" % self.foreign.getLocale() - elif extraAttr == DestAttr.HOSTNAME: - dstHostname = self.foreign.getHostname() - - if dstHostname: - dstAddress += " (%s)" % uiTools.cropStr(dstHostname, spaceAvailable) - - return dstAddress[:maxLength] - - def isPrivate(self): - """ - Returns true if the endpoint is private, possibly belonging to a client - connection or exit traffic. - """ - - myType = self.getType() - - if myType == Category.INBOUND: - # if the connection doesn't belong to a known relay then it might be - # client traffic - - return self.foreign.getFingerprint() == "UNKNOWN" - elif myType == Category.EXIT: - # DNS connections exiting us aren't private (since they're hitting our - # resolvers). Everything else, however, is. - - # TODO: Ideally this would also double check that it's a UDP connection - # (since DNS is the only UDP connections Tor will relay), however this - # will take a bit more work to propagate the information up from the - # connection resolver. - return self.foreign.getPort() != "53" - - # for everything else this isn't a concern - return False - - def getSortValues(self, sortAttrs, listingType): - """ - Provides the value used in comparisons to sort based on the given - attribute. - - Arguments: - sortAttrs - list of SortAttr values for the field being sorted on - listingType - primary attribute we're listing connections by - """ - - return [self._getSortValue(attr, listingType) for attr in sortAttrs] - - def getLabel(self, listingType, width): - """ - Provides the formatted display string for this entry in the listing with - the given constraints. Labels are made up of six components: - <src> --> <dst> <etc> <uptime> (<type>) - this provides the first three components padded to fill up to the uptime. - - Listing.IP_ADDRESS: - src - <internal addr:port> --> <external addr:port> - dst - <destination addr:port> - etc - <fingerprint> <nickname> - - Listing.HOSTNAME: - src - localhost:<port> - dst - <destination hostname:port> - etc - <destination addr:port> <fingerprint> <nickname> - - Listing.FINGERPRINT: - src - localhost - dst - <destination fingerprint> - etc - <nickname> <destination addr:port> - - Listing.NICKNAME: - src - <source nickname> - dst - <destination nickname> - etc - <fingerprint> <destination addr:port> - - Arguments: - listingType - primary attribute we're listing connections by - width - maximum length of the entry - """ - - # late import for the Listing enum (doing it in the header errors due to a - # circular import) - from interface.connections import connPanel - - # if our cached entries are still valid then use that - if self._labelCacheArgs == (listingType, width): - return self._labelCache - - conn = torTools.getConn() - myType = self.getType() - dstAddress = self.getDestinationLabel(26, DestAttr.LOCALE) - - # The required widths are the sum of the following: - # - room for LABEL_FORMAT and LABEL_MIN_PADDING (11 characters) - # - base data for the listing - # - that extra field plus any previous - - usedSpace = len(LABEL_FORMAT % tuple([""] * 4)) + LABEL_MIN_PADDING - - src, dst, etc = "", "", "" - if listingType == connPanel.Listing.IP_ADDRESS: - myExternalIpAddr = conn.getInfo("address", self.local.getIpAddr()) - addrDiffer = myExternalIpAddr != self.local.getIpAddr() - - srcAddress = "%s:%s" % (myExternalIpAddr, self.local.getPort()) - src = "%-21s" % srcAddress # ip:port = max of 21 characters - dst = "%-26s" % dstAddress # ip:port (xx) = max of 26 characters - - usedSpace += len(src) + len(dst) # base data requires 47 characters - - if width > usedSpace + 42 and CONFIG["features.connection.showColumn.fingerprint"]: - # show fingerprint (column width: 42 characters) - etc += "%-40s " % self.foreign.getFingerprint() - usedSpace += 42 - - if addrDiffer and width > usedSpace + 28 and CONFIG["features.connection.showColumn.expanedIp"]: - # include the internal address in the src (extra 28 characters) - internalAddress = "%s:%s" % (self.local.getIpAddr(), self.local.getPort()) - src = "%-21s --> %s" % (internalAddress, src) - usedSpace += 28 - - if width > usedSpace + 10 and CONFIG["features.connection.showColumn.nickname"]: - # show nickname (column width: remainder) - nicknameSpace = width - usedSpace - nicknameLabel = uiTools.cropStr(self.foreign.getNickname(), nicknameSpace, 0) - etc += ("%%-%is " % nicknameSpace) % nicknameLabel - usedSpace += nicknameSpace + 2 - elif listingType == connPanel.Listing.HOSTNAME: - # 15 characters for source, and a min of 40 reserved for the destination - src = "localhost:%-5s" % self.local.getPort() - usedSpace += len(src) - minHostnameSpace = 40 - - if width > usedSpace + minHostnameSpace + 28 and CONFIG["features.connection.showColumn.destination"]: - # show destination ip/port/locale (column width: 28 characters) - etc += "%-26s " % dstAddress - usedSpace += 28 - - if width > usedSpace + minHostnameSpace + 42 and CONFIG["features.connection.showColumn.fingerprint"]: - # show fingerprint (column width: 42 characters) - etc += "%-40s " % self.foreign.getFingerprint() - usedSpace += 42 - - if width > usedSpace + minHostnameSpace + 17 and CONFIG["features.connection.showColumn.nickname"]: - # show nickname (column width: min 17 characters, uses half of the remainder) - nicknameSpace = 15 + (width - (usedSpace + minHostnameSpace + 17)) / 2 - nicknameLabel = uiTools.cropStr(self.foreign.getNickname(), nicknameSpace, 0) - etc += ("%%-%is " % nicknameSpace) % nicknameLabel - usedSpace += (nicknameSpace + 2) - - hostnameSpace = width - usedSpace - usedSpace = width # prevents padding at the end - if self.isPrivate(): - dst = ("%%-%is" % hostnameSpace) % "<scrubbed>" - else: - hostname = self.foreign.getHostname(self.foreign.getIpAddr()) - port = self.foreign.getPort() - - # truncates long hostnames and sets dst to <hostname>:<port> - hostname = uiTools.cropStr(hostname, hostnameSpace, 0) - dst = "%s:%-5s" % (hostname, port) - dst = ("%%-%is" % hostnameSpace) % dst - elif listingType == connPanel.Listing.FINGERPRINT: - src = "localhost" - if myType == Category.CONTROL: dst = "localhost" - else: dst = self.foreign.getFingerprint() - dst = "%-40s" % dst - - usedSpace += len(src) + len(dst) # base data requires 49 characters - - if width > usedSpace + 17: - # show nickname (column width: min 17 characters, consumes any remaining space) - nicknameSpace = width - usedSpace - - # if there's room then also show a column with the destination - # ip/port/locale (column width: 28 characters) - isIpLocaleIncluded = width > usedSpace + 45 - isIpLocaleIncluded &= CONFIG["features.connection.showColumn.destination"] - if isIpLocaleIncluded: nicknameSpace -= 28 - - if CONFIG["features.connection.showColumn.nickname"]: - nicknameSpace = width - usedSpace - 28 if isIpLocaleIncluded else width - usedSpace - nicknameLabel = uiTools.cropStr(self.foreign.getNickname(), nicknameSpace, 0) - etc += ("%%-%is " % nicknameSpace) % nicknameLabel - usedSpace += nicknameSpace + 2 - - if isIpLocaleIncluded: - etc += "%-26s " % dstAddress - usedSpace += 28 - else: - # base data requires 50 min characters - src = self.local.getNickname() - if myType == Category.CONTROL: dst = self.local.getNickname() - else: dst = self.foreign.getNickname() - minBaseSpace = 50 - - if width > usedSpace + minBaseSpace + 42 and CONFIG["features.connection.showColumn.fingerprint"]: - # show fingerprint (column width: 42 characters) - etc += "%-40s " % self.foreign.getFingerprint() - usedSpace += 42 - - if width > usedSpace + minBaseSpace + 28 and CONFIG["features.connection.showColumn.destination"]: - # show destination ip/port/locale (column width: 28 characters) - etc += "%-26s " % dstAddress - usedSpace += 28 - - baseSpace = width - usedSpace - usedSpace = width # prevents padding at the end - - if len(src) + len(dst) > baseSpace: - src = uiTools.cropStr(src, baseSpace / 3) - dst = uiTools.cropStr(dst, baseSpace - len(src)) - - # pads dst entry to its max space - dst = ("%%-%is" % (baseSpace - len(src))) % dst - - if myType == Category.INBOUND: src, dst = dst, src - padding = " " * (width - usedSpace + LABEL_MIN_PADDING) - self._labelCache = LABEL_FORMAT % (src, dst, etc, padding) - self._labelCacheArgs = (listingType, width) - - return self._labelCache - - def _getType(self): - """ - Provides our best guess at the current type of the connection. This - depends on consensus results, our current client circuts, etc. - """ - - if self.baseType == Category.OUTBOUND: - # Currently the only non-static categories are OUTBOUND vs... - # - EXIT since this depends on the current consensus - # - CLIENT if this is likely to belong to our guard usage - # - DIRECTORY if this is a single-hop circuit (directory mirror?) - # - # The exitability, circuits, and fingerprints are all cached by the - # torTools util keeping this a quick lookup. - - conn = torTools.getConn() - destFingerprint = self.foreign.getFingerprint() - - if destFingerprint == "UNKNOWN": - # Not a known relay. This might be an exit connection. - - if conn.isExitingAllowed(self.foreign.getIpAddr(), self.foreign.getPort()): - return Category.EXIT - elif self._possibleClient or self._possibleDirectory: - # This belongs to a known relay. If we haven't eliminated ourselves as - # a possible client or directory connection then check if it still - # holds true. - - myCircuits = conn.getCircuits() - - if self._possibleClient: - # Checks that this belongs to the first hop in a circuit that's - # either unestablished or longer than a single hop (ie, anything but - # a built 1-hop connection since those are most likely a directory - # mirror). - - for status, _, path in myCircuits: - if path[0] == destFingerprint and (status != "BUILT" or len(path) > 1): - return Category.CLIENT # matched a probable guard connection - - # fell through, we can eliminate ourselves as a guard in the future - self._possibleClient = False - - if self._possibleDirectory: - # Checks if we match a built, single hop circuit. - - for status, _, path in myCircuits: - if path[0] == destFingerprint and status == "BUILT" and len(path) == 1: - return Category.DIRECTORY - - # fell through, eliminate ourselves as a directory connection - self._possibleDirectory = False - - return self.baseType - - def _getSortValue(self, sortAttr, listingType): - """ - Provides the value of a single attribute used for sorting purposes. - """ - - from interface.connections import connPanel - - if sortAttr == SortAttr.IP_ADDRESS: return self.sortIpAddr - elif sortAttr == SortAttr.PORT: return self.sortPort - elif sortAttr == SortAttr.HOSTNAME: return self.foreign.getHostname("") - elif sortAttr == SortAttr.FINGERPRINT: return self.foreign.getFingerprint() - elif sortAttr == SortAttr.NICKNAME: - myNickname = self.foreign.getNickname() - - if myNickname == "UNKNOWN": return "z" * 20 # orders at the end - else: return myNickname.lower() - elif sortAttr == SortAttr.CATEGORY: return Category.indexOf(self.getType()) - elif sortAttr == SortAttr.UPTIME: return self.startTime - elif sortAttr == SortAttr.COUNTRY: - if connections.isIpAddressPrivate(self.foreign.getIpAddr()): return "" - else: return self.foreign.getLocale() - elif sortAttr == SortAttr.LISTING: - if listingType == connPanel.Listing.IP_ADDRESS: - return self._getSortValue(SortAttr.IP_ADDRESS, listingType) - elif listingType == connPanel.Listing.HOSTNAME: - return self._getSortValue(SortAttr.HOSTNAME, listingType) - elif listingType == connPanel.Listing.FINGERPRINT: - return self._getSortValue(SortAttr.FINGERPRINT, listingType) - elif listingType == connPanel.Listing.NICKNAME: - return self._getSortValue(SortAttr.NICKNAME, listingType) - - return "" - -def _ipToInt(ipAddr): - """ - Provides an integer representation of the ip address, suitable for sorting. - - Arguments: - ipAddr - ip address to be converted - """ - - total = 0 - - for comp in ipAddr.split("."): - total *= 255 - total += int(comp) - - return total - Modified: arm/trunk/src/interface/controller.py =================================================================== --- arm/trunk/src/interface/controller.py 2011-03-13 01:38:32 UTC (rev 24348) +++ arm/trunk/src/interface/controller.py 2011-03-13 04:58:18 UTC (rev 24349) @@ -25,7 +25,8 @@ import fileDescriptorPopup import interface.connections.connPanel -import interface.connections.listings +import interface.connections.connEntry +import interface.connections.entries from util import conf, log, connections, hostnames, panel, sysTools, torConfig, torTools, uiTools import graphing.bandwidthStats import graphing.connStats @@ -425,7 +426,7 @@ config = conf.getConfig("arm") config.update(CONFIG) graphing.graphPanel.loadConfig(config) - interface.connections.listings.loadConfig(config) + interface.connections.connEntry.loadConfig(config) # adds events needed for arm functionality to the torTools REQ_EVENTS mapping # (they're then included with any setControllerEvents call, and log a more @@ -1602,7 +1603,7 @@ panel.CURSES_LOCK.release() elif page == 2 and (key == ord('l') or key == ord('L')): # provides a menu to pick the primary information we list connections by - options = interface.connections.connPanel.Listing.values() + options = interface.connections.entries.ListingType.values() initialSelection = options.index(panels["conn2"]._listingType) # hides top label of connection panel and pauses the display @@ -1624,9 +1625,9 @@ elif page == 2 and (key == ord('s') or key == ord('S')): # set ordering for connection options titleLabel = "Connection Ordering:" - options = interface.connections.listings.SortAttr.values() + options = interface.connections.entries.SortAttr.values() oldSelection = panels["conn2"]._sortOrdering - optionColors = dict([(attr, interface.connections.listings.SORT_COLORS[attr]) for attr in options]) + optionColors = dict([(attr, interface.connections.entries.SORT_COLORS[attr]) for attr in options]) results = showSortDialog(stdscr, panels, isPaused, page, titleLabel, options, oldSelection, optionColors) if results: Modified: arm/trunk/src/util/connections.py =================================================================== --- arm/trunk/src/util/connections.py 2011-03-13 01:38:32 UTC (rev 24348) +++ arm/trunk/src/util/connections.py 2011-03-13 04:58:18 UTC (rev 24349) @@ -153,6 +153,22 @@ return False +def ipToInt(ipAddr): + """ + Provides an integer representation of the ip address, suitable for sorting. + + Arguments: + ipAddr - ip address to be converted + """ + + total = 0 + + for comp in ipAddr.split("."): + total *= 255 + total += int(comp) + + return total + def getPortUsage(port): """ Provides the common use of a given port. If no useage is known then this Modified: arm/trunk/src/util/enum.py =================================================================== --- arm/trunk/src/util/enum.py 2011-03-13 01:38:32 UTC (rev 24348) +++ arm/trunk/src/util/enum.py 2011-03-13 04:58:18 UTC (rev 24349) @@ -12,12 +12,12 @@ >>> pets.DOG 'Skippy' >>> pets.CAT -"Cat" +'Cat' or with entirely custom string components as an unordered enum with: >>> pets = LEnum(DOG="Skippy", CAT="Kitty", FISH="Nemo") >>> pets.CAT -"Kitty" +'Kitty' """ def toCamelCase(label): Modified: arm/trunk/src/util/uiTools.py =================================================================== --- arm/trunk/src/util/uiTools.py 2011-03-13 01:38:32 UTC (rev 24348) +++ arm/trunk/src/util/uiTools.py 2011-03-13 04:58:18 UTC (rev 24349) @@ -409,6 +409,53 @@ except ValueError: raise ValueError(errorMsg) +class DrawEntry: + """ + Renderable content, encapsulating the text and formatting. These can be + chained together to compose lines with multiple types of formatting. + """ + + def __init__(self, text, format=curses.A_NORMAL, nextEntry=None): + self.text = text + self.format = format + self.nextEntry = nextEntry + + def getNext(self): + """ + Provides the next DrawEntry in the chain. + """ + + return self.nextEntry + + def setNext(self, nextEntry): + """ + Sets additional content to be drawn after this entry. If None then + rendering is terminated after this entry. + + Arguments: + nextEntry - DrawEntry instance to be rendered after this one + """ + + self.nextEntry = nextEntry + + def render(self, drawPanel, y, x, extraFormat=curses.A_NORMAL): + """ + Draws this content at the given position. + + Arguments: + drawPanel - context in which to be drawn + y - vertical location + x - horizontal location + extraFormat - additional formatting + """ + + drawFormat = self.format | extraFormat + drawPanel.addstr(y, x, self.text, drawFormat) + + # if there's additional content to show then render it too + if self.nextEntry: + self.nextEntry.render(drawPanel, y, x + len(self.text), extraFormat) + class Scroller: """ Tracks the scrolling position when there might be a visible cursor. This

1 0